sudo touch /etc/ld.so.conf.d/user.conf sudo echo "/usr/local/yourLib/lib" >> /etc/ld.so.conf.d/user.conf sudo ldconfig
export LD_LIBRARY_PATH=/usr/local/yourLib/lib
$ pip install numpy $ pip install scipy $ pip install -U scikit-image $ pip install dlib
# build dlib locally wget http://dlib.net/files/dlib-19.6.tar.bz2 tar xvf dlib-19.9.tar.bz2 cd dlib-19.6/ mkdir build cd build cmake .. cmake --build . --config Release sudo make install # update the link libraries, so that programs knows where to find the dynamic lib during run time sudo ldconfig cd .. # Now you can use pkg-config to provide path to Dlib’s include directory and link Dlib library file. This is useful during build time pkg-config --libs --cflags dlib-1
sudo apt-get install screen sudo apt-get install libxmu-dev libxi-dev libglu1-mesa libglu1-mesa-dev sudo apt-get install libjpeg-dev libpng-dev libtiff-dev sudo apt-get install libavcodec-dev libavformat-dev libswscale-del libv4l-dev sudo apt-get install libavcodec-dev libavformat-dev libswscale-dev libv4l-dev sudo apt-get install libxvidcore-dev libx264-dev sudo apt-get install libopenblas-dev libatlas-base-dev liblapack-dev gfortran sudo apt-get install libhdf5-dev sudo apt-get install python3-dev python3-tk python-pil.imagetk sudo apt-get install libgtk-3-dev
cd ~ mkdir installers cd installers/ wget https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux =========== = Summary = =========== Driver: Not Selected Toolkit: Installed in /usr/local/cuda-10.0 Samples: Installed in /home/sulfred Please make sure that - PATH includes /usr/local/cuda-10.0/bin - LD_LIBRARY_PATH includes /usr/local/cuda-10.0/lib64, or, add /usr/local/cuda-10.0/lib64 to /etc/ld.so.conf and run ldconfig as root To uninstall the CUDA Toolkit, run the uninstall script in /usr/local/cuda-10.0/bin Please see CUDA_Installation_Guide_Linux.pdf in /usr/local/cuda-10.0/doc/pdf for detailed information on setting up CUDA.
vim ~/.bashrc # NVIDIA CUDA Toolkit export PATH=/usr/local/cuda-10.0/bin:$PATH export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64:$LD_LIBRARY_PATH
cd ~/installers tar -zxf cudnn-10.0-linux-x64-v7.4.2.24.tgz cd cuda sudo cp -P lib64/* /usr/local/cuda/lib64/ sudo cp -P include/* /usr/local/cuda/include/
# standard image processing pip install opencv-contrib-python pip install scikit-image pip install pillow pip install imutils # machine learning libraries pip install scikit-learn pip install matplotlib pip install pregressbar2 pip install beautifulsoup4 pip install pandas
#!/bin/bash buildPath="$PWD" # remove old tags rm ${buildPath}/GPATH rm ${buildPath}/GRTAGS rm ${buildPath}/GTAGS rm ${buildPath}/cscope.files rm ${buildPath}/cscope.out rm ${buildPath}/cscope.in.out rm ${buildPath}/cscope.po.out # make tags echo "######### make GTag" gtags echo "######### make cscope" find . -name "*.cc" -o -name "*.c" -o -name "*.cpp" -o -name "*.h" -o -name "*.hpp" > cscope.files cscope -q -R -b -k -i cscope.files
$ cd /path/to/project/root $ prepareSpacemacsTags.sh
#!/bin/bash basePath=`pwd` tempTagPath="$basePath/../tempTagFolder" appFolders=("APPFolderName") # filter out # move all folders beyond root if [ ! -d "$tempTagPath" ]; then mkdir "$tempTagPath" fi if [ ! -d "$tempTagPath/app" ]; then mkdir "$tempTagPath/app" fi for folder in ${appFolders[@]}; do nextFolder="$basePath/app/$folder" mv $nextFolder $tempTagPath/app done prepareSpacemacsTags.sh mv $tempTagPath/app/* ./app if [ -d "$tempTagPath/app" ] && [ -z "$(ls -A $tempTagPath/app/)" ]; then rm -rf $tempTagPath/app else echo "No need to have further action $tempTagPath/app" fi if [ -d "$tempTagPath" ] && [ -z "$(ls -A $tempTagPath)" ]; then rm -rf $tempTagPath else echo "No need to have further action $tempTagPath" fi
$ cd /path/to/project/root $ ls ./app ./lib $ prepareTags.YourProject.sh
# Install gdrive $ wget https://docs.google.com/uc?id=0B3X9GlR6EmbnWksyTEtCM0VfaFE&export=download $ mv uc\?id\=0B3X9GlR6EmbnWksyTEtCM0VfaFE gdrive $ chmod +x gdrive $ sudo install gdrive /usr/local/bin/gdrive
# Connect gdrive with google drive, just use any command, for example $ gdrive list # Follow the steps suggested by gdrive # You can find the token file from $ ls ~/.gdrive
# After that you can upload file to target folder, you can list and see the location $ gdrive list $ gdrive upload --parent die231 backup.txt
Filesystem Size Used Avail Use% Mounted on udev 1.8G 0 1.8G 0% /dev tmpfs 370M 876K 369M 1% /run /dev/sda1 9.6G 3.6G 6.0G 38% / tmpfs 1.9G 0 1.9G 0% /dev/shm tmpfs 5.0M 0 5.0M 0% /run/lock tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup /dev/loop0 89M 89M 0 100% /snap/core/7270 /dev/loop1 89M 89M 0 100% /snap/core/7396 /dev/loop2 61M 61M 0 100% /snap/google-cloud-sdk/93 /dev/loop3 61M 61M 0 100% /snap/google-cloud-sdk/94 /dev/sda15 105M 3.6M 101M 4% /boot/efi tmpfs 370M 0 370M 0% /run/user/1001
$ gcloud auth login
$ gcloud compute instances describe garch-trader | grep disk Did you mean zone [us-central1-a] for instance: [garch-trader] (Y/n)? Y disks: source: https://www.googleapis.com/compute/v1/projects/idyllic-silo-248808/zones/us-central1-a/disks/garch-trader
$ gcloud compute disks resize garch-trader --zone us-central1-a --size 50GB $ sudo growpart /dev/sda 1 $ sudo resize2fs /dev/sda1
mysql> CREATE DATABASE securities_master; mysql> USE securities_master;
# Create a new user and let remote access possible mysql> CREATE USER 'username'@'localhost' IDENTIFIED BY 'password'; mysql> GRANT ALL PRIVILEGES ON securities_master.* TO 'username'@'localhost' WITH GRANT OPTION; mysql> CREATE USER 'username'@'%' IDENTIFIED BY 'password'; mysql> GRANT ALL PRIVILEGES ON securities_master.* TO 'username'@'%' WITH GRANT OPTION; mysql> FLUSH PRIVILEGES;
CREATE TABLE `exchange` ( `id` int NOT NULL AUTO_INCREMENT, `abbrev` varchar(32) NOT NULL, `name` varchar(255) NOT NULL, `city` varchar(255) NULL, `country` varchar(255) NULL, `currency` varchar(64) NULL, `timezone_offset` time NULL, `created_date` datetime NOT NULL, `last_updated_date` datetime NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; CREATE TABLE `data_vendor` ( `id` int NOT NULL AUTO_INCREMENT, `name` varchar(64) NOT NULL, `website_url` varchar(255) NULL, `support_email` varchar(255) NULL, `created_date` datetime NOT NULL, `last_updated_date` datetime NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; CREATE TABLE `symbol` ( `id` int NOT NULL AUTO_INCREMENT, `exchange_id` int NULL, `ticker` varchar(32) NOT NULL, `instrument` varchar(64) NOT NULL, `name` varchar(255) NULL, `sector` varchar(255) NULL, `currency` varchar(32) NULL, `created_date` datetime NOT NULL, `last_updated_date` datetime NOT NULL, PRIMARY KEY (`id`), KEY `index_exchange_id` (`exchange_id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; CREATE TABLE `daily_price` ( `id` int NOT NULL AUTO_INCREMENT, `data_vendor_id` int NOT NULL, `symbol_id` int NOT NULL, `price_date` datetime NOT NULL, `created_date` datetime NOT NULL, `last_updated_date` datetime NOT NULL, `open_price` decimal(19,4) NULL, `high_price` decimal(19,4) NULL, `low_price` decimal(19,4) NULL, `close_price` decimal(19,4) NULL, `adj_close_price` decimal(19,4) NULL, `volume` bigint NULL, PRIMARY KEY (`id`), KEY `index_data_vendor_id` (`data_vendor_id`), KEY `index_synbol_id` (`symbol_id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
pip install pandas pip install lxml pip install yfinance --upgrade --no-cache-dir sudo apt-get install libmysqlclient-dev pip install MySQL-python
#!/usr/bin/python # -*- coding: utf-8 -*- import datetime import lxml.html from lxml import etree from urllib2 import urlopen import MySQLdb as mdb from math import ceil def obtain_parse_wiki_snp500(): """Download and parse the Wikipedia list of S&P500 constituents using requests and libxml. Returns a list of tuples for to add to MySQL.""" # Stores the current time, for the created_at record now = datetime.datetime.utcnow() # Use libxml to download the list of S&P500 companies and obtain the symbol table # page = lxml.html.parse("http://en.wikipedia.org/wiki/List_of_S%26P_500_companies") # this line is failed due to https problem page = lxml.html.parse(urlopen("http://en.wikipedia.org/wiki/List_of_S%26P_500_companies")) # print etree.tostring(page.getroot()) # check to see if the content has downloaded symbolslist = page.xpath('//table[1]/tbody/tr')[1:] # Obtain the symbol information for each row in the S&P500 constituent table symbols = [] for symbol in symbolslist: tds = symbol.getchildren() sd = {'ticker': tds[0].getchildren()[0].text, 'name': tds[1].getchildren()[0].text, 'sector': tds[3].text} # Create a tuple (for the DB format) and append to the grand list symbols.append( (sd['ticker'], 'stock', sd['name'], sd['sector'], 'USD', now, now) ) return symbols def insert_snp500_symbols(symbols): """Insert the S&P500 symbols into the MySQL database.""" # Connect to the MySQL instance db_host = '127.0.0.1' db_user = 'sec_user' db_pass = '1234' db_name = 'securities_master' con = mdb.connect(host=db_host, user=db_user, passwd=db_pass, db=db_name) # Create the insert strings column_str = "ticker, instrument, name, sector, currency, created_date, last_updated_date" insert_str = ("%s, " * 7)[:-2] final_str = "INSERT INTO symbol (%s) VALUES (%s)" % (column_str, insert_str) print final_str, len(symbols) # Using the MySQL connection, carry out an INSERT INTO for every symbol with con: cur = con.cursor() # This line avoids the MySQL MAX_PACKET_SIZE # Although of course it could be set larger! for i in range(0, int(ceil(len(symbols) / 100.0))): cur.executemany(final_str, symbols[i*100:(i+1)*100-1]) if __name__ == "__main__": symbols = obtain_parse_wiki_snp500() insert_snp500_symbols(symbols)
#!/usr/bin/python # -*- coding: utf-8 -*- import datetime import MySQLdb as mdb import urllib2 import yfinance as yf import math # Obtain a database connection to the MySQL instance db_host = '127.0.0.1' db_user = 'sec_user' db_pass = '1234' db_name = 'securities_master' con = mdb.connect(db_host, db_user, db_pass, db_name) def obtain_list_of_db_tickers(): """Obtains a list of the ticker symbols in the database.""" with con: cur = con.cursor() cur.execute("SELECT id, ticker FROM symbol") data = cur.fetchall() return [(d[0], d[1]) for d in data] def get_daily_historic_data_yahoo(ticker, start_date=(2000,1,1), end_date=datetime.date.today().timetuple()[0:3]): """Obtains data from Yahoo Finance returns and a list of tuples. ticker: Yahoo Finance ticker symbol, e.g. "GOOG" for Google, Inc. start_date: Start date in (YYYY, M, D) format end_date: End date in (YYYY, M, D) format""" # Construct the Yahoo URL with the correct integer query parameters # for start and end dates. Note that some parameters are zero-based! yahoo_url = "http://ichart.finance.yahoo.com/table.csv?s=%s&a=%s&b=%s&c=%s&d=%s&e=%s&f=%s" % \ (ticker, start_date[1] - 1, start_date[2], start_date[0], end_date[1] - 1, end_date[2], end_date[0]) startDate = str(start_date[0]) + '-' + str(start_date[1]) + '-' + str(start_date[2]) endDate = str(end_date[0]) + '-' + str(end_date[1]) + '-' + str(end_date[2]) # Try connecting to Yahoo Finance and obtaining the data # On failure, print an error message. try: # yf_data = urllib2.urlopen(yahoo_url).readlines()[1:] # Ignore the header yf_ticker = yf.Ticker(ticker) # yf_data = yf_ticker.history(period="max") # yf_data = yf_ticker.history(start=startDate, end=endDate) yf_data = yf.download(ticker, start=startDate, end=endDate) prices = [] for index, row in yf_data.iterrows(): p = str(index).strip().split(' ') if math.isnan(row['Open']) or math.isnan(row['High']) or math.isnan(row['Low']) or math.isnan(row['Close']) or math.isnan(row['Volume']) or math.isnan(row['Adj Close']): continue prices.append( (datetime.datetime.strptime(p[0], '%Y-%m-%d'), row['Open'], row['High'], row['Low'], row['Close'], row['Volume'], row['Adj Close']) ) except Exception, e: print "Could not download Yahoo data: %s" % e return prices def insert_daily_data_into_db(data_vendor_id, symbol_id, daily_data): """Takes a list of tuples of daily data and adds it to the MySQL database. Appends the vendor ID and symbol ID to the data. daily_data: List of tuples of the OHLC data (with adj_close and volume)""" # Create the time now now = datetime.datetime.utcnow() # Amend the data to include the vendor ID and symbol ID daily_data = [(data_vendor_id, symbol_id, d[0], now, now, d[1], d[2], d[3], d[4], d[5], d[6]) for d in daily_data] # Create the insert strings column_str = """data_vendor_id, symbol_id, price_date, created_date, last_updated_date, open_price, high_price, low_price, close_price, volume, adj_close_price""" insert_str = ("%s, " * 11)[:-2] final_str = "INSERT INTO daily_price (%s) VALUES (%s)" % (column_str, insert_str) # Using the MySQL connection, carry out an INSERT INTO for every symbol with con: cur = con.cursor() cur.executemany(final_str, daily_data) if __name__ == "__main__": # Loop over the tickers and insert the daily historical # data into the database tickers = obtain_list_of_db_tickers() for t in tickers: print "Adding data for %s" % t[1] yf_data = get_daily_historic_data_yahoo(t[1]) insert_daily_data_into_db('1', t[0], yf_data)
#!/usr/bin/python # -*- coding: utf-8 -*- import pandas as pd import pandas.io.sql as psql import MySQLdb as mdb # Connect to the MySQL instance db_host = '127.0.0.1' db_user = 'sec_user' db_pass = '1234' db_name = 'securities_master' con = mdb.connect(db_host, db_user, db_pass, db_name) # Select all of the historic Google adjusted close data sql = """SELECT dp.price_date, dp.adj_close_price FROM symbol AS sym INNER JOIN daily_price AS dp ON dp.symbol_id = sym.id WHERE sym.ticker = 'GOOG' ORDER BY dp.price_date ASC;""" # Create a pandas dataframe from the SQL query # goog = psql.frame_query(sql, con=con, index_col='price_date') goog = psql.read_sql(sql, con=con, index_col='price_date') # Output the dataframe tail print goog.tail()
$ sudo apt-get install apt-transport-https ca-certificates curl software-properties-common $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable edge" $ sudo apt-get update $ apt-cache policy docker-ce $ sudo apt-get install -y docker-ce $ sudo systemctl status docker
sudo apt install docker.io
sudo systemctl enable --now docker
docker --version
To display all users run following command:
$ compgen -u