stocks-list
Reference : https://github.com/akashgiri/stocks-list
nav_change.py
#!/usr/bin/python
import json
import re
import requests
from fuzzywuzzy import fuzz
from price_changes import get_stock_price_data
GOOGLE_FINANCE_URL = "http://finance.google.com/finance/info?client=ig&q=NSE:"
class MutualFundNavAnalysis:
matched_stocks_data = {}
def __init__(self):
MutualFundNavAnalysis.matched_stocks_data = {}
def get_mf_stock_data(self):
stocks_data = open("stocks-list.json", "r")
stocks = json.load(stocks_data)
stocks_data.close()
return stocks
def get_listed_stocks_dict(self):
#stocks_data = open("new-formatted.json", "r")
stocks_data = open("letter_wise_formatting.json", "r")
stocks = json.load(stocks_data)
stocks_data.close()
return stocks
def get_matched_stocks_list(self):
mf_stocks = self.get_mf_stock_data()
listed_stocks = self.get_listed_stocks_dict()
all_mf_stocks = []
## file to store matched and unmatched stocks data
data_file = open("analysis_data.txt", "w")
for key in mf_stocks.keys():
all_mf_stocks = mf_stocks[key]["stocks-data"]
misc_data = mf_stocks[key]["miscellaneous"]
print "Total stocks in mf: " + key + " : " + str(len(all_mf_stocks))
MutualFundNavAnalysis.matched_stocks_data.setdefault(key, [])
match_count = 0
matched_stocks = []
data_file.write("MATCHED STOCKS\n")
## loop to generate the matching stocks and the stock codes
## which will be used to fetch the stock data from google finance
for stock in all_mf_stocks:
stock_name = stock["stock"]
first_letter = stock_name[0].upper()
## Branch directly to stocks starting letter
## The data for stocks is segregated based on first letter of stock name
for current_stock in listed_stocks[first_letter]:
if self.is_fuzzy_matching_valid(stock_name, current_stock):
data_file.write(stock_name+" ==> "+current_stock+"\n")
match_count += 1
matched_stocks.append(stock_name)
## Prepare data for matched stocks
## stock data appended to matched_stocks_data
## in format: [[STOCK_CODE_1, WEIGHTING_1], [STOCK_CODE_2, WEIGHTING_2], ..]
stock_code = listed_stocks[first_letter][current_stock]
append_data = [stock_name, misc_data["cash_allocation"], stock_code, stock["weighting"]]
self.append_matched_data(key, append_data)
self.dump_matching_analysis(data_file, match_count, all_mf_stocks, matched_stocks)
self.append_price_change_data_in_matched_stocks()
def is_fuzzy_matching_valid(self, stock_name, current_stock):
## Get the token sort ratio from fuzzywuzzy
ratio = fuzz.token_sort_ratio(stock_name, current_stock)
return ratio > 95
def append_matched_data(self, key, append_data):
size = len(MutualFundNavAnalysis.matched_stocks_data[key])
MutualFundNavAnalysis.matched_stocks_data[key].append([])
## add all data
for data in append_data:
MutualFundNavAnalysis.matched_stocks_data[key][size].append(data)
def dump_matching_analysis(self, data_file, match_count, all_mf_stocks, matched_stocks):
#data_file = open("analysis_data.txt", "a")
print "Total matches : " + str(match_count)
print "Stocks not matched are: \n"
data_file.write("\n\nNOT MATCHED STOCKS\n")
## write all non-matched stocks for analysis
for s in all_mf_stocks:
stock = s["stock"]
if stock not in matched_stocks:
print stock
data_file.write(stock+"\n")
print "\n"
data_file.close()
def append_price_change_data_in_matched_stocks(self):
## Fetch all the price changes for matched stocks with stock codes
for key in MutualFundNavAnalysis.matched_stocks_data.keys():
print "\nPrice changes in MF %s \n" % key
for data in MutualFundNavAnalysis.matched_stocks_data[key]:
name = data[0]
code = data[2]
url = GOOGLE_FINANCE_URL + code
received_data = get_stock_price_data(url, name, code)
percent_change = received_data[0]
time = received_data[1]
data.append(percent_change)
data.append(time)
#print matched_stocks_data
with open("change_data.json", "w") as out:
json.dump(MutualFundNavAnalysis.matched_stocks_data, out)
#return matched_stocks_data
def nav_change_analysis(self):
data_file = open("change_data.json", "r")
content = json.load(data_file)
print "Read from file!"
total, total_w, total_change, cash = 0, 0, 0, 0
for key in content.keys():
total, total_w, total_change, cash = 0, 0, 0, 0
for current in content[key]:
# get cash allocation
cash = current[1]
weighting = current[3]
weighting = float(weighting)
total_w += weighting
change = current[4]
change = float(change)
total += (weighting * change)
#print content
#total_change = str((total / 100) + (cash / 100))
total_change = str(total / 100)
print "Expected NAV change for %s :: %s%%" % (key ,total_change)
def get_complete_nav_analysis(self):
self.get_matched_stocks_list()
self.nav_change_analysis()
if __name__ == "__main__":
mf_nav = MutualFundNavAnalysis()
mf_nav.get_complete_nav_analysis()getStocksList.py
#!/usr/bin/python
import json
import re
import requests
try:
from BeautifulSoup import BeautifulSoup
except ImportError:
from bs4 import BeautifulSoup
try:
inputFile = open("input.json")
except Exception, e:
print e
exit()
try:
urls = json.load(inputFile)['urls']
except Exception, e:
print e
exit()
## Object to store the stocks list
## key: fund name
## value: stocks list
detailed_portfolio = {}
keys = []
## Function to add some additional data to output
## currently adding asset allocation, i.e. cash, stock and number of stocks held
def add_additional_data(soup, count, key, detailed_portfolio):
stock_allocation = 0
cash_allocation = 0
## get the asset allocation fields
parent = soup.body.find_all(class_=re.compile("pf_assetalloc"))
td_list = parent[0].find_all('td')
for td in td_list:
if td.string == "Stock":
stock_allocation = td.next_sibling.next_sibling.string
elif td.string == "Cash":
cash_allocation = td.next_sibling.next_sibling.string
## write the asset allocation and stocks count
misc_data = {}
misc_data["stock_allocation"] = (misc_data.get("stock_allocation", 0)
+ float(stock_allocation))
misc_data["cash_allocation"] = (misc_data.get("cash_allocation", 0)
+ float(cash_allocation))
misc_data["number_of_stocks"] = (misc_data.get("number_of_stocks", 0)
+ count)
detailed_portfolio[key]["miscellaneous"] = misc_data
return detailed_portfolio
for url in urls:
print url
key = url.rpartition('/')[0].rpartition('/')[2]
keys.append(key)
detailed_portfolio.setdefault(key, {})
detailed_portfolio[key].setdefault("stocks-data", [])
## Fetch detailed portfolio data
response = requests.get(url)
print response.status_code
if (response.status_code != 200) or (response.content == ""):
print "No content to parse. Please try again!"
exit()
page = response.content
soup = BeautifulSoup(page, "lxml")
text = soup.body.find_all("td", string=re.compile("Equity"))
count = 0
for td in text:
stock_td = td.previous_sibling.previous_sibling.a
if not stock_td:
stock_td = td.previous_sibling.previous_sibling.span
sector_td = td.next_sibling.next_sibling
weighting_td = sector_td.next_sibling.next_sibling.next_sibling.next_sibling
current = {}
try:
stock_name = stock_td.string
stock_name = stock_name.replace(' Corp ', ' Corporation ')
if not re.match(r"(.*?)Ltd(?=.)", stock_name):
stock_name = stock_name.replace("Ltd", "Ltd.")
stock_name = stock_name.replace(" Limited", " Ltd.")
stock_name = stock_name.replace(" Co ", " Company ")
current["stock"] = stock_name
current["sector"] = sector_td.string
current["weighting"] = weighting_td.string
detailed_portfolio[key]["stocks-data"].append(current)
count += 1
except:
pass
add_additional_data(soup, count, key, detailed_portfolio)
outfile = open('stocks-list.json', 'w')
json.dump(detailed_portfolio, outfile)
outfile.close()
exit() price_changes.py
#!/usr/bin/python
import json
import re
import requests
def get_parsed_content(content):
content = content.replace('// [', '').replace(']', '').replace('\n', '')
return content
def get_percent_change_in_price(content):
percent_change = 0
try:
percent_change = content["cp"]
except Exception as e:
print e
return percent_change
def get_stock_price_data(url, name, code):
response = requests.get(url)
print response.status_code
if (response.status_code != 200) or (response.content == ""):
print "No content received. Exiting!"
print "name : %s :: code : %s" % (name, code)
return [0.0, "0"]
content = response.content
content_formatted = json.loads(get_parsed_content(content))
percent_change = get_percent_change_in_price(content_formatted)
#print content_formatted
print "STOCK :: %s PRICE PERCENT CHANGE :: %s TIME :: %s" % (content_formatted["t"].ljust(15), str(percent_change).ljust(15),
content_formatted["lt"].ljust(15))
return [percent_change, content_formatted["lt"]]
#url = "http://finance.google.com/finance/info?client=ig&q=NSE:TATAMOTORS"
#get_stock_price_data(url)
def nav_change_analysis(*args):
content = []
if args:
print "arguments are here"
content = args[0]
#print content
else:
data_file = open("change_data.json", "r")
content = json.load(data_file)
print "Read from file!"
total = 0
total_w = 0
cash = 0
for key in content.keys():
total_change = 0
total = 0
cash = 0
for current in content[key]:
# get cash allocation
cash = current[1]
weighting = current[3]
weighting = float(weighting)
total_w += weighting
change = current[4]
change = float(change)
total += (weighting * change)
#print content
total_change = str((total / 100) + (cash / 100))
print "Expected NAV change for %s :: %s%%" % (key ,total_change)
#nav_change_analysis()Last updated