stocks-list

Reference : https://github.com/akashgiri/stocks-list

nav_change

nav_change.py

#!/usr/bin/python

import json
import re
import requests
from fuzzywuzzy import fuzz
from price_changes import get_stock_price_data

GOOGLE_FINANCE_URL = "http://finance.google.com/finance/info?client=ig&q=NSE:"

class MutualFundNavAnalysis:
    matched_stocks_data = {}
    
    def __init__(self):
        MutualFundNavAnalysis.matched_stocks_data = {}

    def get_mf_stock_data(self):
        stocks_data = open("stocks-list.json", "r")
        stocks = json.load(stocks_data)
        stocks_data.close()
        
        return stocks
        
    def get_listed_stocks_dict(self):
        #stocks_data = open("new-formatted.json", "r")
        stocks_data = open("letter_wise_formatting.json", "r")
        stocks = json.load(stocks_data)
        stocks_data.close()
        
        return stocks
        
    def get_matched_stocks_list(self):
        mf_stocks = self.get_mf_stock_data()
        listed_stocks = self.get_listed_stocks_dict()
        all_mf_stocks = []        
        
        ## file to store matched and unmatched stocks data
        data_file = open("analysis_data.txt", "w")

        for key in mf_stocks.keys():
            all_mf_stocks = mf_stocks[key]["stocks-data"]
            misc_data = mf_stocks[key]["miscellaneous"]

            print "Total stocks in mf: " + key + " : " + str(len(all_mf_stocks))
            
            MutualFundNavAnalysis.matched_stocks_data.setdefault(key, [])
            
            match_count = 0
            matched_stocks = []
            data_file.write("MATCHED STOCKS\n")
            
            ## loop to generate the matching stocks and the stock codes
            ## which will be used to fetch the stock data from google finance
            for stock in all_mf_stocks:
                stock_name = stock["stock"]
                first_letter = stock_name[0].upper()
                
                ## Branch directly to stocks starting letter
                ## The data for stocks is segregated based on first letter of stock name
                for current_stock in listed_stocks[first_letter]:
                    if self.is_fuzzy_matching_valid(stock_name, current_stock):
                        data_file.write(stock_name+" ==> "+current_stock+"\n")

                        match_count += 1
                        matched_stocks.append(stock_name)
                        
                        ## Prepare data for matched stocks
                        ## stock data appended to matched_stocks_data
                        ## in format: [[STOCK_CODE_1, WEIGHTING_1], [STOCK_CODE_2, WEIGHTING_2], ..]
                        stock_code = listed_stocks[first_letter][current_stock]
                        
                        append_data = [stock_name, misc_data["cash_allocation"], stock_code, stock["weighting"]]
                        self.append_matched_data(key, append_data)
                        
        self.dump_matching_analysis(data_file, match_count, all_mf_stocks, matched_stocks)
        self.append_price_change_data_in_matched_stocks()

    def is_fuzzy_matching_valid(self, stock_name, current_stock):
        ## Get the token sort ratio from fuzzywuzzy
        ratio = fuzz.token_sort_ratio(stock_name, current_stock)
        return ratio > 95

    def append_matched_data(self, key, append_data):
        size = len(MutualFundNavAnalysis.matched_stocks_data[key])
        MutualFundNavAnalysis.matched_stocks_data[key].append([])

        ## add all data
        for data in append_data:
            MutualFundNavAnalysis.matched_stocks_data[key][size].append(data)

    def dump_matching_analysis(self, data_file, match_count, all_mf_stocks, matched_stocks):
        #data_file = open("analysis_data.txt", "a")
        print "Total matches : " + str(match_count)
        print "Stocks not matched are: \n"
        data_file.write("\n\nNOT MATCHED STOCKS\n")
        
        ## write all non-matched stocks for analysis
        for s in all_mf_stocks:
            stock = s["stock"]
            if stock not in matched_stocks:
                print stock
                data_file.write(stock+"\n")
                print "\n"
                
        data_file.close()
        
    def append_price_change_data_in_matched_stocks(self):
        ## Fetch all the price changes for matched stocks with stock codes
        for key in MutualFundNavAnalysis.matched_stocks_data.keys():
            print "\nPrice changes in MF %s \n" % key
            for data in MutualFundNavAnalysis.matched_stocks_data[key]:
                name = data[0]
                code = data[2]
                url = GOOGLE_FINANCE_URL + code
                received_data = get_stock_price_data(url, name, code)
                percent_change = received_data[0]
                time = received_data[1]
                data.append(percent_change)
                data.append(time)

        #print matched_stocks_data
        with open("change_data.json", "w") as out:
            json.dump(MutualFundNavAnalysis.matched_stocks_data, out)
    
        #return matched_stocks_data
        
    def nav_change_analysis(self):
        data_file = open("change_data.json", "r")
        content = json.load(data_file)
        print "Read from file!"
            
        total, total_w, total_change, cash = 0, 0, 0, 0
        for key in content.keys():
            total, total_w, total_change, cash = 0, 0, 0, 0
            for current in content[key]:
                # get cash allocation
                cash = current[1]
                weighting = current[3]
                weighting = float(weighting)
                total_w += weighting
                change = current[4]
                change = float(change)
                total += (weighting * change)
                
            #print content
            #total_change = str((total / 100) + (cash / 100))
            total_change = str(total / 100)
            print "Expected NAV change for %s :: %s%%" % (key ,total_change)

    def get_complete_nav_analysis(self):
        self.get_matched_stocks_list()
        self.nav_change_analysis()
            
if __name__ == "__main__":
    mf_nav = MutualFundNavAnalysis()
    mf_nav.get_complete_nav_analysis()

getStocksList

getStocksList.py

#!/usr/bin/python

import json
import re
import requests
try:
    from BeautifulSoup import BeautifulSoup
except ImportError:
    from bs4 import BeautifulSoup

try:
    inputFile = open("input.json")
except Exception, e:
    print e
    exit()

try:
    urls = json.load(inputFile)['urls']
except Exception, e:
    print e
    exit()

## Object to store the stocks list
## key: fund name
## value: stocks list
detailed_portfolio = {}
keys = []

## Function to add some additional data to output
## currently adding asset allocation, i.e. cash, stock and number of stocks held
def add_additional_data(soup, count, key, detailed_portfolio):
    stock_allocation = 0
    cash_allocation = 0

    ## get the asset allocation fields
    parent = soup.body.find_all(class_=re.compile("pf_assetalloc"))
    td_list = parent[0].find_all('td')
    for td in td_list:
        if td.string == "Stock":
            stock_allocation = td.next_sibling.next_sibling.string
        elif td.string == "Cash":
            cash_allocation = td.next_sibling.next_sibling.string

    ## write the asset allocation and stocks count
    misc_data = {}
    misc_data["stock_allocation"] = (misc_data.get("stock_allocation", 0)
                                     + float(stock_allocation))
    misc_data["cash_allocation"] = (misc_data.get("cash_allocation", 0)
                                    + float(cash_allocation))
    misc_data["number_of_stocks"] = (misc_data.get("number_of_stocks", 0)
                                    + count)

    detailed_portfolio[key]["miscellaneous"] = misc_data

    return detailed_portfolio

for url in urls:
    print url
    key = url.rpartition('/')[0].rpartition('/')[2]
    keys.append(key)
    detailed_portfolio.setdefault(key, {})
    detailed_portfolio[key].setdefault("stocks-data", [])
    
    ## Fetch detailed portfolio data
    response = requests.get(url)

    print response.status_code

    if (response.status_code != 200) or (response.content == ""):
        print "No content to parse. Please try again!"
        exit()
        
    page = response.content
    soup = BeautifulSoup(page, "lxml")
    text = soup.body.find_all("td", string=re.compile("Equity"))
    count = 0

    for td in text:
        stock_td = td.previous_sibling.previous_sibling.a
        if not stock_td:
            stock_td = td.previous_sibling.previous_sibling.span            
        sector_td = td.next_sibling.next_sibling
        weighting_td = sector_td.next_sibling.next_sibling.next_sibling.next_sibling
        current = {}

        try:
            stock_name = stock_td.string
            stock_name = stock_name.replace(' Corp ', ' Corporation ')
            if not re.match(r"(.*?)Ltd(?=.)", stock_name):
                stock_name = stock_name.replace("Ltd", "Ltd.")
            stock_name = stock_name.replace(" Limited", " Ltd.")
            stock_name = stock_name.replace(" Co ", " Company ")
            current["stock"] = stock_name
            current["sector"] = sector_td.string
            current["weighting"] = weighting_td.string
            detailed_portfolio[key]["stocks-data"].append(current)
            count += 1
        except:
            pass

    add_additional_data(soup, count, key, detailed_portfolio)

outfile = open('stocks-list.json', 'w')
json.dump(detailed_portfolio, outfile)
outfile.close()

exit()    

price_changes

price_changes.py

#!/usr/bin/python

import json
import re
import requests

def get_parsed_content(content):
    content = content.replace('// [', '').replace(']', '').replace('\n', '')
    return content

def get_percent_change_in_price(content):
    percent_change = 0
    try:
        percent_change = content["cp"]
    except Exception as e:
        print e

    return percent_change
    
def get_stock_price_data(url, name, code):
    response = requests.get(url)

    print response.status_code
    
    if (response.status_code != 200) or (response.content == ""):
        print "No content received. Exiting!"
        print "name : %s :: code : %s" % (name, code)
        return [0.0, "0"]

    content = response.content
    
    content_formatted = json.loads(get_parsed_content(content))
    percent_change = get_percent_change_in_price(content_formatted)
    #print content_formatted
    print "STOCK :: %s  PRICE PERCENT CHANGE :: %s TIME :: %s" % (content_formatted["t"].ljust(15), str(percent_change).ljust(15), 
                                                                    content_formatted["lt"].ljust(15))
    
    return [percent_change, content_formatted["lt"]]

#url = "http://finance.google.com/finance/info?client=ig&q=NSE:TATAMOTORS"
#get_stock_price_data(url)

def nav_change_analysis(*args):
    content = []
    if args:
        print "arguments are here"
        content = args[0]
        #print content
    else:
        data_file = open("change_data.json", "r")
        content = json.load(data_file)
        print "Read from file!"

    total = 0
    total_w = 0
    cash = 0
    for key in content.keys():
        total_change = 0
        total = 0
        cash = 0
        for current in content[key]:
            # get cash allocation
            cash = current[1]
            weighting = current[3]
            weighting = float(weighting)
            total_w += weighting
            change = current[4]
            change = float(change)
            total += (weighting * change)
            
        #print content
        total_change = str((total / 100) + (cash / 100))
        print "Expected NAV change for %s :: %s%%" % (key ,total_change)

#nav_change_analysis()