diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..c99e38b --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,27 @@ +name: Pylint + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + pip install -r requirements.txt + + - name: Analysing the code with pylint + run: | + pylint $(git ls-files '*.py') diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..8f8b325 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,12 @@ +[DESIGN] + +# Maximum number of locals for function / method body +max-locals=25 + +# Maximum number of arguments for function / method +max-args=10 + +[FORMAT] + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$|^\s*(\w*\s*=\s*)?(\"|\').*(\"|\'),?\s*$ \ No newline at end of file diff --git a/README.md b/README.md index d09d09a..e79a8c6 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ symbol TCS atoSellQty NaN NaN NaN ... NaN NaN 491 #get_companyinfo json format -{"info":{"symbol":"TCS","companyName":"Tata Consultancy Services Limited","industry":"COMPUTERS - SOFTWARE","activeSeries":["EQ"],"debtSeries":[],"tempSuspendedSeries":[],"isFNOSec":true,"isCASec":false,"isSLBSec":true,"isDebtSec":false,"isSuspended":false,"isETFSec":false,"isDelisted":false," ......} +{"info":{"symbol":"TCS","companyName":"Tata Consultancy Services Limited","industry":"COMPUTERS - SOFTWARE","activeSeries":["EQ"],"debtSeries":[],"tempSuspendedSeries":[],"isFNOSec":true,"isCASec":false,"isSLBSec":true,"isDebtSec":false,"isSuspended":false,"isETFSec":false,"isDelisted":false, ......} #get_marketstatus @@ -66,3 +66,33 @@ atoSellQty NaN ``` +--- + +# API Documentation + +## Functions + +### get_companyinfo + +Function description goes here. + +### get_marketstatus + +Function description goes here. + +### get_price + +Function description goes here. + +### get_corpinfo + +Function description goes here. + +### get_event + +Function description goes here. + +### get_chartdata + +Function description goes here. + diff --git a/nsedt/equity.py b/nsedt/equity.py index 869e4d3..81fafff 100644 --- a/nsedt/equity.py +++ b/nsedt/equity.py @@ -1,16 +1,21 @@ -import datetime +""" +get data for Equity +""" import concurrent +import datetime +import logging +import urllib from concurrent.futures import ALL_COMPLETED + import pandas as pd + from nsedt import utils -from nsedt.utils import data_format -import urllib from nsedt.resources import constants as cns -import logging +from nsedt.utils import data_format logging.basicConfig( level=logging.INFO, - format=cns.log_format, + format=cns.LOG_FORMAT, datefmt="%m/%d/%Y %I:%M:%S %p", ) @@ -32,19 +37,19 @@ def get_companyinfo( """ params = {} cookies = utils.get_cookies() - base_url = cns.base_url - event_api = cns.equity_info + base_url = cns.BASE_URL + event_api = cns.EQUITY_INFO params["symbol"] = symbol url = base_url + event_api + urllib.parse.urlencode(params) - data = utils.fetch_url(url, cookies, key="info") + data = utils.fetch_url(url, cookies) - if response_type == "json": - return data.to_json() - else: + if response_type == "panda_df": return data + return data.to_json() + def get_marketstatus( response_type="panda_df", @@ -59,17 +64,17 @@ def get_marketstatus( """ cookies = utils.get_cookies() - base_url = cns.base_url - event_api = cns.marketStatus + base_url = cns.BASE_URL + event_api = cns.MARKETSTATUS url = base_url + event_api data = utils.fetch_url(url, cookies, key="marketState") - if response_type == "json": - return data.to_json() - else: + if response_type == "panda_df": return data + return data.to_json() + def get_price( start_date, @@ -89,29 +94,25 @@ def get_price( Pandas DataFrame: df containing data for symbol of provided date range """ cookies = utils.get_cookies() - base_url = cns.base_url - price_api = cns.equity_price_histroy + base_url = cns.BASE_URL + price_api = cns.EQUITY_PRICE_HISTROY url_list = [] # set the window size to one year - window_size = datetime.timedelta(days=cns.window_size) + window_size = datetime.timedelta(days=cns.WINDOW_SIZE) current_window_start = start_date while current_window_start < end_date: current_window_end = current_window_start + window_size # check if the current window extends beyond the end_date - if current_window_end > end_date: - current_window_end = end_date - - st = current_window_start.strftime("%d-%m-%Y") - et = current_window_end.strftime("%d-%m-%Y") + current_window_end = min(current_window_end, end_date) if input_type == "stock": params = { "symbol": symbol, - "from": st, - "to": et, + "from": current_window_start.strftime("%d-%m-%Y"), + "to": current_window_end.strftime("%d-%m-%Y"), "dataType": "priceVolumeDeliverable", "series": series, } @@ -122,18 +123,19 @@ def get_price( current_window_start = current_window_end + datetime.timedelta(days=1) result = pd.DataFrame() - with concurrent.futures.ThreadPoolExecutor(max_workers=cns.max_workers) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=cns.MAX_WORKERS) as executor: future_to_url = { - executor.submit(utils.fetch_url, url, cookies): url for url in url_list + executor.submit(utils.fetch_url, url, cookies, "data"): url + for url in url_list } concurrent.futures.wait(future_to_url, return_when=ALL_COMPLETED) for future in concurrent.futures.as_completed(future_to_url): url = future_to_url[future] try: - df = future.result() - result = pd.concat([result, df]) + dataframe = future.result() + result = pd.concat([result, dataframe]) except Exception as exc: - logging.error(f"{url} got exception: {exc}. Please try again later.") + logging.error("%s got exception: %s. Please try again later.", url, exc) raise exc return data_format.price(result) @@ -152,6 +154,8 @@ def get_corpinfo( symbol (str, optional): stock symbol. Defaults to None. Returns: Pandas DataFrame: df containing data for symbol of provided date range + or + Json: json containing data for symbol of provided date range """ cookies = utils.get_cookies() params = { @@ -160,17 +164,16 @@ def get_corpinfo( "to_date": end_date, "index": "equities", } - base_url = cns.base_url - price_api = cns.equity_corpinfo + base_url = cns.BASE_URL + price_api = cns.EQUITY_CORPINFO url = base_url + price_api + urllib.parse.urlencode(params) data = utils.fetch_url(url, cookies) - if response_type == "json": - return data.to_json() - else: + if response_type == "panda_df": return data - return + + return data.to_json() def get_event( @@ -188,8 +191,8 @@ def get_event( """ params = {} cookies = utils.get_cookies() - base_url = cns.base_url - event_api = cns.equity_event + base_url = cns.BASE_URL + event_api = cns.EQUITY_EVENT params["index"] = index if start_date is not None: @@ -214,8 +217,8 @@ def get_chartdata( """ params = {} cookies = utils.get_cookies() - base_url = cns.base_url - event_api = cns.equity_chart + base_url = cns.BASE_URL + event_api = cns.EQUITY_CHART try: identifier = get_companyinfo(symbol)["info"]["identifier"] except KeyError: diff --git a/nsedt/resources/constants.py b/nsedt/resources/constants.py index 768675d..2f492c3 100644 --- a/nsedt/resources/constants.py +++ b/nsedt/resources/constants.py @@ -1,6 +1,10 @@ -window_size = 50 -max_workers = 10 -log_format = """{ +""" +Constants +""" + +WINDOW_SIZE = 50 +MAX_WORKERS = 10 +LOG_FORMAT = """{ "time": "%(asctime)s", "lineno": "%(lineno)d", "name": "[%(name)s]", @@ -11,10 +15,10 @@ "logmessage": "%(message)s", }""" -base_url = "https://www.nseindia.com/" -equity_price_histroy = "api/historical/securityArchives?" -equity_corpinfo = "api/corporates-corporateActions?" -marketStatus = "api/marketStatus" -equity_event = "api/event-calendar?" -equity_chart = "api/chart-databyindex?" -equity_info = "api/quote-equity?" +BASE_URL = "https://www.nseindia.com/" +EQUITY_PRICE_HISTROY = "api/historical/securityArchives?" +EQUITY_CORPINFO = "api/corporates-corporateActions?" +MARKETSTATUS = "api/marketStatus" +EQUITY_EVENT = "api/event-calendar?" +EQUITY_CHART = "api/chart-databyindex?" +EQUITY_INFO = "api/quote-equity?" diff --git a/nsedt/utils/__init__.py b/nsedt/utils/__init__.py index b5e9576..f057f84 100644 --- a/nsedt/utils/__init__.py +++ b/nsedt/utils/__init__.py @@ -1,12 +1,23 @@ +""" +utils for nsedt +""" + import json + import pandas as pd import requests -from nsedt.resources import constants as cns -base_url = cns.base_url +from nsedt.resources import constants as cns def get_headers(): + """ + Args: + --- + Returns: + Json: json containing nse header + """ + return { "Host": "www.nseindia.com", "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0", @@ -20,19 +31,36 @@ def get_headers(): def get_cookies(): - response = requests.get(base_url, timeout=30, headers=get_headers()) - if response.status_code != requests.codes.ok: + """ + Args: + --- + Returns: + Json: json containing nse cookies + """ + + response = requests.get(cns.BASE_URL, timeout=30, headers=get_headers()) + if response.status_code != 200: raise ValueError("Retry again in a minute.") return response.cookies.get_dict() -def fetch_url(url, cookies, key="data"): +def fetch_url(url, cookies, key=None): + """ + Args: + url (str): URL to fetch + cookies (str): NSE cokies + key (str, Optional): + Returns: + Pandas DataFrame: df containing url data + + """ + response = requests.get(url, timeout=30, headers=get_headers(), cookies=cookies) - if response.status_code == requests.codes.ok: + if response.status_code == 200: json_response = json.loads(response.content) - try: - return pd.DataFrame.from_dict(json_response[key]) - except: + if key is None: return pd.DataFrame.from_dict(json_response) - else: - raise ValueError("Please try again in a minute.") + + return pd.DataFrame.from_dict(json_response[key]) + + raise ValueError("Please try again in a minute.") diff --git a/nsedt/utils/data_format.py b/nsedt/utils/data_format.py index 21c4a35..a64ae9c 100644 --- a/nsedt/utils/data_format.py +++ b/nsedt/utils/data_format.py @@ -1,7 +1,18 @@ +""" +return data in specific format +""" + import pandas as pd def price(result): + """ + Args: + result (Pandas DataFrame): result + Returns: + Pandas DataFrame: df containing data in specific format + """ + columns_required = [ "CH_TIMESTAMP", "CH_OPENING_PRICE", @@ -20,7 +31,7 @@ def price(result): ] try: result = result[columns_required] - except: + except: # pylint: disable=W0702 return result result = result.set_axis( [ diff --git a/requirements.txt b/requirements.txt index 686bba6..bee5550 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ -beautifulsoup4 requests numpy pandas six -click lxml diff --git a/setup.py b/setup.py index 74ee6db..54c2a2f 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,16 @@ # -*- coding: utf-8 -*- - +""" +Install script +""" from setuptools import setup, find_packages +with open("README.md", "r", encoding="utf-8") as file: + long_description = file.read() setup( name="nsedt", - version="0.0.4", + version="0.0.5", author="Pratik Anand", - long_description=open("README.md", "r", encoding="utf-8").read(), + long_description=long_description, long_description_content_type="text/markdown", description="Library to collect NSE data in pandas dataframe", packages=find_packages(), @@ -16,16 +20,13 @@ "numpy", "pandas", "six", - "click", ], classifiers=[ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], )