Bug fixes, DISABLE_TQDM env var (#118)

Bug fixes, DISABLE_TQDM env var

README.rst

@@ -148,6 +148,11 @@ Downloading charts
     # ta='1' > display technical analysis
     # ta='0' > ignore technical analysis
     
+Environment Variables
+======================
+
+Set ``DISABLE_TQDM=1`` in your environment to disable the progress bar.
+
 Documentation
 ==============
 

finviz/helper_functions/request_functions.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 from typing import Callable, Dict, List
 
 import aiohttp
@@ -62,7 +63,7 @@ def sequential_data_scrape(
 ) -> List[Dict]:
     data = []
 
-    for url in tqdm(urls):
+    for url in tqdm(urls, disable="DISABLE_TQDM" in os.environ):
         try:
             response = finviz_request(url, user_agent)
             kwargs["URL"] = url

finviz/helper_functions/save_data.py

@@ -47,12 +47,15 @@ def export_to_db(headers, data, filename):
     c = conn.cursor()
 
     for field in headers:
-
         field_cleaned = re.sub(r"[^\w\s]", "", field)
         field_cleaned = field_cleaned.replace(" ", "")
+        field_cleaned = field_cleaned.replace("50DHigh", "High50D")
+        field_cleaned = field_cleaned.replace("50DLow", "Low50D")
+        field_cleaned = field_cleaned.replace("52WHigh", "High52W")
+        field_cleaned = field_cleaned.replace("52WLow", "Low52W")
         field_list += field_cleaned + " TEXT, "
 
-    c.execute(f"CREATE TABLE IF NOT EXISTS {table_name} ({field_list[:-2]})")
+    c.execute(f"CREATE TABLE IF NOT EXISTS {table_name} ({field_list[:-2]});")
 
     inserts = ""
     for row in data:
@@ -67,7 +70,7 @@ def export_to_db(headers, data, filename):
     insert_lines = inserts[:-2]
 
     try:
-        c.execute(f"INSERT INTO {table_name} VALUES {insert_lines}")
+        c.execute(f"INSERT INTO {table_name} VALUES {insert_lines};")
     except sqlite3.Error as error:
         print("An error has occurred", error.args[0])
 

finviz/main_func.py

@@ -57,7 +57,12 @@ def get_insider(ticker):
 
     get_page(ticker)
     page_parsed = STOCK_PAGE[ticker]
-    table = page_parsed.cssselect('table[class="body-table"]')[0]
+    outer_table = page_parsed.cssselect('table[class="body-table"]')
+
+    if len(outer_table) == 0:
+        return []
+
+    table = outer_table[0]
     headers = table[0].xpath("td//text()")
     data = [dict(zip(headers, row.xpath("td//text()"))) for row in table[1:]]
 
@@ -74,7 +79,12 @@ def get_news(ticker):
 
     get_page(ticker)
     page_parsed = STOCK_PAGE[ticker]
-    rows = page_parsed.cssselect('table[id="news-table"]')[0].xpath('./tr[not(@id)]')
+    news_table = page_parsed.cssselect('table[id="news-table"]')
+
+    if len(news_table) == 0:
+        return []
+
+    rows = news_table[0].xpath('./tr[not(@id)]')
 
     results = []
     date = None

finviz/screener.py

@@ -27,7 +27,7 @@
 
 
 class Screener(object):
-    """ Used to download data from http://www.finviz.com/screener.ashx. """
+    """ Used to download data from https://www.finviz.com/screener.ashx. """
 
     @classmethod
     def init_from_url(cls, url, rows=None):
@@ -103,7 +103,7 @@ def __init__(
             self._filters = filters
 
         if table is None:
-            self._table = "Overview"
+            self._table = "111"
         else:
             self._table = self.__check_table(table)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "finviz"
-version = "1.4.2"
+version = "1.4.3"
 description = "Unofficial API for finviz.com"
 authors = ["Mario Stoev <bg.mstoev@gmail.com>", "James Bury <jabury@sympatico.ca>"]
 license = "MIT"

setup.py

@@ -3,13 +3,13 @@
 setup(
     name="finviz",
     packages=["finviz", "finviz.helper_functions"],
-    version="1.4.2",
+    version="1.4.3",
     license="MIT",
     description="Unofficial API for FinViz.com",
     author="Mario Stoev, James Bury",
     author_email="bg.mstoev@gmail.com, jabury@sympatico.ca",
     url="/~https://github.com/mariostoev/finviz",
-    download_url="/~https://github.com/mariostoev/finviz/archive/1.4.2.tar.gz",
+    download_url="/~https://github.com/mariostoev/finviz/archive/1.4.3.tar.gz",
     keywords=["finviz", "api", "screener", "finviz api", "charts", "scraper"],
     install_requires=[
         "lxml",