Merge pull request #15 from IBM/14-implement-progress-bars-during-pro…

…cessing

Fixes #14 Implemented progressbars

javacore_set.py

@@ -16,6 +16,7 @@
 
 from lxml import etree
 from lxml.etree import XMLSyntaxError
+from tqdm import tqdm
 
 import tips
 from code_snapshot_collection import CodeSnapshotCollection
@@ -235,7 +236,7 @@ def parse_common_data(self, filename):
 
     def parse_javacores(self):
         """ creates a Javacore object for each javacore...txt file in the given path """
-        for filename in self.files:
+        for filename in tqdm(self.files, "Parsing Javacore files", unit=" javacores"):
             filename = os.path.join(self.path, filename)
             javacore = Javacore()
             javacore.create(filename, self)
@@ -480,17 +481,21 @@ def generate_htmls_from_xmls_xsls(report_xml_file, data_input_dir, output_dir):
             os.mkdir(output_dir)
         shutil.copy2(report_xml_file, data_input_dir)
 
+        list_files = os.listdir(data_input_dir)
+        progress_bar = tqdm(desc="Generating html files", unit=' files')
+
         # Generating list of tuples. This is required attribute for p.map function executed few lines below.
         generate_html_from_xml_xsl_files_params = []
-        for file in os.listdir(data_input_dir):
-            generate_html_from_xml_xsl_files_params.append((file, data_input_dir, output_dir))
+        for file in list_files:
+            generate_html_from_xml_xsl_files_params.append((file, data_input_dir, output_dir, progress_bar))
 
         # https://docs.python.org/3.8/library/multiprocessing.html
         threads_no = JavacoreSet.get_number_of_parallel_threads()
         logging.info(f"Using {threads_no} threads to generate html files")
         with Pool(threads_no) as p:
             p.map(JavacoreSet.generate_html_from_xml_xsl_files, generate_html_from_xml_xsl_files_params)
 
+        progress_bar.close()
         logging.info(f"Generated html files in {output_dir}")
 
     # Run with the same number of threads as you have processes but leave one thread for something else.
@@ -501,7 +506,7 @@ def get_number_of_parallel_threads():
     @staticmethod
     def generate_html_from_xml_xsl_files(args):
 
-        collection_file, collection_input_dir, output_dir = args
+        collection_file, collection_input_dir, output_dir, progress_bar = args
 
         if not collection_file.endswith(".xsl"): return
 
@@ -528,13 +533,15 @@ def generate_html_from_xml_xsl_files(args):
         logging.debug("Generating file " + html_file)
         output_doc.write(html_file, pretty_print=True)
 
+        progress_bar.update(1)
+
     def create_xml_xsl_for_collection(self, tmp_dir, xml_xsls_prefix_path, collection, output_file_prefix):
         logging.info("Creating xmls and xsls in " + tmp_dir)
         os.mkdir(tmp_dir)
         extensions = [".xsl", ".xml"]
         for extension in extensions:
             file_content = Path(xml_xsls_prefix_path + extension).read_text()
-            for element in collection:
+            for element in tqdm(collection, desc="Creating xml/xsl files", unit=" files"):
                 element_id = element.get_id()
                 filename = output_file_prefix + "_" + str(element_id) + extension
                 if filename.startswith("_"):

requirements.txt

@@ -2,4 +2,4 @@ dicttoxml
 py7zr
 lxml
 pyana
-flask # WSGI server for development the code
+tqdm