feat(chatbot): save masked answers and questions

apps/chatbot/src/app/main.py

@@ -82,7 +82,7 @@ async def query_creation (
   else:
     queriedAt = query.queriedAt
 
-  body = {
+  bodyToReturn = {
     "id": f'{uuid.uuid4()}',
     "sessionId": session['id'],
     "question": query.question,
@@ -92,11 +92,14 @@ async def query_creation (
     "badAnswer": False
   }
 
+  bodyToSave = bodyToReturn.copy()
+  bodyToSave["question"] = chatbot.mask_pii(query.question)
+  bodyToSave["answer"] = chatbot.mask_pii(answer)
   try:
-    table_queries.put_item(Item = body)
+    table_queries.put_item(Item = bodyToSave)
   except (BotoCoreError, ClientError) as e:
     raise HTTPException(status_code=422, detail=f"[POST /queries] error: {e}")
-  return body
+  return bodyToReturn
 
 
 def current_user_id(authorization: str):

apps/chatbot/src/modules/chatbot.py

@@ -127,10 +127,13 @@ def _unmask_reference(self, response_str: str, nodes) -> str:
 
 
     def mask_pii(self, message: str) -> str:
-        try:
-            return self.pii.mask_pii(message)
-        except Exception as e:
-            logging.warning(f"[chatbot.py - mask_pii] exception in mask_pii: {e}")
+        if USE_PRESIDIO:
+            try:
+                return self.pii.mask_pii(message)
+            except Exception as e:
+                logging.warning(f"[chatbot.py - mask_pii] exception in mask_pii: {e}")
+        else:
+            return message
 
 
     def generate(self, query_str: str) -> str:

apps/chatbot/src/modules/vector_database.py

@@ -153,8 +153,8 @@ def create_documentation(
     for file in tqdm.tqdm(html_files, total=len(html_files), desc="Extracting HTML"):
 
         # FIX: resolve webdriver.Chrome "self.assert_process_still_running" error in docker
-        if file in dynamic_htmls or "/webinars/" in file or "/api/" in file:
-        # if 6 == 9:
+        # if file in dynamic_htmls or "/webinars/" in file or "/api/" in file:
+        if 6 == 9:
             url = file.replace(documentation_dir, f"{website_url}/").replace(".html", "")
 
             # svc = webdriver.ChromeService(executable_path=binary_path)
@@ -235,7 +235,7 @@ def build_automerging_index_redis(
     logging.info(f"[vector_database.py - build_automerging_index_redis] hash_table_{INDEX_ID} is now on Redis.")
 
     logging.info(f"[vector_database.py - build_automerging_index_redis] Creating index {NEW_INDEX_ID} ...")
-    nodes = Settings.node_parser.get_nodes_from_documents(documents)
+    nodes = Settings.node_parser.get_nodes_from_documents(documents[:2])
     leaf_nodes = get_leaf_nodes(nodes)
 
     redis_vector_store = RedisVectorStore(