Merge pull request #32 from hhlohwv/main

Fix errors associated with trying to incorporate FAISS into workflow

app/langchain_api.py

@@ -33,7 +33,7 @@
 import pyalex
 import PyPDF2
 import io
-#import tiktoken
+import tiktoken
 #from demo import read_single 
 
 #TODO: IF doi -> then search open alex -> determine relevant metadata to return. -> Together once everything is up to date. 
@@ -384,10 +384,10 @@ async def langchain_paper_search(node):
     document = Document(page_content = text)
 
     splitter = RecursiveCharacterTextSplitter(chunk_size = 5000, chunk_overlap = 1000)
-    texts = splitter.split_documents(document)
+    split_texts = splitter.split_text(text)
     embeddings = OpenAIEmbeddings()
-
-    db = FAISS.from_documents(texts, embeddings)
+    db = FAISS.from_texts(split_texts, embeddings)
+    print("Database built...\n")
 
     # Define all the queries and corresponding schemas in a list
     queries_schemas_docs = [