Update app.py

app.py

@@ -29,7 +29,7 @@ def util_upload_file_and_return_list_docs(uploaded_files):
         save_path = Path(os.getcwd(), uploaded_file.name)
         with open(save_path, mode='wb') as w:
             w.write(uploaded_file.getvalue())
-        print('save_path:', save_path)
+        #print('save_path:', save_path)
         docs = fitz.open(save_path) 
         list_docs.append(docs)
         list_save_path.append(save_path)
@@ -104,18 +104,18 @@ def split_into_sentences_with_offsets(text):
     return [(sent.text, sent.start_char, sent.end_char) for sent in doc.sents]
     
 def util_get_list_page_and_passage(list_docs, list_save_path):
-    page_documents = []
+    #page_documents = []
     passage_documents = []
     for ind_doc, docs in enumerate(list_docs):
         for txt_index, txt_page in enumerate(docs):
             page_document = txt_page.get_text()##.encode("utf8") # get plain text (is in UTF-8)
-            page_documents.append(page_document)
+            #page_documents.append(page_document)
             sections = split_into_sentences_with_offsets(page_document) 
             for sub_sub_index, sub_sub_item in enumerate(sections):
                 sub_text=sub_sub_item[0]
-                passage_document = Document(page_content=sub_text, metadata={"page_index": txt_index, "file_name" : str(list_save_path[ind_doc])}) 
+                passage_document = Document(page_content=sub_text, metadata={"page_content": page_document,"page_index": txt_index, "file_name" : str(list_save_path[ind_doc])}) 
                 passage_documents.append(passage_document) 
-    return(page_documents,passage_documents)
+    return(passage_documents)
     
 # def util_index_chromadb_passages():
 #     ##### PROCESSING
@@ -150,26 +150,16 @@ def util_get_list_pageno_and_contents(some_query_passage, page_documents,passage
     print('rescore :: ',rescore)
     tmp_array = []
     for i, x in enumerate(passage_nodes):
-        tmp_dict = {"passage_content":x.page_content, "page_no":int(x.metadata['page_index'])+1, "file_name": str(x.metadata['file_name']), "score" : float(rescore[i])}
+        tmp_dict = {"passage_content":x.page_content, 
+                    "page_no":int(x.metadata['page_index'])+1, 
+                    "page_content":str(x.metadata['page_content']), 
+                    "file_name": str(x.metadata['file_name']), 
+                    "score" : float(rescore[i])}
         tmp_array.append(tmp_dict)
     df = pd.DataFrame(tmp_array)
     df = df.sort_values(by='score', ascending=False)
     df = df.drop_duplicates(subset=['file_name'], keep='first')
-    
-    # print('rescore ' , rescore)
-    # print(rescore)
-    # max_pos_index = rescore.index(max(rescore))
-    # print("Maximum Index position: ",max_pos_index)
-    # print(passage_nodes[max_pos_index].page_content)
-
-    # #Document(page_content=sub_text, metadata={"page_index": txt_index}) 
-    
-    
-    # for index, item in enumerate(passage_nodes):
-    #     page_no = passage_nodes[index]
-    #     page_content = page_documents[page_no]
-    #     if(index==max_pos_index):
-    #         return_value.append((page_no+1,page_content))
+    df = df[["passage_content","file_name""page_no","page_content"]]
     return(df)
     
 # # def util_openai_extract_entity(example_passage, example_entity, page_content):
@@ -243,7 +233,7 @@ def util_openai_modify_prompt(example_prompt, page_content):
 #     tmp_list = sorted(tmp_list, key=itemgetter(2), reverse=True)
 #     return(tmp_list)
     
-page_documents = []
+
 passage_documents = []
 
 with st.form("my_form"):
@@ -275,7 +265,7 @@ with st.form("my_form"):
         list_docs, list_save_path = util_upload_file_and_return_list_docs(uploaded_files)
         # print('list_docs ' ,list_docs)
         # print('list_save_path ' , list_save_path)
-        page_documents , passage_documents = util_get_list_page_and_passage(list_docs, list_save_path)
+        passage_documents = util_get_list_page_and_passage(list_docs, list_save_path)
         bm25_retriever = BM25Retriever.from_documents(passage_documents)
         bm25_retriever.k = 2
         chroma_vectorstore = Chroma.from_documents(passage_documents, embedding)
@@ -283,7 +273,7 @@ with st.form("my_form"):
         ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, chroma_retriever],weights=[0.25, 0.75])
         passage_nodes = ensemble_retriever.get_relevant_documents(single_example_passage)
         print('len(passage_nodes):', len(passage_nodes))
-        df = util_get_list_pageno_and_contents(single_example_passage, page_documents,passage_documents,passage_nodes)
+        df = util_get_list_pageno_and_contents(single_example_passage,passage_documents,passage_nodes)
         st.write(df)
         # print('len(page_list_retrieve):', len(page_list_retrieve))
         # if(len(page_list_retrieve)>0):