Update app.py

app.py

@@ -8,6 +8,7 @@ from pathlib import Path
 import os
 os.environ['OPENAI_API_KEY'] = os.environ['OPEN_API_KEY']
 st.title("Contracts Multiple File Search ")
+import pandas as pd
 
 from langchain.retrievers import BM25Retriever, EnsembleRetriever
 from langchain.schema import Document
@@ -102,17 +103,18 @@ def split_into_sentences_with_offsets(text):
     doc = nlp(text)
     return [(sent.text, sent.start_char, sent.end_char) for sent in doc.sents]
     
-def util_get_list_page_and_passage(docs):
+def util_get_list_page_and_passage(list_docs, list_save_path):
     page_documents = []
     passage_documents = []
-    for txt_index, txt_page in enumerate(docs):
-        page_document = txt_page.get_text()##.encode("utf8") # get plain text (is in UTF-8)
-        page_documents.append(page_document)
-        sections = split_into_sentences_with_offsets(page_document) 
-        for sub_sub_index, sub_sub_item in enumerate(sections):
-            sub_text=sub_sub_item[0]
-            passage_document = Document(page_content=sub_text, metadata={"page_index": txt_index}) 
-            passage_documents.append(passage_document) 
+    for ind_doc, docs in enumerate(list_docs):
+        for txt_index, txt_page in enumerate(docs):
+            page_document = txt_page.get_text()##.encode("utf8") # get plain text (is in UTF-8)
+            page_documents.append(page_document)
+            sections = split_into_sentences_with_offsets(page_document) 
+            for sub_sub_index, sub_sub_item in enumerate(sections):
+                sub_text=sub_sub_item[0]
+                passage_document = Document(page_content=sub_text, metadata={"page_index": txt_index, "page_file" : list_save_path[ind_doc]}) 
+                passage_documents.append(passage_document) 
     return(page_documents,passage_documents)
     
 # def util_index_chromadb_passages():
@@ -144,23 +146,29 @@ def util_get_list_pageno_and_contents(some_query_passage, page_documents,passage
     ''' page no starts with index 1 '''
 
     return_value = []
-
     rescore = reranker.compute_score([[some_query_passage , x.page_content] for x in passage_nodes])
-    print('rescore ' , rescore)
-    print(rescore)
-    max_pos_index = rescore.index(max(rescore))
-    print("Maximum Index position: ",max_pos_index)
-    print(passage_nodes[max_pos_index].page_content)
 
-    #Document(page_content=sub_text, metadata={"page_index": txt_index}) 
+    tmp_array = []
+    for i, x in enumerate(passage_nodes):
+        tmp_dict = {"passage_content":x.page_content, "page_no":x.metadata['page_index']+1, "page_content": passage_documents[x.metadata['page_index']], "score": rescore[i] }
+    df = pd.DataFrame(tmp_array)
+
+    
+    # print('rescore ' , rescore)
+    # print(rescore)
+    # max_pos_index = rescore.index(max(rescore))
+    # print("Maximum Index position: ",max_pos_index)
+    # print(passage_nodes[max_pos_index].page_content)
+
+    # #Document(page_content=sub_text, metadata={"page_index": txt_index}) 
     
     
-    for index, item in enumerate(passage_nodes):
-        page_no = passage_nodes[index].metadata['page_index']
-        page_content = page_documents[page_no]
-        if(index==max_pos_index):
-            return_value.append((page_no+1,page_content))
-    return(passage_nodes[max_pos_index].page_content, return_value)
+    # for index, item in enumerate(passage_nodes):
+    #     page_no = passage_nodes[index]
+    #     page_content = page_documents[page_no]
+    #     if(index==max_pos_index):
+    #         return_value.append((page_no+1,page_content))
+    return(df)
     
 # # def util_openai_extract_entity(example_passage, example_entity, page_content):
 # #     import openai
@@ -263,8 +271,9 @@ with st.form("my_form"):
     
     if submitted and (uploaded_files is not None):
         list_docs, list_save_path = util_upload_file_and_return_list_docs(uploaded_files)
-        print('list_docs ' ,list_docs)
-        print('list_save_path ' , list_save_path)
+        # print('list_docs ' ,list_docs)
+        # print('list_save_path ' , list_save_path)
+        page_documents , passage_documents = util_get_list_page_and_passage(list_docs, list_save_path)
         bm25_retriever = BM25Retriever.from_documents(passage_documents)
         bm25_retriever.k = 2
         chroma_vectorstore = Chroma.from_documents(passage_documents, embedding)
@@ -272,18 +281,19 @@ with st.form("my_form"):
         ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, chroma_retriever],weights=[0.25, 0.75])
         passage_nodes = ensemble_retriever.get_relevant_documents(single_example_passage)
         print('len(passage_nodes):', len(passage_nodes))
-        found_passage, page_list_retrieve = util_get_list_pageno_and_contents(single_example_passage, page_documents,passage_documents,passage_nodes)
-        print('len(page_list_retrieve):', len(page_list_retrieve))
-        if(len(page_list_retrieve)>0):
-            page_list_retrieve = list(set(page_list_retrieve))
-            for iindex in page_list_retrieve:
-                page_no = iindex[0]
-                page_content = iindex[1]
-                annotated_text("  ",annotation("RELEVANT PAGENO : ", str(page_no), font_family="Comic Sans MS", border="2px dashed red"),)
-                util_openai_format(single_example_passage, page_content)
-                annotated_text("  ",annotation("RELEVANT PASSAGE : ", "", font_family="Comic Sans MS", border="2px dashed red"),)
-                st.write(found_passage)
-        pchroma_client = chromadb.Client()
-        for citem in pchroma_client.list_collections():
-            print(citem.name)
+        df = util_get_list_pageno_and_contents(single_example_passage, page_documents,passage_documents,passage_nodes)
+        st.write(df)
+        # print('len(page_list_retrieve):', len(page_list_retrieve))
+        # if(len(page_list_retrieve)>0):
+        #     page_list_retrieve = list(set(page_list_retrieve))
+        #     for iindex in page_list_retrieve:
+        #         page_no = iindex[0]
+        #         page_content = iindex[1]
+        #         annotated_text("  ",annotation("RELEVANT PAGENO : ", str(page_no), font_family="Comic Sans MS", border="2px dashed red"),)
+        #         util_openai_format(single_example_passage, page_content)
+        #         annotated_text("  ",annotation("RELEVANT PASSAGE : ", "", font_family="Comic Sans MS", border="2px dashed red"),)
+        #         st.write(found_passage)
+        # pchroma_client = chromadb.Client()
+        # for citem in pchroma_client.list_collections():
+        #     print(citem.name)