updated the app

app.py

@@ -18,6 +18,10 @@ import uuid
 from pathlib import Path
 from haystack.pipelines import Pipeline
 from haystack.nodes import TextConverter, PreProcessor, FileTypeClassifier, PDFToTextConverter, DocxToTextConverter
+from sentence_transformers import SentenceTransformer
+import pinecone
+index_name = "qa_demo"
+
 
 
 preprocessor = PreProcessor(
@@ -33,6 +37,17 @@ text_converter = TextConverter()
 pdf_converter = PDFToTextConverter()
 docx_converter = DocxToTextConverter()
 
+# check if the abstractive-question-answering index exists
+if index_name not in pinecone.list_indexes():
+    # create the index if it does not exist
+    pinecone.create_index(
+        index_name,
+        dimension=768,
+        metric="cosine"
+    )
+
+# connect to abstractive-question-answering index we created
+index = pinecone.Index(index_name)
 
 FILE_UPLOAD_PATH= "./data/uploads/"
 os.makedirs(FILE_UPLOAD_PATH, exist_ok=True)
@@ -40,7 +55,7 @@ os.makedirs(FILE_UPLOAD_PATH, exist_ok=True)
 def create_doc_store():
     document_store = PineconeDocumentStore(
         api_key= st.secrets["pinecone_apikey"],
-        index='qa_demo',
+        index=index_name,
         similarity="cosine",
         embedding_dim=768
     )
@@ -73,14 +88,19 @@ def query(pipe, question, top_k_reader, top_k_retriever):
 
 document_store = create_doc_store()
 # pipe = create_pipe(document_store)
+retriever_model = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
 retriever = EmbeddingRetriever(
 document_store=document_store,
-embedding_model="sentence-transformers/multi-qa-mpnet-base-dot-v1",
+embedding_model=retriever_model",
 model_format="sentence_transformers",
 )
+# load the retriever model from huggingface model hub
+sentence_encoder = SentenceTransformer(retriever_model)
+
 reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=False)
 pipe = ExtractiveQAPipeline(reader, retriever)
 
+
 indexing_pipeline_with_classification = Pipeline()
 indexing_pipeline_with_classification.add_node(
     component=file_type_classifier, name="FileTypeClassifier", inputs=["File"]
@@ -99,9 +119,6 @@ indexing_pipeline_with_classification.add_node(
     name="Preprocessor",
     inputs=["TextConverter", "PdfConverter", "DocxConverter"],
 )
-indexing_pipeline_with_classification.add_node(
-    component=document_store, name="DocumentStore", inputs=["Preprocessor"]
-)
 
 def set_state_if_absent(key, value):
     if key not in st.session_state:
@@ -148,6 +165,7 @@ data_files = st.sidebar.file_uploader(
     "upload", type=["pdf", "txt", "docx"], accept_multiple_files=True, label_visibility="hidden"
 )
 ALL_FILES = []
+META_DATA = []
 for data_file in data_files:
     # Upload file
     if data_file:
@@ -156,11 +174,35 @@ for data_file in data_files:
             shutil.copyfileobj(data_file.file, buffer)
         ALL_FILES.append(file_path)
         st.sidebar.write(str(data_file.name) + "    ✅ ")
-        indexing_pipeline_with_classification.run(file_paths=ALL_FILES)
+        META_DATA.append({"filename":data_file.name})
+        
 
 if len(ALL_FILES) > 0:
-    document_store.update_embeddings(retriever, update_existing_embeddings=False)
-        
+    # document_store.update_embeddings(retriever, update_existing_embeddings=False)
+    docs = indexing_pipeline_with_classification.run(file_paths=ALL_FILES, meta=META_DATA)[""]
+    index_name = "qa_demo"
+    # we will use batches of 64
+    batch_size = 64
+    docs  = docs['documents']
+    with st.spinner(
+            "🧠    Performing indexing of uplaoded documents... \n "
+        ):
+        for i range(0, len(docs), batch_size):
+            # find end of batch
+            i_end = min(i+batch_size, len(docs))
+            # extract batch
+            batch = [doc.content for doc in docs[i:i_end]]
+            # generate embeddings for batch
+            emb = retriever.encode(batch).tolist()
+            # get metadata
+            meta = [doc.meta for doc in docs[i:i_end]]
+            # create unique IDs
+            ids = [doc.id for doc in docs[i:i_end]]
+            # add all to upsert list
+            to_upsert = list(zip(ids, emb, meta))
+            # upsert/insert these records to pinecone
+            _ = index.upsert(vectors=to_upsert)
+    
 top_k_reader = st.sidebar.slider(
     "Max. number of answers",
     min_value=1,