Update scripts/process.py

scripts/process.py

@@ -20,20 +20,21 @@ os.environ['TOKENIZERS_PARALLELISM'] ="false"
 #    document_store.write_documents(docs)
 
 #pipeline = start_haystack()
-
 def load_document(
-                    file_path: str,       
-                    encoding: Optional[str] = None,
-                    id_hash_keys: Optional[List[str]] = None,
-                  ) -> List[Document]:
+    file_path: str,
+    file_name,
+    encoding: Optional[str] = None,
+    id_hash_keys: Optional[List[str]] = None,
+) -> List[Document]:
     
     """
-    Takes docx, txt and pdf files as input and extracts text as well as the 
-    filename as metadata. Image pdf will not be handled in this notebook.
-    
+    takes docx, txt and pdf files as input and \
+    extracts text as well as the filename as metadata. \
+    Since haystack does not take care of all pdf files, \
+    pdfplumber is attached to the pipeline in case the pdf \ 
+    extraction fails via Haystack.
     Returns a list of type haystack.schema.Document
     """
-    file_name = str.split(file_path,'/')[-1]
 
     if file_name.endswith('.pdf'):
         converter = PDFToTextConverter(remove_numeric_tables=True)
@@ -44,17 +45,18 @@ def load_document(
 
 
     documents = []
-
-    #logger.info("Converting {}".format(file_name))
-    print("Converting '{}'".format(file_name))
-    # PDFToTextConverter, TextConverter, and DocxToTextConverter return a list containing a single Document
+    logger.info("Converting {}".format(file_name))
+    # PDFToTextConverter, TextConverter, and DocxToTextConverter 
+    # return a list containing a single Document
     document = converter.convert(
-                file_path=file_path, meta=None, encoding=encoding, id_hash_keys=id_hash_keys
-            )[0]
+                file_path=file_path, meta=None, 
+                encoding=encoding, id_hash_keys=id_hash_keys
+                )[0]
     text = document.content
-
-    # creating the Haystack document by extracting 'content' from the returned object and passing meta information
-    documents.append(Document(content=text, meta={"name": file_name}, id_hash_keys=id_hash_keys))
+    documents.append(Document(content=text, 
+                              meta={"name": file_name}, 
+                              id_hash_keys=id_hash_keys))
+    
     return documents
     
  def preprocessing(document,