Spaces:
Sleeping
Sleeping
Commit
·
90fc7ac
1
Parent(s):
3edae51
adding online PDF loader
Browse files
app.py
CHANGED
@@ -28,7 +28,7 @@ def get_openai_chat_model(API_key):
|
|
28 |
|
29 |
def process_documents(documents,data_chunk=1000,chunk_overlap=50):
|
30 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=data_chunk, chunk_overlap=chunk_overlap)
|
31 |
-
texts = text_splitter.split_documents(documents
|
32 |
return texts
|
33 |
|
34 |
def get_hugging_face_model(model_id,API_key,temperature=0.1):
|
@@ -56,6 +56,7 @@ def document_loader(file_path,api_key,doc_type='pdf',llm='Huggingface'):
|
|
56 |
elif doc_type == 'word':
|
57 |
document = process_word_document(document_file=file_path)
|
58 |
if document:
|
|
|
59 |
texts = process_documents(documents=document)
|
60 |
vector_db = FAISS.from_documents(documents=texts, embedding= embedding_model)
|
61 |
global qa
|
@@ -91,7 +92,7 @@ def process_word_document(document_file):
|
|
91 |
|
92 |
def process_pdf_document(document_file):
|
93 |
print("Document File Name :",document_file.name)
|
94 |
-
loader =
|
95 |
document = loader.load()[0]
|
96 |
return document
|
97 |
|
|
|
28 |
|
29 |
def process_documents(documents,data_chunk=1000,chunk_overlap=50):
|
30 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=data_chunk, chunk_overlap=chunk_overlap)
|
31 |
+
texts = text_splitter.split_documents(documents)
|
32 |
return texts
|
33 |
|
34 |
def get_hugging_face_model(model_id,API_key,temperature=0.1):
|
|
|
56 |
elif doc_type == 'word':
|
57 |
document = process_word_document(document_file=file_path)
|
58 |
if document:
|
59 |
+
print("Document :",document)
|
60 |
texts = process_documents(documents=document)
|
61 |
vector_db = FAISS.from_documents(documents=texts, embedding= embedding_model)
|
62 |
global qa
|
|
|
92 |
|
93 |
def process_pdf_document(document_file):
|
94 |
print("Document File Name :",document_file.name)
|
95 |
+
loader = PDFMinerLoader(document_file.name)
|
96 |
document = loader.load()[0]
|
97 |
return document
|
98 |
|