Spaces:
Running
Running
luanpoppe
commited on
Commit
·
edd5b40
1
Parent(s):
449ce0a
feat: adicionando OCR em casos de PDFs com problema
Browse files- _utils/gerar_documento.py +65 -75
- _utils/gerar_documento_utils/GerarDocumento.py +1 -2
- _utils/gerar_documento_utils/utils.py +2 -11
- _utils/google_integration/google_cloud.py +3 -1
- _utils/langchain_utils/Splitter_class.py +150 -1
- _utils/langchain_utils/Vector_store_class.py +7 -1
- requirements.txt +0 -0
_utils/gerar_documento.py
CHANGED
@@ -54,101 +54,91 @@ async def gerar_documento(
|
|
54 |
# Initialize enhanced summarizer
|
55 |
summarizer = GerarDocumento(serializer, axiom_instance)
|
56 |
|
57 |
-
all_PDFs_chunks, full_text_as_array
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
isBubble,
|
63 |
-
)
|
64 |
)
|
65 |
axiom_instance.send_axiom(
|
66 |
f"INÍCIO DO TEXTO COMPLETO DOS PDFS: {full_text_as_array[0:5]}"
|
67 |
)
|
68 |
|
69 |
-
|
70 |
-
is_contextualized_chunk = serializer.should_have_contextual_chunks
|
71 |
-
|
72 |
-
if is_contextualized_chunk:
|
73 |
-
response_auxiliar_summary = (
|
74 |
-
await get_response_from_auxiliar_contextual_prompt(
|
75 |
-
full_text_as_array
|
76 |
-
)
|
77 |
-
)
|
78 |
-
axiom_instance.send_axiom(
|
79 |
-
f"RESUMO INICIAL DO PROCESSO: {response_auxiliar_summary}"
|
80 |
-
)
|
81 |
-
|
82 |
-
axiom_instance.send_axiom(
|
83 |
-
"COMEÇANDO A FAZER AS REQUISIÇÕES DO CONTEXTUAL"
|
84 |
-
)
|
85 |
-
contextualized_chunks = (
|
86 |
-
await contextual_retriever.contextualize_all_chunks(
|
87 |
-
all_PDFs_chunks, response_auxiliar_summary, axiom_instance
|
88 |
-
)
|
89 |
-
)
|
90 |
-
axiom_instance.send_axiom(
|
91 |
-
"TERMINOU DE FAZER TODAS AS REQUISIÇÕES DO CONTEXTUAL"
|
92 |
-
)
|
93 |
-
chunks_processados = contextualized_chunks
|
94 |
-
axiom_instance.send_axiom(
|
95 |
-
f"CHUNKS PROCESSADOS INICIALMENTE: {chunks_processados}"
|
96 |
-
)
|
97 |
-
else:
|
98 |
-
chunks_processados = all_PDFs_chunks
|
99 |
-
|
100 |
-
llm = LLM()
|
101 |
-
prompt_para_gerar_query_dinamico = prompt_gerar_query_dinamicamente(
|
102 |
-
cast(str, response_auxiliar_summary)
|
103 |
-
)
|
104 |
|
105 |
-
|
106 |
-
|
|
|
107 |
)
|
108 |
-
|
109 |
-
|
110 |
-
prompt_para_gerar_query_dinamico, "gemini-2.0-flash"
|
111 |
-
)
|
112 |
)
|
113 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
114 |
axiom_instance.send_axiom(
|
115 |
-
f"
|
116 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
117 |
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
|
122 |
-
|
|
|
123 |
)
|
|
|
|
|
|
|
|
|
|
|
124 |
|
125 |
-
|
126 |
-
|
127 |
-
|
128 |
-
|
129 |
-
bm25,
|
130 |
-
chunk_ids,
|
131 |
-
llm_ultimas_requests,
|
132 |
-
cast(
|
133 |
-
str, query_gerado_dinamicamente_para_o_vector_store.content
|
134 |
-
), # prompt_auxiliar_SEM_CONTEXT,
|
135 |
)
|
136 |
-
|
137 |
|
138 |
-
|
139 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
140 |
|
141 |
-
|
|
|
142 |
|
143 |
-
|
144 |
|
145 |
-
|
146 |
-
texto_completo = texto_completo + x["content"] + "\n"
|
147 |
-
x["source"]["text"] = x["source"]["text"][0:200]
|
148 |
-
x["source"]["context"] = x["source"]["context"][0:200]
|
149 |
|
150 |
-
|
151 |
-
|
|
|
|
|
152 |
|
153 |
texto_completo_como_html = convert_markdown_to_HTML(texto_completo).replace(
|
154 |
"resposta_segunda_etapa:", "<br><br>"
|
|
|
54 |
# Initialize enhanced summarizer
|
55 |
summarizer = GerarDocumento(serializer, axiom_instance)
|
56 |
|
57 |
+
all_PDFs_chunks, full_text_as_array = await get_full_text_and_all_PDFs_chunks(
|
58 |
+
listaPDFs,
|
59 |
+
summarizer.splitter,
|
60 |
+
serializer.should_use_llama_parse,
|
61 |
+
isBubble,
|
|
|
|
|
62 |
)
|
63 |
axiom_instance.send_axiom(
|
64 |
f"INÍCIO DO TEXTO COMPLETO DOS PDFS: {full_text_as_array[0:5]}"
|
65 |
)
|
66 |
|
67 |
+
is_contextualized_chunk = serializer.should_have_contextual_chunks
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
68 |
|
69 |
+
if is_contextualized_chunk:
|
70 |
+
response_auxiliar_summary = (
|
71 |
+
await get_response_from_auxiliar_contextual_prompt(full_text_as_array)
|
72 |
)
|
73 |
+
axiom_instance.send_axiom(
|
74 |
+
f"RESUMO INICIAL DO PROCESSO: {response_auxiliar_summary}"
|
|
|
|
|
75 |
)
|
76 |
|
77 |
+
axiom_instance.send_axiom("COMEÇANDO A FAZER AS REQUISIÇÕES DO CONTEXTUAL")
|
78 |
+
contextualized_chunks = await contextual_retriever.contextualize_all_chunks(
|
79 |
+
all_PDFs_chunks, response_auxiliar_summary, axiom_instance
|
80 |
+
)
|
81 |
+
axiom_instance.send_axiom(
|
82 |
+
"TERMINOU DE FAZER TODAS AS REQUISIÇÕES DO CONTEXTUAL"
|
83 |
+
)
|
84 |
+
chunks_processados = contextualized_chunks
|
85 |
axiom_instance.send_axiom(
|
86 |
+
f"CHUNKS PROCESSADOS INICIALMENTE: {chunks_processados}"
|
87 |
)
|
88 |
+
else:
|
89 |
+
chunks_processados = all_PDFs_chunks
|
90 |
+
|
91 |
+
if len(chunks_processados) == 0:
|
92 |
+
chunks_processados = all_PDFs_chunks
|
93 |
+
llm = LLM()
|
94 |
+
prompt_para_gerar_query_dinamico = prompt_gerar_query_dinamicamente(
|
95 |
+
cast(str, response_auxiliar_summary)
|
96 |
+
)
|
97 |
|
98 |
+
axiom_instance.send_axiom(
|
99 |
+
"COMEÇANDO REQUISIÇÃO PARA GERAR O QUERY DINAMICAMENTE DO VECTOR STORE"
|
100 |
+
)
|
101 |
+
query_gerado_dinamicamente_para_o_vector_store = (
|
102 |
+
await llm.google_gemini_ainvoke(
|
103 |
+
prompt_para_gerar_query_dinamico, "gemini-2.0-flash"
|
104 |
)
|
105 |
+
)
|
106 |
+
|
107 |
+
axiom_instance.send_axiom(
|
108 |
+
f"query_gerado_dinamicamente_para_o_vector_store: {query_gerado_dinamicamente_para_o_vector_store.content}",
|
109 |
+
)
|
110 |
|
111 |
+
# Create enhanced vector store and BM25 index
|
112 |
+
vector_store, bm25, chunk_ids = (
|
113 |
+
summarizer.vector_store.create_enhanced_vector_store(
|
114 |
+
chunks_processados, is_contextualized_chunk, axiom_instance
|
|
|
|
|
|
|
|
|
|
|
|
|
115 |
)
|
116 |
+
)
|
117 |
|
118 |
+
llm_ultimas_requests = serializer.llm_ultimas_requests
|
119 |
+
axiom_instance.send_axiom("COMEÇANDO A FAZER ÚLTIMA REQUISIÇÃO")
|
120 |
+
structured_summaries = await summarizer.gerar_documento_final(
|
121 |
+
vector_store,
|
122 |
+
bm25,
|
123 |
+
chunk_ids,
|
124 |
+
llm_ultimas_requests,
|
125 |
+
cast(
|
126 |
+
str, query_gerado_dinamicamente_para_o_vector_store.content
|
127 |
+
), # prompt_auxiliar_SEM_CONTEXT,
|
128 |
+
)
|
129 |
+
axiom_instance.send_axiom("TERMINOU DE FAZER A ÚLTIMA REQUISIÇÃO")
|
130 |
|
131 |
+
if not isinstance(structured_summaries, list):
|
132 |
+
from rest_framework.response import Response
|
133 |
|
134 |
+
return Response({"erro": structured_summaries})
|
135 |
|
136 |
+
texto_completo = summarizer.resumo_gerado + "\n\n"
|
|
|
|
|
|
|
137 |
|
138 |
+
for x in structured_summaries:
|
139 |
+
texto_completo = texto_completo + x["content"] + "\n"
|
140 |
+
x["source"]["text"] = x["source"]["text"][0:200]
|
141 |
+
x["source"]["context"] = x["source"]["context"][0:200]
|
142 |
|
143 |
texto_completo_como_html = convert_markdown_to_HTML(texto_completo).replace(
|
144 |
"resposta_segunda_etapa:", "<br><br>"
|
_utils/gerar_documento_utils/GerarDocumento.py
CHANGED
@@ -4,7 +4,7 @@ from typing import Any, List, Dict, Literal, Tuple, Optional, Union, cast
|
|
4 |
|
5 |
from pydantic import SecretStr
|
6 |
from _utils.langchain_utils.Chain_class import Chain
|
7 |
-
from _utils.langchain_utils.LLM_class import LLM
|
8 |
from _utils.langchain_utils.Prompt_class import Prompt
|
9 |
from _utils.langchain_utils.Vector_store_class import VectorStore
|
10 |
from gerar_documento.serializer import (
|
@@ -26,7 +26,6 @@ from _utils.models.gerar_documento import (
|
|
26 |
from cohere import Client
|
27 |
from _utils.langchain_utils.Splitter_class import Splitter
|
28 |
import time
|
29 |
-
|
30 |
from setup.logging import Axiom
|
31 |
|
32 |
|
|
|
4 |
|
5 |
from pydantic import SecretStr
|
6 |
from _utils.langchain_utils.Chain_class import Chain
|
7 |
+
from _utils.langchain_utils.LLM_class import LLM, Google_llms
|
8 |
from _utils.langchain_utils.Prompt_class import Prompt
|
9 |
from _utils.langchain_utils.Vector_store_class import VectorStore
|
10 |
from gerar_documento.serializer import (
|
|
|
26 |
from cohere import Client
|
27 |
from _utils.langchain_utils.Splitter_class import Splitter
|
28 |
import time
|
|
|
29 |
from setup.logging import Axiom
|
30 |
|
31 |
|
_utils/gerar_documento_utils/utils.py
CHANGED
@@ -106,13 +106,11 @@ async def get_full_text_and_all_PDFs_chunks(
|
|
106 |
splitterObject: Splitter,
|
107 |
should_use_llama_parse: bool,
|
108 |
isBubble: bool,
|
109 |
-
) -> Tuple[List[DocumentChunk], List[str]
|
110 |
all_PDFs_chunks: List[DocumentChunk] = []
|
111 |
|
112 |
pages: List[str] = []
|
113 |
|
114 |
-
vertex_response = None # Só terá valor se for necessário usar Vertex da Google para enviar o pdf e gerar resposta
|
115 |
-
|
116 |
# Load and process document
|
117 |
for pdf_path in listaPDFs:
|
118 |
chunks, pages = await splitterObject.load_and_split_document(
|
@@ -120,14 +118,7 @@ async def get_full_text_and_all_PDFs_chunks(
|
|
120 |
)
|
121 |
all_PDFs_chunks = all_PDFs_chunks + chunks
|
122 |
|
123 |
-
|
124 |
-
llm = LLM()
|
125 |
-
prompt = create_prompt_auxiliar_do_contextual_prompt(None)
|
126 |
-
vertex_response = await llm.google_gemini_vertex_ainvoke(
|
127 |
-
prompt, listaPDFs, "gemini-2.0-flash"
|
128 |
-
)
|
129 |
-
|
130 |
-
return all_PDFs_chunks, pages, vertex_response
|
131 |
|
132 |
|
133 |
async def generate_document_title(resumo_para_gerar_titulo: str):
|
|
|
106 |
splitterObject: Splitter,
|
107 |
should_use_llama_parse: bool,
|
108 |
isBubble: bool,
|
109 |
+
) -> Tuple[List[DocumentChunk], List[str]]:
|
110 |
all_PDFs_chunks: List[DocumentChunk] = []
|
111 |
|
112 |
pages: List[str] = []
|
113 |
|
|
|
|
|
114 |
# Load and process document
|
115 |
for pdf_path in listaPDFs:
|
116 |
chunks, pages = await splitterObject.load_and_split_document(
|
|
|
118 |
)
|
119 |
all_PDFs_chunks = all_PDFs_chunks + chunks
|
120 |
|
121 |
+
return all_PDFs_chunks, pages
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
122 |
|
123 |
|
124 |
async def generate_document_title(resumo_para_gerar_titulo: str):
|
_utils/google_integration/google_cloud.py
CHANGED
@@ -2,10 +2,12 @@ import os
|
|
2 |
from google.cloud import storage
|
3 |
|
4 |
GCP_PROJECT = "gen-lang-client-0350149082"
|
|
|
|
|
|
|
5 |
|
6 |
|
7 |
def upload_to_gcs(LOCAL_PDF_PATH: str) -> str:
|
8 |
-
GCS_BUCKET_NAME = "vella-pdfs"
|
9 |
|
10 |
# Path in GCS
|
11 |
GCS_DESTINATION_BLOB_NAME = "gemini_uploads/" + os.path.basename(LOCAL_PDF_PATH)
|
|
|
2 |
from google.cloud import storage
|
3 |
|
4 |
GCP_PROJECT = "gen-lang-client-0350149082"
|
5 |
+
GCP_REGION = "us-central1"
|
6 |
+
DOCUMENT_API_ID = "b34a20d22dee16bb"
|
7 |
+
GCS_BUCKET_NAME = "vella-pdfs"
|
8 |
|
9 |
|
10 |
def upload_to_gcs(LOCAL_PDF_PATH: str) -> str:
|
|
|
11 |
|
12 |
# Path in GCS
|
13 |
GCS_DESTINATION_BLOB_NAME = "gemini_uploads/" + os.path.basename(LOCAL_PDF_PATH)
|
_utils/langchain_utils/Splitter_class.py
CHANGED
@@ -1,3 +1,5 @@
|
|
|
|
|
|
1 |
from _utils.bubble_integrations.obter_arquivo import get_pdf_from_bubble
|
2 |
from _utils.handle_files import return_document_list_with_llama_parser
|
3 |
from _utils.langchain_utils.splitter_util import (
|
@@ -18,6 +20,16 @@ from _utils.models.gerar_documento import (
|
|
18 |
DocumentChunk,
|
19 |
)
|
20 |
import uuid
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
21 |
|
22 |
|
23 |
class Splitter:
|
@@ -34,7 +46,10 @@ class Splitter:
|
|
34 |
self.chunk_metadata = {} # Store chunk metadata for tracing
|
35 |
|
36 |
async def load_and_split_document(
|
37 |
-
self,
|
|
|
|
|
|
|
38 |
):
|
39 |
"""Load PDF and split into chunks with metadata"""
|
40 |
# loader = PyPDFLoader(pdf_path)
|
@@ -144,6 +159,11 @@ class Splitter:
|
|
144 |
# char_count += len(text)
|
145 |
print("TERMINOU DE ORGANIZAR PDFS EM CHUNKS")
|
146 |
|
|
|
|
|
|
|
|
|
|
|
147 |
return chunks, chunks_of_string_only
|
148 |
|
149 |
def load_and_split_text(self, text: str) -> List[DocumentChunk]:
|
@@ -185,3 +205,132 @@ class Splitter:
|
|
185 |
char_count += len(text)
|
186 |
|
187 |
return chunks
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import time
|
3 |
from _utils.bubble_integrations.obter_arquivo import get_pdf_from_bubble
|
4 |
from _utils.handle_files import return_document_list_with_llama_parser
|
5 |
from _utils.langchain_utils.splitter_util import (
|
|
|
20 |
DocumentChunk,
|
21 |
)
|
22 |
import uuid
|
23 |
+
import json
|
24 |
+
from _utils.google_integration.google_cloud import (
|
25 |
+
DOCUMENT_API_ID,
|
26 |
+
GCP_PROJECT,
|
27 |
+
GCP_REGION,
|
28 |
+
GCS_BUCKET_NAME,
|
29 |
+
upload_to_gcs,
|
30 |
+
)
|
31 |
+
from google.cloud import documentai
|
32 |
+
from google.cloud import storage
|
33 |
|
34 |
|
35 |
class Splitter:
|
|
|
46 |
self.chunk_metadata = {} # Store chunk metadata for tracing
|
47 |
|
48 |
async def load_and_split_document(
|
49 |
+
self,
|
50 |
+
pdf_path: str,
|
51 |
+
should_use_llama_parse: bool,
|
52 |
+
isBubble: bool,
|
53 |
):
|
54 |
"""Load PDF and split into chunks with metadata"""
|
55 |
# loader = PyPDFLoader(pdf_path)
|
|
|
159 |
# char_count += len(text)
|
160 |
print("TERMINOU DE ORGANIZAR PDFS EM CHUNKS")
|
161 |
|
162 |
+
if len(pages) == 0 or len(chunks) == 0:
|
163 |
+
text = await self.getOCRFromGoogleDocumentAPI(pdf_path)
|
164 |
+
chunks = self.load_and_split_text(text) # type: ignore
|
165 |
+
chunks_of_string_only = [chunk.content for chunk in chunks]
|
166 |
+
|
167 |
return chunks, chunks_of_string_only
|
168 |
|
169 |
def load_and_split_text(self, text: str) -> List[DocumentChunk]:
|
|
|
205 |
char_count += len(text)
|
206 |
|
207 |
return chunks
|
208 |
+
|
209 |
+
async def getOCRFromGoogleDocumentAPI(self, pdf_path: str):
|
210 |
+
|
211 |
+
pdf_gcs_uri = upload_to_gcs(pdf_path)
|
212 |
+
|
213 |
+
GCS_OUTPUT_PREFIX = "documentai_output/"
|
214 |
+
# GCS_INPUT_URI = f"gs://{GCS_BUCKET_NAME}/{f"gemini_uploads/{pdf_gcs_uri}"}"
|
215 |
+
GCS_INPUT_URI = pdf_gcs_uri
|
216 |
+
GCS_OUTPUT_URI = f"gs://{GCS_BUCKET_NAME}/{GCS_OUTPUT_PREFIX}"
|
217 |
+
|
218 |
+
docai_client = documentai.DocumentProcessorServiceClient()
|
219 |
+
|
220 |
+
processor_name = docai_client.processor_path(
|
221 |
+
project=GCP_PROJECT, location="us", processor=DOCUMENT_API_ID
|
222 |
+
)
|
223 |
+
|
224 |
+
gcs_document = documentai.GcsDocument(
|
225 |
+
gcs_uri=GCS_INPUT_URI,
|
226 |
+
mime_type="application/pdf", # Mime type is specified here for GcsDocument
|
227 |
+
)
|
228 |
+
|
229 |
+
gcs_documents = documentai.GcsDocuments(documents=[gcs_document])
|
230 |
+
|
231 |
+
# 3. Create the BatchDocumentsInputConfig
|
232 |
+
input_config = documentai.BatchDocumentsInputConfig(gcs_documents=gcs_documents)
|
233 |
+
# Note: If GCS_INPUT_URI was a prefix for multiple files, you'd use GcsPrefix:
|
234 |
+
# gcs_prefix = documentai.GcsPrefix(gcs_uri_prefix=GCS_INPUT_URI_PREFIX)
|
235 |
+
# input_config = documentai.BatchDocumentsInputConfig(gcs_prefix=gcs_prefix, mime_type="application/pdf")
|
236 |
+
|
237 |
+
# 4. Create the DocumentOutputConfig
|
238 |
+
# GCS_OUTPUT_URI should be a gs:// URI prefix where the output JSONs will be stored
|
239 |
+
output_config = documentai.DocumentOutputConfig(
|
240 |
+
gcs_output_config=documentai.DocumentOutputConfig.GcsOutputConfig(
|
241 |
+
gcs_uri=GCS_OUTPUT_URI
|
242 |
+
)
|
243 |
+
)
|
244 |
+
|
245 |
+
# 5. Construct the BatchProcessRequest
|
246 |
+
request = documentai.BatchProcessRequest(
|
247 |
+
name=processor_name,
|
248 |
+
input_documents=input_config, # Use 'input_documents'
|
249 |
+
document_output_config=output_config, # Use 'document_output_config'
|
250 |
+
)
|
251 |
+
|
252 |
+
# Submit the batch process request (this is a long-running operation)
|
253 |
+
operation = docai_client.batch_process_documents(request)
|
254 |
+
|
255 |
+
print("Batch processing operation started. Waiting for completion...")
|
256 |
+
while not operation.done():
|
257 |
+
time.sleep(15) # Wait for 30 seconds before checking again
|
258 |
+
print("Waiting...")
|
259 |
+
|
260 |
+
print("Batch processing operation finished.")
|
261 |
+
|
262 |
+
# --- Download the results from GCS ---
|
263 |
+
storage_client = storage.Client(
|
264 |
+
project=GCP_PROJECT
|
265 |
+
) # Uses GOOGLE_APPLICATION_CREDENTIALS/ADC
|
266 |
+
bucket = storage_client.bucket(GCS_BUCKET_NAME)
|
267 |
+
|
268 |
+
output_blobs = storage_client.list_blobs(
|
269 |
+
GCS_BUCKET_NAME, prefix=GCS_OUTPUT_PREFIX
|
270 |
+
)
|
271 |
+
|
272 |
+
downloaded_files_texts = []
|
273 |
+
try:
|
274 |
+
for blob in output_blobs:
|
275 |
+
# Document AI adds suffixes and subdirectories. Look for the actual JSON output files.
|
276 |
+
# The exact naming depends on the processor and options. Common pattern is ending with .json
|
277 |
+
if blob.name.endswith(".json"):
|
278 |
+
local_download_path = os.path.basename(
|
279 |
+
blob.name
|
280 |
+
) # Download to current directory with blob name
|
281 |
+
print(f"Downloading {blob.name} to {local_download_path}...")
|
282 |
+
blob.download_to_filename(local_download_path)
|
283 |
+
|
284 |
+
with open(local_download_path, "r", encoding="utf-8") as f:
|
285 |
+
document_data = json.load(f)
|
286 |
+
|
287 |
+
# The top-level 'text' field contains the concatenated plain text.
|
288 |
+
if "text" in document_data and document_data["text"] is not None:
|
289 |
+
raw_text = document_data["text"]
|
290 |
+
print(f"\n--- Raw Text Extracted from {blob.name} ---")
|
291 |
+
# Print only a snippet or process as needed
|
292 |
+
print(
|
293 |
+
raw_text[:1000] + "..."
|
294 |
+
if len(raw_text) > 1000
|
295 |
+
else raw_text
|
296 |
+
)
|
297 |
+
print("--------------------------------------------")
|
298 |
+
|
299 |
+
return raw_text
|
300 |
+
|
301 |
+
# Optional: Store the text. If you processed a batch of files,
|
302 |
+
# you might want to associate the text with the original file name.
|
303 |
+
# Document AI metadata might link output JSONs back to input files.
|
304 |
+
# For simplicity here, let's just show the extraction.
|
305 |
+
# If you know it was a single input PDF, this is all the text.
|
306 |
+
# If it was multiple, you'd need a mapping or process each JSON.
|
307 |
+
|
308 |
+
else:
|
309 |
+
print(
|
310 |
+
f"Warning: 'text' field not found in {blob.name} or is empty."
|
311 |
+
)
|
312 |
+
|
313 |
+
# Optional: Read and print a snippet of the JSON content
|
314 |
+
# with open(local_download_path, 'r', encoding='utf-8') as f:
|
315 |
+
# data = json.load(f)
|
316 |
+
# # Print some extracted text, for example (structure varies by processor)
|
317 |
+
# if 'text' in data:
|
318 |
+
# print(f"Extracted text snippet: {data['text'][:500]}...") # Print first 500 chars
|
319 |
+
# elif 'entities' in data:
|
320 |
+
# print(f"Number of entities found: {len(data['entities'])}")
|
321 |
+
# else:
|
322 |
+
# print("Output JSON structure not immediately recognizable.")
|
323 |
+
# break # Uncomment if you only expect/need to process the first output file
|
324 |
+
|
325 |
+
if len(downloaded_files_texts) == 0 or not downloaded_files_texts:
|
326 |
+
print("No JSON output files found in the specified output location.")
|
327 |
+
|
328 |
+
except Exception as e:
|
329 |
+
print(f"Error listing or downloading output files: {e}")
|
330 |
+
|
331 |
+
print("\nProcess complete.")
|
332 |
+
if downloaded_files_texts:
|
333 |
+
print(f"Downloaded output file(s): {', '.join(downloaded_files_texts)}")
|
334 |
+
print("These files contain the OCR results in JSON format.")
|
335 |
+
else:
|
336 |
+
print("No output files were successfully downloaded.")
|
_utils/langchain_utils/Vector_store_class.py
CHANGED
@@ -22,6 +22,8 @@ class VectorStore:
|
|
22 |
axiom_instance: Axiom,
|
23 |
) -> Tuple[Chroma, BM25Okapi, List[str]]:
|
24 |
"""Create vector store and BM25 index with contextualized chunks"""
|
|
|
|
|
25 |
try:
|
26 |
# Prepare texts with context
|
27 |
if is_contextualized_chunk:
|
@@ -69,5 +71,9 @@ class VectorStore:
|
|
69 |
return vector_store, bm25, chunk_ids
|
70 |
|
71 |
except Exception as e:
|
|
|
|
|
|
|
|
|
72 |
self.logger.error(f"Error creating enhanced vector store: {str(e)}")
|
73 |
-
|
|
|
22 |
axiom_instance: Axiom,
|
23 |
) -> Tuple[Chroma, BM25Okapi, List[str]]:
|
24 |
"""Create vector store and BM25 index with contextualized chunks"""
|
25 |
+
contador_erro = 0
|
26 |
+
|
27 |
try:
|
28 |
# Prepare texts with context
|
29 |
if is_contextualized_chunk:
|
|
|
71 |
return vector_store, bm25, chunk_ids
|
72 |
|
73 |
except Exception as e:
|
74 |
+
contador_erro += 1
|
75 |
+
if contador_erro >= 2:
|
76 |
+
raise Exception(f"Error creating enhanced vector store: {str(e)}")
|
77 |
+
|
78 |
self.logger.error(f"Error creating enhanced vector store: {str(e)}")
|
79 |
+
return self.create_enhanced_vector_store(chunks, False, axiom_instance)
|
requirements.txt
CHANGED
Binary files a/requirements.txt and b/requirements.txt differ
|
|