fix: lidar com arquivos grandes com muitos tokens

_utils/gerar_relatorio_modelo_usuario/utils.py

@@ -89,35 +89,101 @@ def validate_many_chunks_in_one_request(
 
 
 # Esta função gera a resposta que será usada em cada um das requisições de cada chunk
-async def get_response_from_auxiliar_contextual_prompt(full_text_as_array: List[str]):
-    full_text = ""
-    for x in full_text_as_array:
-        full_text += x
+# async def get_response_from_auxiliar_contextual_prompt(
+#     full_text_as_array: List[str], full_text_as_string: str
+# ):
+#     print("full_text_as_string: ", full_text_as_string)
 
-    print("\nCRIANDO PROMPT AUXILIAR DO CONTEXTUAL")
-    # PROMPT PARA GERAR O RESUMO INICIAL DO PROCESSO
-    prompt_auxiliar_summary = create_prompt_auxiliar_do_contextual_prompt(full_text)
+#     print("\nCRIANDO PROMPT AUXILIAR DO CONTEXTUAL")
+#     # PROMPT PARA GERAR O RESUMO INICIAL DO PROCESSO
+#     prompt_auxiliar_summary = create_prompt_auxiliar_do_contextual_prompt(
+#         full_text_as_string
+#     )
 
-    print("\n\nprompt_auxiliar_summary[0:500]: ", prompt_auxiliar_summary[0:500])
+#     import tiktoken
+#     import re
 
-    # Claude comentado pois o limite de tokens estava sendo passado pela requisição e dava erro
-    # response_auxiliar_summary = await aclaude_answer(
-    #     self.claude_client, self.claude_context_model, prompt_auxiliar_summary
-    # )
+#     # full_text_as_string = re.sub(r"\s+", " ", full_text_as_string).strip()
 
-    llms = LLM()
-    print("\nCOMEÇANDO REQUISIÇÃO AUXILIAR DO CONTEXTUAL")
-    response_auxiliar_summary = await llms.google_gemini().ainvoke(
-        [HumanMessage(content=prompt_auxiliar_summary)]
-    )
-    print("TERMINOU REQUISIÇÃO AUXILIAR DO CONTEXTUAL")
+#     encoding = tiktoken.get_encoding("cl100k_base")
+#     # Count tokens
+#     num_tokens = len(encoding.encode(full_text_as_string))
+#     with open("output.txt", "w", encoding="utf-8") as file:
+#         file.write(full_text_as_string)
+#     print(f"CONTAGEM DE TOKENS - {num_tokens}")
 
-    print(
-        "\n\nresponse_auxiliar_summary.content[0:500]: ",
-        response_auxiliar_summary.content[0:500],
-    )
+#     print("\n\nprompt_auxiliar_summary[0:500]: ", prompt_auxiliar_summary[0:500])
+
+#     # Claude comentado pois o limite de tokens estava sendo passado pela requisição e dava erro
+#     # response_auxiliar_summary = await aclaude_answer(
+#     #     self.claude_client, self.claude_context_model, prompt_auxiliar_summary
+#     # )
+
+#     llms = LLM()
+#     print("\nCOMEÇANDO REQUISIÇÃO AUXILIAR DO CONTEXTUAL")
+#     response_auxiliar_summary = await llms.google_gemini().ainvoke(
+#         [HumanMessage(content=prompt_auxiliar_summary)]
+#     )
+#     print("TERMINOU REQUISIÇÃO AUXILIAR DO CONTEXTUAL")
+
+#     print(
+#         "\n\nresponse_auxiliar_summary.content[0:500]: ",
+#         response_auxiliar_summary.content[0:500],
+#     )
+
+#     return response_auxiliar_summary.content
+
+
+async def get_response_from_auxiliar_contextual_prompt(
+    full_text_as_array: List[str], full_text_as_string: str
+):
+    import tiktoken
+
+    encoding = tiktoken.get_encoding("cl100k_base")
+    llms = LLM()
+    responses = []
+
+    current_chunk = []
+    current_token_count = 0
+    chunk_counter = 1
+
+    for part in full_text_as_array:
+        part_tokens = len(encoding.encode(part))
+
+        # Check if adding this part would EXCEED the limit
+        if current_token_count + part_tokens > 600000:
+            # Process the accumulated chunk before it exceeds the limit
+            chunk_text = "".join(current_chunk)
+            print(
+                f"\nProcessing chunk {chunk_counter} with {current_token_count} tokens"
+            )
+
+            prompt = create_prompt_auxiliar_do_contextual_prompt(chunk_text)
+            response = await llms.google_gemini().ainvoke(
+                [HumanMessage(content=prompt)]
+            )
+            responses.append(response.content)
+
+            # Start new chunk with current part
+            current_chunk = [part]
+            current_token_count = part_tokens
+            chunk_counter += 1
+        else:
+            # Safe to add to current chunk
+            current_chunk.append(part)
+            current_token_count += part_tokens
+
+    # Process the final remaining chunk
+    if current_chunk:
+        chunk_text = "".join(current_chunk)
+        print(
+            f"\nProcessing final chunk {chunk_counter} with {current_token_count} tokens"
+        )
+        prompt = create_prompt_auxiliar_do_contextual_prompt(chunk_text)
+        response = await llms.google_gemini().ainvoke([HumanMessage(content=prompt)])
+        responses.append(response.content)
 
-    return response_auxiliar_summary.content
+    return "".join(responses)
 
 
 async def get_full_text_and_all_PDFs_chunks(
@@ -132,12 +198,14 @@ async def get_full_text_and_all_PDFs_chunks(
 
     # Load and process document
     for pdf_path in listaPDFs:
-        chunks, pages = await splitterObject.load_and_split_document(
-            pdf_path, should_use_llama_parse, isBubble
+        chunks, pages, full_text_as_string = (
+            await splitterObject.load_and_split_document(
+                pdf_path, should_use_llama_parse, isBubble
+            )
         )
         all_PDFs_chunks = all_PDFs_chunks + chunks
 
-    return all_PDFs_chunks, pages
+    return all_PDFs_chunks, pages, full_text_as_string
 
 
 async def generate_document_title(resumo_para_gerar_titulo: str):

_utils/main.py

@@ -1,73 +0,0 @@
-import os
-from _utils.utils import create_prompt_llm_chain, create_retriever, getPDF, create_llm, create_prompt_llm_chain_summary, process_embedding_summary
-from _utils import utils
-from langchain.chains import create_retrieval_chain
-from langchain_huggingface import HuggingFaceEmbeddings
-from langchain_chroma import Chroma
-from langchain_openai import OpenAIEmbeddings
-from langchain.chains.summarize import load_summarize_chain
-
-os.environ.get("OPENAI_API_KEY")
-
-def get_llm_answer(system_prompt, user_prompt, pdf_url, model, embedding):
-  if embedding == "gpt":
-    embedding_object = OpenAIEmbeddings()
-  else:
-    embedding_object = HuggingFaceEmbeddings(model_name=embedding)
-
-  vectorstore = Chroma(
-      collection_name="documents",
-      embedding_function=embedding_object
-    )
-
-  print('model: ', model)
-  print('embedding: ', embedding)
-  pages = []
-  if pdf_url:
-    pages = getPDF(pdf_url)
-  else:
-    pages = getPDF()
-  retriever = create_retriever(pages, vectorstore)
-  rag_chain = create_retrieval_chain(retriever, create_prompt_llm_chain(system_prompt, model))
-  results = rag_chain.invoke({"input": user_prompt})
-  # print('allIds ARQUIVO MAIN: ',  utils.allIds)
-  vectorstore.delete( utils.allIds)
-  vectorstore.delete_collection()
-  utils.allIds = []
-  # print('utils.allIds: ', utils.allIds)
-  return results
-
-def get_llm_answer_summary(system_prompt, user_prompt, pdf_url, model, isIterativeRefinement):
-  print('model: ', model)
-  print('isIterativeRefinement: ', isIterativeRefinement)
-  print('\n\n\n')
-  pages = getPDF(pdf_url)
-  if not isIterativeRefinement:
-    rag_chain = create_prompt_llm_chain_summary(system_prompt, model)
-
-    results = rag_chain.invoke({"input": user_prompt, "context": pages})
-
-    return results
-  else:
-    chain = load_summarize_chain(create_llm(model), "refine", True)
-    result = chain.invoke({"input_documents": pages})
-    print('result: ', result)
-    return result
-    # Obs --> Para passar informações personalizadas --> chain = load_summarize_chain(llm, "refine", True, question_prompt=initial_prompt, refine_prompt=PromptTemplate.from_template(refine_prompt))
-    # Para ver mais opções --> Acessa a origem da função load_summarize_chain , e nela acessa a origem da função _load_refine_chain --> As opções são os parâmetros que esta última função recebe
-
-def get_llm_answer_summary_with_embedding(system_prompt, user_prompt, pdf_url, model, isIterativeRefinement):
-  print('model: ', model)
-  print('isIterativeRefinement: ', isIterativeRefinement)
-  print('\n\n\n')
-  pages = getPDF(pdf_url)
-  full_texto = ""
-  for p in pages:
-    full_texto += p.page_content
-  print('full_texto: ', full_texto)
-
-  rag_chain = process_embedding_summary(system_prompt, model)
-
-  results = rag_chain.invoke({"input": user_prompt, "context": pages})
-
-  return results

_utils/resumo_completo_cursor.py

@@ -86,18 +86,22 @@ async def get_llm_summary_answer_by_cursor_complete(
             reciprocal_rank_fusion=reciprocal_rank_fusion,
         )
 
-        all_PDFs_chunks, full_text_as_array = await get_full_text_and_all_PDFs_chunks(
-            listaPDFs,
-            summarizer.splitter,
-            serializer["should_use_llama_parse"],
-            isBubble,
+        all_PDFs_chunks, full_text_as_array, full_text_as_string = (
+            await get_full_text_and_all_PDFs_chunks(
+                listaPDFs,
+                summarizer.splitter,
+                serializer["should_use_llama_parse"],
+                isBubble,
+            )
         )
 
         is_contextualized_chunk = serializer["should_have_contextual_chunks"]
 
         if is_contextualized_chunk:
             response_auxiliar_summary = (
-                await get_response_from_auxiliar_contextual_prompt(full_text_as_array)
+                await get_response_from_auxiliar_contextual_prompt(
+                    full_text_as_array, full_text_as_string
+                )
             )
 
             print("\nCOMEÇANDO A FAZER AS REQUISIÇÕES DO CONTEXTUAL")

_utils/splitters/Splitter_class.py

@@ -48,6 +48,9 @@ class Splitter:
             page_boundaries, combined_text = (
                 combine_documents_without_losing_pagination(pages)
             )
+            full_text_as_string = ""
+            for page in pages:
+                full_text_as_string = full_text_as_string + page.page_content
             initial_chunks = initial_chunks + self.text_splitter.split_text(
                 combined_text
             )
@@ -126,7 +129,7 @@ class Splitter:
             # char_count += len(text)
         print("TERMINOU DE ORGANIZAR PDFS EM CHUNKS")
 
-        return chunks, initial_chunks
+        return chunks, initial_chunks, full_text_as_string
 
     def load_and_split_text(self, text: str) -> List[DocumentChunk]:
         """Load Text and split into chunks with metadata - Criei essa função apenas para o ragas"""

gerar_documento/views.py

@@ -125,7 +125,7 @@ class GerarEmentaView(AsyncAPIView):
                 listaPDFs = [l["link_arquivo"] for l in data["files"]]
                 print("\n\nlistaPDFs: ", listaPDFs)
 
-                all_PDFs_chunks, full_text_as_array = (
+                all_PDFs_chunks, full_text_as_array, full_text_as_string = (
                     await get_full_text_and_all_PDFs_chunks(
                         listaPDFs,
                         Splitter(data["chunk_size"], data["chunk_overlap"]),
@@ -177,7 +177,7 @@ class GerarEmentaComPDFProprioView(AsyncAPIView):
             listaPDFs = [l["link_arquivo"] for l in data["files"]]
             print("\n\nlistaPDFs: ", listaPDFs)
 
-            all_PDFs_chunks, full_text_as_array = (
+            all_PDFs_chunks, full_text_as_array, full_text_as_string = (
                 await get_full_text_and_all_PDFs_chunks(
                     listaPDFs,
                     Splitter(data["chunk_size"], data["chunk_overlap"]),