feat: criando classe de utilitários

.gitignore

@@ -172,4 +172,5 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-# End of https://www.toptal.com/developers/gitignore/api/django
+# End of https://www.toptal.com/developers/gitignore/api/django
+vella_gcp_luan_credentials.json

_utils/Handle_Files_Class.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+from typing import List, Tuple
+
+from _utils.langchain_utils.Splitter_class import Splitter
+from _utils.models.gerar_documento import DocumentChunk
+
+
+@dataclass
+class HandleFilesClass:
+    async def get_full_text_and_all_PDFs_chunks(
+        self,
+        listaPDFs: List[str],
+        splitterObject: Splitter,
+        should_use_llama_parse: bool,
+        isBubble: bool,
+    ) -> Tuple[List[DocumentChunk], List[str]]:
+        all_PDFs_chunks: List[DocumentChunk] = []
+
+        pages: List[str] = []
+
+        # Load and process document
+        for pdf_path in listaPDFs:
+            chunks, pages = await splitterObject.load_and_split_document(
+                pdf_path, should_use_llama_parse, isBubble
+            )
+            all_PDFs_chunks = all_PDFs_chunks + chunks
+
+        return all_PDFs_chunks, pages

_utils/Utils_Class.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+from typing import List
+
+from _utils.Handle_Files_Class import HandleFilesClass
+
+
+@dataclass
+class UtilsClass:
+    lista_pdfs: List[str] | None = None
+    handle_files = HandleFilesClass()
+
+    def configure_lista_pdfs(self, lista_pdfs):
+        self.lista_pdfs = lista_pdfs

_utils/files/ragas_testset.csv

@@ -1,18 +0,0 @@
-position,user_input,reference,response,retrieved_contexts
-0,What are the global implications of the USA Supreme Court ruling on abortion?,"The global implications of the USA Supreme Court ruling on abortion are significant. The ruling has led to limited or no access to abortion for one in three women and girls of reproductive age in states where abortion access is restricted. These states also have weaker maternal health support, higher maternal death rates, and higher child poverty rates. Additionally, the ruling has had an impact beyond national borders due to the USA's geopolitical and cultural influence globally. Organizations and activists worldwide are concerned that the ruling may inspire anti-abortion legislative and policy attacks in other countries. The ruling has also hindered progressive law reform and the implementation of abortion guidelines in certain African countries. Furthermore, the ruling has created a chilling effect in international policy spaces, empowering anti-abortion actors to undermine human rights protections.","The global implications of the USA Supreme Court ruling on abortion can be significant, as it sets a precedent for other countries and influences the global discourse on reproductive rights. Here are some potential implications:
-
-1. Influence on other countries: The Supreme Court's ruling can serve as a reference point for other countries grappling with their own abortion laws. It can provide legal arguments and reasoning that advocates for reproductive rights can use to challenge restrictive abortion laws in their respective jurisdictions.
-
-2. Strengthening of global reproductive rights movements: A favorable ruling by the Supreme Court can energize and empower reproductive rights movements worldwide. It can serve as a rallying point for activists and organizations advocating for women's rights, leading to increased mobilization and advocacy efforts globally.
-
-3. Counteracting anti-abortion movements: Conversely, a ruling that restricts abortion rights can embolden anti-abortion movements globally. It can provide legitimacy to their arguments and encourage similar restrictive measures in other countries, potentially leading to a rollback of existing reproductive rights.
-
-4. Impact on international aid and policies: The Supreme Court's ruling can influence international aid and policies related to reproductive health. It can shape the priorities and funding decisions of donor countries and organizations, potentially leading to increased support for reproductive rights initiatives or conversely, restrictions on funding for abortion-related services.
-
-5. Shaping international human rights standards: The ruling can contribute to the development of international human rights standards regarding reproductive rights. It can influence the interpretation and application of existing human rights treaties and conventions, potentially strengthening the recognition of reproductive rights as fundamental human rights globally.
-
-6. Global health implications: The Supreme Court's ruling can have implications for global health outcomes, particularly in countries with restrictive abortion laws. It can impact the availability and accessibility of safe and legal abortion services, potentially leading to an increase in unsafe abortions and related health complications.
-
-It is important to note that the specific implications will depend on the nature of the Supreme Court ruling and the subsequent actions taken by governments, activists, and organizations both within and outside the United States.","[""- In 2022, the USA Supreme Court handed down a decision ruling that overturned 50 years of jurisprudence recognizing a constitutional right to abortion.\n- This decision has had a massive impact: one in three women and girls of reproductive age now live in states where abortion access is either totally or near-totally inaccessible.\n- The states with the most restrictive abortion laws have the weakest maternal health support, higher maternal death rates, and higher child poverty rates.\n- The USA Supreme Court ruling has also had impacts beyond national borders due to the geopolitical and cultural influence wielded by the USA globally and the aid it funds.\n- SRR organizations and activists across the world have expressed fear about the ruling laying the groundwork for anti-abortion legislative and policy attacks in other countries.\n- Advocates have also observed the ruling's impact on progressive law reform and the stalling of the adoption and enforcement of abortion guidelines in certain African countries.\n- The ruling has created a chilling effect in international policy spaces, emboldening anti-abortion state and non-state actors to undermine human rights protections.""
- 'The USA Supreme Court ruling on abortion has sparked intense debates and discussions not only within the country but also around the world. Many countries look to the United States as a leader in legal and social issues, so the decision could potentially influence the policies and attitudes towards abortion in other nations.'
- ""The ruling may also impact international organizations and non-governmental groups that work on reproductive rights and women's health issues. Depending on the outcome, there could be shifts in funding, advocacy efforts, and collaborations with American counterparts, leading to ripple effects in the global fight for reproductive justice.""]"

_utils/gerar_documento.py

@@ -1,6 +1,7 @@
 import os
 from langchain_core.messages import HumanMessage
 from typing import Any, Union, cast
+from _utils.Utils_Class import UtilsClass
 from _utils.langchain_utils.LLM_class import LLM
 from _utils.bubble_integrations.enviar_resposta_final import enviar_resposta_final
 from _utils.custom_exception_handler import custom_exception_handler_wihout_api_handler
@@ -16,7 +17,6 @@ from _utils.gerar_documento_utils.contextual_retriever import (
 from _utils.gerar_documento_utils.utils import (
     generate_document_title,
     gerar_resposta_compilada,
-    get_full_text_and_all_PDFs_chunks,
     get_response_from_auxiliar_contextual_prompt,
 )
 from _utils.models.gerar_documento import (
@@ -49,20 +49,23 @@ async def gerar_documento(
     isBubble=False,
 ):
     try:
+        axiom = axiom_instance.send_axiom
+        utils = UtilsClass()
+        handle_files = utils.handle_files
         contextual_retriever = ContextualRetriever(serializer)
-
-        # Initialize enhanced summarizer
         summarizer = GerarDocumento(serializer, axiom_instance)
 
-        all_PDFs_chunks, full_text_as_array = await get_full_text_and_all_PDFs_chunks(
-            listaPDFs,
-            summarizer.splitter,
-            serializer.should_use_llama_parse,
-            isBubble,
-        )
-        axiom_instance.send_axiom(
-            f"INÍCIO DO TEXTO COMPLETO DOS PDFS: {full_text_as_array[0:5]}"
+        params = {
+            "listaPDFs": listaPDFs,
+            "splitterObject": summarizer.splitter,
+            "should_use_llama_parse": serializer.should_use_llama_parse,
+            "isBubble": isBubble,
+        }
+
+        all_PDFs_chunks, full_text_as_array = (
+            await handle_files.get_full_text_and_all_PDFs_chunks(**params)
         )
+        axiom(f"INÍCIO DO TEXTO COMPLETO DOS PDFS: {full_text_as_array[0:5]}")
 
         is_contextualized_chunk = serializer.should_have_contextual_chunks
 
@@ -70,21 +73,15 @@ async def gerar_documento(
             response_auxiliar_summary = (
                 await get_response_from_auxiliar_contextual_prompt(full_text_as_array)
             )
-            axiom_instance.send_axiom(
-                f"RESUMO INICIAL DO PROCESSO: {response_auxiliar_summary}"
-            )
+            axiom(f"RESUMO INICIAL DO PROCESSO: {response_auxiliar_summary}")
 
-            axiom_instance.send_axiom("COMEÇANDO A FAZER AS REQUISIÇÕES DO CONTEXTUAL")
+            axiom("COMEÇANDO A FAZER AS REQUISIÇÕES DO CONTEXTUAL")
             contextualized_chunks = await contextual_retriever.contextualize_all_chunks(
                 all_PDFs_chunks, response_auxiliar_summary, axiom_instance
             )
-            axiom_instance.send_axiom(
-                "TERMINOU DE FAZER TODAS AS REQUISIÇÕES DO CONTEXTUAL"
-            )
+            axiom("TERMINOU DE FAZER TODAS AS REQUISIÇÕES DO CONTEXTUAL")
             chunks_processados = contextualized_chunks
-            axiom_instance.send_axiom(
-                f"CHUNKS PROCESSADOS INICIALMENTE: {chunks_processados}"
-            )
+            axiom(f"CHUNKS PROCESSADOS INICIALMENTE: {chunks_processados}")
         else:
             chunks_processados = all_PDFs_chunks
 
@@ -93,16 +90,14 @@ async def gerar_documento(
             cast(str, response_auxiliar_summary)
         )
 
-        axiom_instance.send_axiom(
-            "COMEÇANDO REQUISIÇÃO PARA GERAR O QUERY DINAMICAMENTE DO VECTOR STORE"
-        )
+        axiom("COMEÇANDO REQUISIÇÃO PARA GERAR O QUERY DINAMICAMENTE DO VECTOR STORE")
         query_gerado_dinamicamente_para_o_vector_store = (
             await llm.google_gemini_ainvoke(
                 prompt_para_gerar_query_dinamico, "gemini-2.0-flash"
             )
         )
 
-        axiom_instance.send_axiom(
+        axiom(
             f"query_gerado_dinamicamente_para_o_vector_store: {query_gerado_dinamicamente_para_o_vector_store.content}",
         )
 
@@ -114,7 +109,7 @@ async def gerar_documento(
         )
 
         llm_ultimas_requests = serializer.llm_ultimas_requests
-        axiom_instance.send_axiom("COMEÇANDO A FAZER ÚLTIMA REQUISIÇÃO")
+        axiom("COMEÇANDO A FAZER ÚLTIMA REQUISIÇÃO")
         structured_summaries = await summarizer.gerar_documento_final(
             vector_store,
             bm25,
@@ -124,7 +119,7 @@ async def gerar_documento(
                 str, query_gerado_dinamicamente_para_o_vector_store.content
             ),  # prompt_auxiliar_SEM_CONTEXT,
         )
-        axiom_instance.send_axiom("TERMINOU DE FAZER A ÚLTIMA REQUISIÇÃO")
+        axiom("TERMINOU DE FAZER A ÚLTIMA REQUISIÇÃO")
 
         if not isinstance(structured_summaries, list):
             from rest_framework.response import Response
@@ -141,9 +136,7 @@ async def gerar_documento(
         texto_completo_como_html = convert_markdown_to_HTML(texto_completo).replace(
             "resposta_segunda_etapa:", "<br><br>"
         )
-        axiom_instance.send_axiom(
-            f"texto_completo_como_html: {texto_completo_como_html}"
-        )
+        axiom(f"texto_completo_como_html: {texto_completo_como_html}")
 
         if is_contextualized_chunk:
             prompt_titulo_do_documento = response_auxiliar_summary
@@ -154,7 +147,7 @@ async def gerar_documento(
         )
 
         if isBubble:
-            axiom_instance.send_axiom("COMEÇANDO A REQUISIÇÃO FINAL PARA O BUBBLE")
+            axiom("COMEÇANDO A REQUISIÇÃO FINAL PARA O BUBBLE")
             enviar_resposta_final(
                 serializer.doc_id,  # type: ignore
                 serializer.form_response_id,  # type: ignore
@@ -163,7 +156,7 @@ async def gerar_documento(
                 False,
                 cast(str, titulo_do_documento),
             )
-            axiom_instance.send_axiom("TERMINOU A REQUISIÇÃO FINAL PARA O BUBBLE")
+            axiom("TERMINOU A REQUISIÇÃO FINAL PARA O BUBBLE")
 
         return {
             "texto_completo": texto_completo_como_html,

_utils/gerar_documento_utils/GerarDocumento.py

@@ -26,7 +26,7 @@ from _utils.models.gerar_documento import (
 from cohere import Client
 from _utils.langchain_utils.Splitter_class import Splitter
 import time
-
+from setup.tokens import openai_api_key, cohere_api_key
 from setup.logging import Axiom
 
 
@@ -81,10 +81,22 @@ class GerarDocumentoUtils:
         else:
             return documento_gerado
 
+    def create_retrieval_config(
+        self,
+        serializer: Union[
+            GerarDocumentoSerializerData, GerarDocumentoComPDFProprioSerializerData, Any
+        ],
+    ):
+        return RetrievalConfig(
+            num_chunks=serializer.num_chunks_retrieval,
+            embedding_weight=serializer.embedding_weight,
+            bm25_weight=serializer.bm25_weight,
+            context_window=serializer.context_window,
+            chunk_overlap=serializer.chunk_overlap,
+        )
+
 
 class GerarDocumento:
-    openai_api_key = os.environ.get("OPENAI_API_KEY", "")
-    cohere_api_key = os.environ.get("COHERE_API_KEY", "")
     resumo_gerado = ""
     gerar_documento_utils = GerarDocumentoUtils()
 
@@ -95,21 +107,14 @@ class GerarDocumento:
         ],
         axiom_instance: Axiom,
     ):
-        self.config = RetrievalConfig(
-            num_chunks=serializer.num_chunks_retrieval,
-            embedding_weight=serializer.embedding_weight,
-            bm25_weight=serializer.bm25_weight,
-            context_window=serializer.context_window,
-            chunk_overlap=serializer.chunk_overlap,
-        )
+        self.config = self.gerar_documento_utils.create_retrieval_config(serializer)
         self.logger = logging.getLogger(__name__)
         # self.prompt_auxiliar = prompt_auxiliar
         self.gpt_model = serializer.model
         self.gpt_temperature = serializer.gpt_temperature
         self.prompt_gerar_documento = serializer.prompt_gerar_documento
 
-        self.openai_api_key = self.openai_api_key
-        self.cohere_client = Client(self.cohere_api_key)
+        self.cohere_client = Client(cohere_api_key)
         self.embeddings = HuggingFaceEmbeddings(model_name=serializer.hf_embedding)
         self.num_k_rerank = serializer.num_k_rerank
         self.model_cohere_rerank = serializer.model_cohere_rerank
@@ -219,7 +224,7 @@ class GerarDocumento:
             llm = ChatOpenAI(
                 temperature=self.gpt_temperature,
                 model=self.gpt_model,
-                api_key=SecretStr(self.openai_api_key),
+                api_key=SecretStr(openai_api_key),
             )
         elif llm_ultimas_requests == "deepseek-chat":
             llm = llm_instance.deepseek()

_utils/gerar_documento_utils/utils.py

@@ -101,26 +101,6 @@ def split_text_by_tokens(full_text: str):
     return text_chunks
 
 
-async def get_full_text_and_all_PDFs_chunks(
-    listaPDFs: List[str],
-    splitterObject: Splitter,
-    should_use_llama_parse: bool,
-    isBubble: bool,
-) -> Tuple[List[DocumentChunk], List[str]]:
-    all_PDFs_chunks: List[DocumentChunk] = []
-
-    pages: List[str] = []
-
-    # Load and process document
-    for pdf_path in listaPDFs:
-        chunks, pages = await splitterObject.load_and_split_document(
-            pdf_path, should_use_llama_parse, isBubble
-        )
-        all_PDFs_chunks = all_PDFs_chunks + chunks
-
-    return all_PDFs_chunks, pages
-
-
 async def generate_document_title(resumo_para_gerar_titulo: str):
     prompt = f"Você é um assistente jurídico e irá receber abaixo o resumo de um documento jurídico. Quero que você gere um título para este documento. Mande como resposta apenas o título gerado, nada mais. Aqui está um título de exemplo pra você se basear ao criar um novo: <titulo_de_exemplo>Ação Penal por Furto Qualificado nº 0002269-86.2009.805.0032<titulo_de_exemplo>\n\nSegue abaixo o resumo do documento jurídico:\n{resumo_para_gerar_titulo}"
     response = await agemini_answer(prompt, "gemini-2.0-flash-lite")

gerar_documento/views.py

@@ -1,5 +1,6 @@
 from typing import Any, Dict, cast
 from langchain.prompts import PromptTemplate
+from _utils.Utils_Class import UtilsClass
 from _utils.bubble_integrations.enviar_resposta_final import enviar_resposta_final
 from _utils.custom_exception_handler import custom_exception_handler_wihout_api_handler
 from _utils.gerar_documento_utils.GerarDocumento import GerarDocumento
@@ -7,7 +8,6 @@ from _utils.langchain_utils.LLM_class import LLM
 from _utils.gerar_documento_utils.utils import (
     generate_document_title,
     gerar_resposta_compilada,
-    get_full_text_and_all_PDFs_chunks,
     split_text_by_tokens,
 )
 from _utils.langchain_utils.Prompt_class import Prompt
@@ -138,6 +138,8 @@ class GerarEmentaView(AsyncAPIView):
 
             async def proccess_data_after_response():
                 try:
+                    util = UtilsClass()
+                    handle_files = util.handle_files
                     data = cast(Dict[str, Any], serializer.validated_data)
                     self.serializer = data
 
@@ -148,7 +150,7 @@ class GerarEmentaView(AsyncAPIView):
                     self.axiom_instance.send_axiom(f"listaPDFs: {listaPDFs}")
 
                     all_PDFs_chunks, full_text_as_array = (
-                        await get_full_text_and_all_PDFs_chunks(
+                        await handle_files.get_full_text_and_all_PDFs_chunks(
                             listaPDFs,
                             Splitter(obj.chunk_size, obj.chunk_overlap),
                             False,
@@ -239,6 +241,8 @@ class GerarEmentaComPDFProprioView(AsyncAPIView):
             f"COMEÇOU NOVA REQUISIÇÃO - request.data: {request.data}"
         )
         serializer = GerarDocumentoComPDFProprioSerializer(data=request.data)
+        util = UtilsClass()
+        handle_files = util.handle_files
         if serializer.is_valid(raise_exception=True):
             data = cast(Dict[str, Any], serializer.validated_data)
             self.axiom_instance.send_axiom(f"data: {data}")
@@ -255,7 +259,7 @@ class GerarEmentaComPDFProprioView(AsyncAPIView):
             self.axiom_instance.send_axiom(f"listaPDFs: {listaPDFs}")
 
             all_PDFs_chunks, full_text_as_array = (
-                await get_full_text_and_all_PDFs_chunks(
+                await handle_files.get_full_text_and_all_PDFs_chunks(
                     listaPDFs,
                     Splitter(serializer_obj.chunk_size, serializer_obj.chunk_overlap),
                     False,