feat: refatorando serializers e nomes de algumas funções

_utils/custom_exception_handler.py

@@ -2,17 +2,18 @@
 
 from datetime import datetime
 import pytz
-from typing import Dict
+from typing import Any, Dict, Union
 from rest_framework.views import exception_handler
 import logging
 from _utils.bubble_integrations.enviar_resposta_final import enviar_resposta_final
+from gerar_documento.serializer import GerarDocumentoSerializerData
 
 logger = logging.getLogger(__name__)
 
 
 def custom_exception_handler(exc, context):
     print("---------------- CHEGOU NA FUNÇÃO PERSONALIZADA DE ERRO ----------------")
-    if (context):
+    if context:
         serializer: Dict = context["view"].serializer
     else:
         serializer = {}
@@ -32,19 +33,21 @@ def custom_exception_handler(exc, context):
     if response and str(response.status_code)[0] != "2":
         logger.error(f"Validation error: {response.data}")
 
-
     return response
 
-def custom_exception_handler_wihout_api_handler(error, serializer: Dict):
+
+def custom_exception_handler_wihout_api_handler(
+    error, serializer: Union[GerarDocumentoSerializerData, Any]
+):
     bahia_tz = pytz.timezone("America/Bahia")
     print("INICIANDO RESPOSTA DE ERRO PARA O BUBBLE")
     resposta_bubble = enviar_resposta_final(
-        serializer.get("doc_id", ""),
-        serializer.get("form_response_id", ""),
-        serializer.get("version", ""),
-        f"------------ ERRO NO BACKEND ÀS {datetime.now(bahia_tz).strftime("%d/%m/%Y - %H:%M:%S")} ------------:\nMensagem de erro: {error} ", # serializer.get("texto_completo", ""),
+        serializer.doc_id,
+        serializer.form_response_id,
+        serializer.version,
+        f"------------ ERRO NO BACKEND ÀS {datetime.now(bahia_tz).strftime("%d/%m/%Y - %H:%M:%S")} ------------:\nMensagem de erro: {error} ",  # serializer.get("texto_completo", ""),
         True,
     )
     print("\n\nresposta_bubble.status_code", resposta_bubble.status_code)
     print("\n\nresposta_bubble.text", resposta_bubble.text)
-    print("\n------------ MOTIVO DO ERRO -----------:", f"\n{error}")
+    print("\n------------ MOTIVO DO ERRO -----------:", f"\n{error}")

_utils/{resumo_completo_cursor.py → gerar_documento.py}

@@ -1,6 +1,6 @@
 import os
 from langchain_core.messages import HumanMessage
-from typing import cast
+from typing import Any, Union, cast
 from _utils.langchain_utils.LLM_class import LLM
 from _utils.bubble_integrations.enviar_resposta_final import enviar_resposta_final
 from _utils.custom_exception_handler import custom_exception_handler_wihout_api_handler
@@ -26,6 +26,7 @@ import markdown
 
 from _utils.langchain_utils.Prompt_class import Prompt
 from _utils.utils import convert_markdown_to_HTML
+from gerar_documento.serializer import GerarDocumentoSerializerData
 
 
 def reciprocal_rank_fusion(result_lists, weights=None):
@@ -53,36 +54,36 @@ os.environ.get("LANGCHAIN_API_KEY")
 os.environ["LANGCHAIN_PROJECT"] = "VELLA"
 
 
-async def get_llm_summary_answer_by_cursor_complete(
-    serializer, listaPDFs, isBubble=False
+async def gerar_documento(
+    serializer: Union[GerarDocumentoSerializerData, Any], listaPDFs, isBubble=False
 ):
     """Parâmetro "contexto" só deve ser passado quando quiser utilizar o teste com ragas, e assim, não quiser passar PDFs"""
     try:
         # Configuration
         config = RetrievalConfig(
-            num_chunks=serializer["num_chunks_retrieval"],
-            embedding_weight=serializer["embedding_weight"],
-            bm25_weight=serializer["bm25_weight"],
-            context_window=serializer["context_window"],
-            chunk_overlap=serializer["chunk_overlap"],
+            num_chunks=serializer.num_chunks_retrieval,
+            embedding_weight=serializer.embedding_weight,
+            bm25_weight=serializer.bm25_weight,
+            context_window=serializer.context_window,
+            chunk_overlap=serializer.chunk_overlap,
         )
 
         contextual_retriever = ContextualRetriever(
-            config, serializer["claude_context_model"]
+            config, serializer.claude_context_model
         )
 
         # Initialize enhanced summarizer
         summarizer = GerarDocumento(
             config=config,
-            embedding_model=serializer["hf_embedding"],
-            chunk_overlap=serializer["chunk_overlap"],
-            chunk_size=serializer["chunk_size"],
-            num_k_rerank=serializer["num_k_rerank"],
-            model_cohere_rerank=serializer["model_cohere_rerank"],
-            # prompt_auxiliar=serializer["prompt_auxiliar"],
-            gpt_model=serializer["model"],
-            gpt_temperature=serializer["gpt_temperature"],
-            prompt_gerar_documento=serializer["prompt_gerar_documento"],
+            embedding_model=serializer.hf_embedding,
+            chunk_overlap=serializer.chunk_overlap,
+            chunk_size=serializer.chunk_size,
+            num_k_rerank=serializer.num_k_rerank,
+            model_cohere_rerank=serializer.model_cohere_rerank,
+            # prompt_auxiliar=serializer.prompt_auxiliar,
+            gpt_model=serializer.model,
+            gpt_temperature=serializer.gpt_temperature,
+            prompt_gerar_documento=serializer.prompt_gerar_documento,
             reciprocal_rank_fusion=reciprocal_rank_fusion,
         )
 
@@ -90,12 +91,12 @@ async def get_llm_summary_answer_by_cursor_complete(
             await get_full_text_and_all_PDFs_chunks(
                 listaPDFs,
                 summarizer.splitter,
-                serializer["should_use_llama_parse"],
+                serializer.should_use_llama_parse,
                 isBubble,
             )
         )
 
-        is_contextualized_chunk = serializer["should_have_contextual_chunks"]
+        is_contextualized_chunk = serializer.should_have_contextual_chunks
 
         if is_contextualized_chunk:
             response_auxiliar_summary = (
@@ -133,7 +134,7 @@ async def get_llm_summary_answer_by_cursor_complete(
             )
         )
 
-        llm_ultimas_requests = serializer["llm_ultimas_requests"]
+        llm_ultimas_requests = serializer.llm_ultimas_requests
         print("\nCOMEÇANDO A FAZER ÚLTIMA REQUISIÇÃO")
         structured_summaries = await summarizer.gerar_documento_final(
             vector_store,
@@ -172,9 +173,9 @@ async def get_llm_summary_answer_by_cursor_complete(
         if isBubble:
             print("COMEÇANDO A REQUISIÇÃO FINAL PARA O BUBBLE")
             enviar_resposta_final(
-                serializer["doc_id"],
-                serializer["form_response_id"],
-                serializer["version"],
+                serializer.doc_id,
+                serializer.form_response_id,
+                serializer.version,
                 texto_completo_como_html,
                 False,
                 cast(str, titulo_do_documento),

_utils/gerar_relatorio_modelo_usuario/utils.py

@@ -1,4 +1,4 @@
-from typing import List, Tuple
+from typing import Any, List, Tuple, Union
 from langchain_core.documents import Document
 from langchain_core.messages import HumanMessage
 
@@ -10,28 +10,28 @@ from _utils.gerar_relatorio_modelo_usuario.prompts import (
 )
 import re
 
+from gerar_documento.serializer import GerarDocumentoSerializerData
 
-def gerar_resposta_compilada(serializer):
+
+def gerar_resposta_compilada(serializer: Union[GerarDocumentoSerializerData, Any]):
     return {
-        "num_chunks_retrieval": serializer["num_chunks_retrieval"],
-        "embedding_weight": serializer["embedding_weight"],
-        "bm25_weight": serializer["bm25_weight"],
-        "context_window": serializer["context_window"],
-        "chunk_overlap": serializer["chunk_overlap"],
-        "num_k_rerank": serializer["num_k_rerank"],
-        "model_cohere_rerank": serializer["model_cohere_rerank"],
-        "more_initial_chunks_for_reranking": serializer[
-            "more_initial_chunks_for_reranking"
-        ],
-        "claude_context_model": serializer["claude_context_model"],
-        "gpt_temperature": serializer["gpt_temperature"],
-        "user_message": serializer["user_message"],
-        "model": serializer["model"],
-        "hf_embedding": serializer["hf_embedding"],
-        "chunk_size": serializer["chunk_size"],
-        "chunk_overlap": serializer["chunk_overlap"],
-        # "prompt_auxiliar": serializer["prompt_auxiliar"],
-        "prompt_gerar_documento": serializer["prompt_gerar_documento"][0:200],
+        "num_chunks_retrieval": serializer.num_chunks_retrieval,
+        "embedding_weight": serializer.embedding_weight,
+        "bm25_weight": serializer.bm25_weight,
+        "context_window": serializer.context_window,
+        "chunk_overlap": serializer.chunk_overlap,
+        "num_k_rerank": serializer.num_k_rerank,
+        "model_cohere_rerank": serializer.model_cohere_rerank,
+        "more_initial_chunks_for_reranking": serializer.more_initial_chunks_for_reranking,
+        "claude_context_model": serializer.claude_context_model,
+        "gpt_temperature": serializer.gpt_temperature,
+        "user_message": serializer.user_message,
+        "model": serializer.model,
+        "hf_embedding": serializer.hf_embedding,
+        "chunk_size": serializer.chunk_size,
+        "chunk_overlap": serializer.chunk_overlap,
+        # "prompt_auxiliar": serializer.prompt_auxiliar,
+        "prompt_gerar_documento": serializer.prompt_gerar_documento[0:200],
     }
 
 

_utils/langchain_utils/embeddings.py

@@ -1,5 +1,3 @@
-import os
-
 from pydantic import Secret
 from setup.easy_imports import OpenAIEmbeddings
 from setup.tokens import openai_api_key

_utils/ragas.py

@@ -1,6 +1,6 @@
 import os
 from langchain_community.document_loaders import PyPDFLoader
-from _utils.resumo_completo_cursor import GerarDocumento, RetrievalConfig
+from _utils.gerar_documento import GerarDocumento, RetrievalConfig
 from rest_framework.response import Response
 from ragas import evaluate
 

gerar_documento/serializer.py

@@ -1,9 +1,12 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
 from rest_framework import serializers
 from _utils.gerar_relatorio_modelo_usuario.prompts import (
     prompt_gerar_documento,
     prompt_auxiliar_padrao,
 )
 from setup.environment import default_model
+from django.core.files.uploadedfile import UploadedFile
 
 user_message = "What are the main points of this document?"
 
@@ -29,12 +32,30 @@ class GerarDocumentoInitialSerializer(serializers.Serializer):
     chunk_overlap = serializers.IntegerField(required=False, default=800)
 
 
+@dataclass
+class GerarDocumentoInitialSerializerData:
+    files: List[dict]
+    system_prompt: str = prompt_template
+    user_message: str = ""
+    model: str = default_model
+    hf_embedding: str = "all-MiniLM-L6-v2"
+    chunk_size: int = 3500
+    chunk_overlap: int = 800
+
+
 class FileInfoSerializer(serializers.Serializer):
     unique_id = serializers.CharField(max_length=255)
     tipo_arquivo = serializers.CharField(max_length=255)
     link_arquivo = serializers.URLField()
 
 
+@dataclass
+class FileInfoSerializerData:
+    unique_id: str
+    tipo_arquivo: str
+    link_arquivo: str
+
+
 class GerarDocumentoSerializer(GerarDocumentoInitialSerializer):
     system_prompt = None
 
@@ -74,6 +95,34 @@ class GerarDocumentoSerializer(GerarDocumentoInitialSerializer):
     form_response_id = serializers.CharField(required=True)
     version = serializers.CharField(required=True)
 
+    def get_obj(self):
+        return GerarDocumentoSerializerData(**self.validated_data)  # type: ignore
+
+
+@dataclass
+class GerarDocumentoSerializerData(GerarDocumentoInitialSerializerData):
+    files: List[FileInfoSerializerData]
+    bubble_editor_version: str = "version-test"
+    prompt_gerar_documento: str = ""
+    user_message: str = ""
+    num_chunks_retrieval: int = 20
+    embedding_weight: float = 0.5
+    bm25_weight: float = 0.5
+    context_window: int = 3
+    chunk_overlap: int = 800
+    num_k_rerank: int = 20
+    model_cohere_rerank: str = "rerank-english-v2.0"
+    more_initial_chunks_for_reranking: int = 100
+    claude_context_model: str = "claude-3-haiku-20240307"
+    gpt_temperature: float = 0.0
+    id_modelo_do_usuario: Optional[int] = None
+    should_have_contextual_chunks: bool = False
+    should_use_llama_parse: bool = False
+    llm_ultimas_requests: str = "gemini-2.0-flash"
+    doc_id: str = ""
+    form_response_id: str = ""
+    version: str = ""
+
 
 class GerarDocumentoComPDFProprioSerializer(GerarDocumentoInitialSerializer):
     system_prompt = None
@@ -103,6 +152,29 @@ class GerarDocumentoComPDFProprioSerializer(GerarDocumentoInitialSerializer):
     should_use_llama_parse = serializers.BooleanField(required=False, default=False)  # type: ignore
     llm_ultimas_requests = serializers.CharField(required=False, default="gpt-4o-mini")
 
+    def get_obj(self):
+        return GerarDocumentoSerializerData(**self.validated_data)  # type: ignore
+
+
+@dataclass
+class GerarDocumentoComPDFProprioData(GerarDocumentoInitialSerializerData):
+    prompt_gerar_documento: Optional[str] = field(default=None)
+    user_message: Optional[str] = field(default=None)
+    num_chunks_retrieval: int = field(default=20)
+    embedding_weight: float = field(default=0.5)
+    bm25_weight: float = field(default=0.5)
+    context_window: int = field(default=3)
+    chunk_overlap: int = field(default=800)
+    num_k_rerank: int = field(default=20)
+    model_cohere_rerank: str = field(default="rerank-english-v2.0")
+    more_initial_chunks_for_reranking: int = field(default=100)
+    claude_context_model: str = field(default="claude-3-haiku-20240307")
+    gpt_temperature: float = field(default=0.0)
+    id_modelo_do_usuario: int = field(default=11)
+    should_have_contextual_chunks: bool = field(default=False)
+    should_use_llama_parse: bool = field(default=False)
+    llm_ultimas_requests: str = field(default="gpt-4o-mini")
+
 
 class GerarEmentaSerializer(serializers.Serializer):
     files = serializers.ListField(child=FileInfoSerializer(), required=True)

gerar_documento/views.py

@@ -14,8 +14,8 @@ from setup.easy_imports import (
 )
 from datetime import datetime
 from _utils.handle_files import handle_pdf_files_from_serializer, remove_pdf_temp_files
-from _utils.resumo_completo_cursor import (
-    get_llm_summary_answer_by_cursor_complete,
+from _utils.gerar_documento import (
+    gerar_documento,
 )
 from _utils.gerar_relatorio_modelo_usuario.prompts import prompt_auxiliar_inicio
 from .serializer import (
@@ -39,6 +39,7 @@ class GerarDocumentoView(AsyncAPIView):
         print("request.data: ", request.data)
         serializer = GerarDocumentoSerializer(data=request.data)
         if serializer.is_valid(raise_exception=True):
+            obj = serializer.get_obj()  # type: ignore
             if not serializer.validated_data:
                 raise ValueError("Erro no validated_data")
 
@@ -48,24 +49,15 @@ class GerarDocumentoView(AsyncAPIView):
                 print("\ndata: ", data)
                 self.serializer = data
 
-                # data["prompt_auxiliar"] = (
-                #     prompt_auxiliar_inicio + "\n" + data["prompt_auxiliar"]
-                # )
-
-                # listaPDFs = handle_pdf_files_from_serializer(data["files"])
                 listaPDFs = [l["link_arquivo"] for l in data["files"]]
 
                 print("\n\nlistaPDFs: ", listaPDFs)
 
-                resposta_llm = await get_llm_summary_answer_by_cursor_complete(
-                    data, listaPDFs, True
-                )
+                resposta_llm = await gerar_documento(obj, listaPDFs, True)
                 print("\n\nresposta_llm: ", resposta_llm)
 
                 # remove_pdf_temp_files(listaPDFs)
 
-                # print("PRÓXIMA LINHA ENVIA A RESPOSTA A QUEM FEZ A REQUISIÇÃO")
-
             # asyncio.create_task(proccess_data_after_response())
             loop = asyncio.get_running_loop()
             loop.run_in_executor(
@@ -89,14 +81,13 @@ class GerarDocumentoComPDFProprioView(AsyncAPIView):
         serializer = GerarDocumentoComPDFProprioSerializer(data=request.data)
         if serializer.is_valid(raise_exception=True):
             data = cast(Dict[str, Any], serializer.validated_data)
+            obj = serializer.get_obj()  # type: ignore
             print("\n\ndata: ", data)
             self.serializer = data
 
             listaPDFs = handle_pdf_files_from_serializer(data["files"])
 
-            resposta_llm = await get_llm_summary_answer_by_cursor_complete(
-                data, listaPDFs
-            )
+            resposta_llm = await gerar_documento(obj, listaPDFs)
             print("\n\nresposta_llm: ", resposta_llm)
 
             remove_pdf_temp_files(listaPDFs)

ragas_api/views.py

@@ -3,8 +3,8 @@ import tempfile, os
 from rest_framework.response import Response
 
 from _utils.ragas import test_ragas
-from _utils.resumo_completo_cursor import (
-    get_llm_summary_answer_by_cursor_complete,
+from _utils.gerar_documento import (
+    gerar_documento,
 )
 from .serializer import (
     RagasFromTextSerializer,
@@ -96,7 +96,7 @@ class RagasFromTextView(APIView):
 
             for i in range(len(reference)):
                 serializer.validated_data["user_message"] = data["user_input"][i]
-                resposta_llm = get_llm_summary_answer_by_cursor_complete(
+                resposta_llm = gerar_documento(
                     serializer.validated_data, contexto=reference[i]
                 )
                 data["response"].append(resposta_llm["texto_completo"])