feat: adicionando melhorias e correções do contextual retriever

_antigos/resumos/serializer.py

@@ -25,5 +25,5 @@ class ResumoCursorSerializer(serializers.Serializer):
     user_message = serializers.CharField(required=False, default="")
     model = serializers.CharField(required=False, default=default_model)
     hf_embedding = serializers.CharField(required=False, default="all-MiniLM-L6-v2")
-    chunk_size = serializers.IntegerField(required=False, default=1000)
-    chunk_overlap = serializers.IntegerField(required=False, default=200)
+    chunk_size = serializers.IntegerField(required=False, default=5000)
+    chunk_overlap = serializers.IntegerField(required=False, default=1600)

_utils/gerar_relatorio_modelo_usuario/contextual_retriever.py

@@ -174,7 +174,8 @@ class ContextualRetriever:
                 ContextualizedChunk(
                     content=chunk.content,
                     page_number=chunk.page_number,
-                    chunk_id=result[index][0],
+                    id_do_processo=result[index][0],
+                    chunk_id=chunk.chunk_id,
                     start_char=chunk.start_char,
                     end_char=chunk.end_char,
                     context=" ".join(result[index][1:2]),

_utils/models/gerar_relatorio.py

@@ -10,6 +10,7 @@ class DocumentChunk:
     chunk_id: str
     start_char: int
     end_char: int
+    id_do_processo: int = 0
 
 
 @dataclass

_utils/vector_stores/Vector_store_class.py

@@ -21,7 +21,7 @@ class VectorStore:
             # Prepare texts with context
             if is_contextualized_chunk:
                 texts = [
-                    f"Document_id: {chunk.chunk_id}\nDocument_context: {chunk.context}\nDocument_content: {chunk.content}"
+                    f"Document_id: {chunk.id_do_processo}\nDocument_context: {chunk.context}\nDocument_content: {chunk.content}"
                     for chunk in chunks
                 ]
             else:
@@ -35,6 +35,7 @@ class VectorStore:
                     metadatas.append(
                         {
                             "chunk_id": chunk.chunk_id,
+                            "id_do_processo": chunk.id_do_processo,
                             "page": chunk.page_number,
                             "start_char": chunk.start_char,
                             "end_char": chunk.end_char,
@@ -46,6 +47,7 @@ class VectorStore:
                     metadatas.append(
                         {
                             "chunk_id": chunk.chunk_id,
+                            "id_do_processo": chunk.id_do_processo,
                             "page": chunk.page_number,
                             "start_char": chunk.start_char,
                             "end_char": chunk.end_char,

gerar_documento/serializer.py

@@ -33,7 +33,7 @@ class GerarDocumentoSerializer(ResumoCursorSerializer):
     embedding_weight = serializers.FloatField(default=0.5)
     bm25_weight = serializers.FloatField(default=0.5)
     context_window = serializers.IntegerField(default=3)
-    chunk_overlap = serializers.IntegerField(default=200)
+    chunk_overlap = serializers.IntegerField(default=1600)
     num_k_rerank = serializers.IntegerField(default=20)
     model_cohere_rerank = serializers.CharField(
         required=False, default="rerank-english-v2.0"
@@ -61,7 +61,7 @@ class GerarDocumentoComPDFProprioSerializer(ResumoCursorSerializer):
     embedding_weight = serializers.FloatField(default=0.5)
     bm25_weight = serializers.FloatField(default=0.5)
     context_window = serializers.IntegerField(default=3)
-    chunk_overlap = serializers.IntegerField(default=200)
+    chunk_overlap = serializers.IntegerField(default=1600)
     num_k_rerank = serializers.IntegerField(default=20)
     model_cohere_rerank = serializers.CharField(
         required=False, default="rerank-english-v2.0"