feat: removendo arquivos e pastas desnecessárias

_antigos/pdfs/admin.py

@@ -1,7 +0,0 @@
-from django.contrib import admin
-
-# from pdfs.models import PDFsModel
-
-# Register your models here.
-
-# admin.site.register(PDFsModel)

_antigos/pdfs/apps.py

@@ -1,6 +0,0 @@
-from django.apps import AppConfig
-
-
-class PdfsConfig(AppConfig):
-    default_auto_field = "django.db.models.BigAutoField"
-    name = "pdfs"

_antigos/pdfs/migrations/0001_initial.py

@@ -1,21 +0,0 @@
-# Generated by Django 4.1 on 2024-11-09 22:42
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='EndpointTesteModel',
-            fields=[
-                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('teste', models.CharField(max_length=300)),
-            ],
-        ),
-    ]

_antigos/pdfs/migrations/0002_delete_endpointtestemodel.py

@@ -1,16 +0,0 @@
-# Generated by Django 4.1 on 2024-11-16 00:46
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('pdfs', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.DeleteModel(
-            name='EndpointTesteModel',
-        ),
-    ]

_antigos/pdfs/models.py

@@ -1,4 +0,0 @@
-from django.db import models
-
-# Create your models here.
-# class PDFsModel(models.Model):

_antigos/pdfs/serializer.py

@@ -1,8 +0,0 @@
-from rest_framework import serializers
-
-class PDFUploadSerializer(serializers.Serializer):
-    files = serializers.ListField(child=serializers.FileField(), required=True)
-    system_prompt = serializers.CharField(required=True)
-    user_message = serializers.CharField(required=True)
-    model = serializers.CharField(required=False)
-    embedding = serializers.CharField(required=False)

_antigos/pdfs/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

_antigos/pdfs/views.py

@@ -1,52 +0,0 @@
-import tempfile, os
-from pdfs.serializer import PDFUploadSerializer
-from setup.environment import default_model
-from drf_spectacular.utils import extend_schema
-
-from rest_framework.decorators import api_view, parser_classes
-from rest_framework.parsers import MultiPartParser
-from rest_framework.response import Response
-
-from _utils.main import get_llm_answer
-
-@extend_schema(
-    request=PDFUploadSerializer,
-)
-@api_view(["POST"])
-@parser_classes([MultiPartParser])
-def getPDF(request):
-    if request.method == "POST":
-        serializer = PDFUploadSerializer(data=request.data)
-        if serializer.is_valid(raise_exception=True):
-            listaPDFs = []
-            print('\n\n')
-            data = request.data
-            print('data: ', data)
-            embedding = serializer.validated_data.get("embedding", "gpt")
-            model = serializer.validated_data.get("model", default_model)
-
-            # pdf_file = serializer.validated_data['file']
-            for file in serializer.validated_data['files']:
-                print("file: ", file)
-                file.seek(0)
-                # Create a temporary file to save the uploaded PDF
-                with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:
-                    # Write the uploaded file content to the temporary file
-                    for chunk in file.chunks():
-                        temp_file.write(chunk)
-                    temp_file_path = temp_file.name  # Get the path of the temporary file
-                    listaPDFs.append(temp_file_path)
-                # print('temp_file_path: ', temp_file_path)
-                print('listaPDFs: ', listaPDFs)
-
-            resposta_llm = None
-            # resposta_llm = get_llm_answer(data["system_prompt"], data["user_message"], temp_file_path, model=model, embedding=embedding)
-            resposta_llm = get_llm_answer(data["system_prompt"], data["user_message"], listaPDFs, model=model, embedding=embedding)
-
-            for file in listaPDFs:
-                os.remove(file)
-            # os.remove(temp_file_path)
-
-            return Response({
-                "Resposta": resposta_llm
-            })

_antigos/resumos/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

_antigos/resumos/apps.py

@@ -1,6 +0,0 @@
-from django.apps import AppConfig
-
-
-class ResumosConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'resumos'

_antigos/resumos/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

_antigos/resumos/serializer.py

@@ -1,29 +0,0 @@
-from rest_framework import serializers
-from setup.environment import default_model
-# from _utils.utils import DEFAULT_SYSTEM_PROMPT
-
-prompt_template = """
-    Based on the following context, provide multiple key points from the document.
-    For each point, create a new paragraph.
-    Each paragraph should be a complete, self-contained insight.
-    
-    Context: {context}
-    
-    Key points:
-    """
-
-class ResumoPDFSerializer(serializers.Serializer):
-    files = serializers.ListField(child=serializers.FileField(), required=True)
-    system_prompt = serializers.CharField(required=False)
-    user_message = serializers.CharField(required=False, default="")
-    model = serializers.CharField(required=False)
-    iterative_refinement = serializers.BooleanField(required=False, default=False)  # type: ignore
-
-class ResumoCursorSerializer(serializers.Serializer):
-    files = serializers.ListField(child=serializers.FileField(), required=True)
-    system_prompt = serializers.CharField(required=False, default=prompt_template)
-    user_message = serializers.CharField(required=False, default="")
-    model = serializers.CharField(required=False, default=default_model)
-    hf_embedding = serializers.CharField(required=False, default="all-MiniLM-L6-v2")
-    chunk_size = serializers.IntegerField(required=False, default=3500)
-    chunk_overlap = serializers.IntegerField(required=False, default=800)

_antigos/resumos/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

_antigos/resumos/views.py

@@ -1,144 +0,0 @@
-from rest_framework.views import APIView
-import tempfile, os
-from rest_framework.response import Response
-from _utils.resumo_simples_cursor import get_llm_summary_answer_by_cursor
-from _utils.utils import DEFAULT_SYSTEM_PROMPT
-from .serializer import (
-    ResumoPDFSerializer,
-    ResumoCursorSerializer,
-)
-from _utils.main import get_llm_answer_summary, get_llm_answer_summary_with_embedding
-from setup.environment import default_model
-from rest_framework.parsers import MultiPartParser
-from drf_spectacular.utils import extend_schema
-
-
-class ResumoView(APIView):
-    parser_classes = [MultiPartParser]
-
-    @extend_schema(
-        request=ResumoPDFSerializer,
-    )
-    def post(self, request):
-        serializer = ResumoPDFSerializer(data=request.data)
-        if serializer.is_valid(raise_exception=True):
-            listaPDFs = []
-            data = serializer.validated_data
-            model = serializer.validated_data.get("model", default_model)
-            print("serializer.validated_data: ", serializer.validated_data)
-
-            for file in serializer.validated_data["files"]:
-                print("file: ", file)
-                file.seek(0)
-                with tempfile.NamedTemporaryFile(
-                    delete=False, suffix=".pdf"
-                ) as temp_file:  # Create a temporary file to save the uploaded PDF
-                    for (
-                        chunk
-                    ) in (
-                        file.chunks()
-                    ):  # Write the uploaded file content to the temporary file
-                        temp_file.write(chunk)
-                    temp_file_path = (
-                        temp_file.name
-                    )  # Get the path of the temporary file
-                    listaPDFs.append(temp_file_path)
-                # print('listaPDFs: ', listaPDFs)
-
-            system_prompt = data.get("system_prompt", DEFAULT_SYSTEM_PROMPT)
-            resposta_llm = get_llm_answer_summary(
-                system_prompt,
-                data["user_message"],
-                listaPDFs,
-                model=model,
-                isIterativeRefinement=data["iterative_refinement"],
-            )
-
-            for file in listaPDFs:
-                os.remove(file)
-
-            return Response({"resposta": resposta_llm})
-
-
-class ResumoEmbeddingView(APIView):
-    parser_classes = [MultiPartParser]
-
-    @extend_schema(
-        request=ResumoPDFSerializer,
-    )
-    def post(self, request):
-        serializer = ResumoPDFSerializer(data=request.data)
-        if serializer.is_valid(raise_exception=True):
-            listaPDFs = []
-            data = serializer.validated_data
-            model = serializer.validated_data.get("model", default_model)
-            print("serializer.validated_data: ", serializer.validated_data)
-
-            for file in serializer.validated_data["files"]:
-                file.seek(0)
-                with tempfile.NamedTemporaryFile(
-                    delete=False, suffix=".pdf"
-                ) as temp_file:  # Create a temporary file to save the uploaded PDF
-                    for (
-                        chunk
-                    ) in (
-                        file.chunks()
-                    ):  # Write the uploaded file content to the temporary file
-                        temp_file.write(chunk)
-                    temp_file_path = (
-                        temp_file.name
-                    )  # Get the path of the temporary file
-                    listaPDFs.append(temp_file_path)
-            print("listaPDFs: ", listaPDFs)
-
-            system_prompt = data.get("system_prompt", DEFAULT_SYSTEM_PROMPT)
-            resposta_llm = get_llm_answer_summary_with_embedding(
-                system_prompt,
-                data["user_message"],
-                listaPDFs,
-                model=model,
-                isIterativeRefinement=data["iterative_refinement"],
-            )
-
-            for file in listaPDFs:
-                os.remove(file)
-
-            return Response({"resposta": resposta_llm})
-
-
-class ResumoSimplesCursorView(APIView):
-    parser_classes = [MultiPartParser]
-
-    @extend_schema(
-        request=ResumoCursorSerializer,
-    )
-    def post(self, request):
-        serializer = ResumoCursorSerializer(data=request.data)
-        if serializer.is_valid(raise_exception=True):
-            listaPDFs = []
-            data = serializer.validated_data
-            print("\nserializer.validated_data: ", serializer.validated_data)
-
-            for file in serializer.validated_data["files"]:
-                file.seek(0)
-                with tempfile.NamedTemporaryFile(
-                    delete=False, suffix=".pdf"
-                ) as temp_file:  # Create a temporary file to save the uploaded PDF
-                    for (
-                        chunk
-                    ) in (
-                        file.chunks()
-                    ):  # Write the uploaded file content to the temporary file
-                        temp_file.write(chunk)
-                    temp_file_path = (
-                        temp_file.name
-                    )  # Get the path of the temporary file
-                    listaPDFs.append(temp_file_path)
-            print("listaPDFs: ", listaPDFs)
-
-            resposta_llm = get_llm_summary_answer_by_cursor(data, listaPDFs)
-
-            for file in listaPDFs:
-                os.remove(file)
-
-            return Response({"resposta": resposta_llm})

_utils/resumo_simples_cursor.py

@@ -1,234 +0,0 @@
-import os
-from typing import List, Dict, Tuple
-from setup.easy_imports import (
-    HuggingFaceEmbeddings,
-    PyPDFLoader,
-    Chroma,
-    ChatOpenAI,
-    create_extraction_chain,
-    PromptTemplate,
-    RecursiveCharacterTextSplitter,
-)
-from dataclasses import dataclass
-import uuid
-import json
-from langchain_huggingface import HuggingFaceEndpoint
-from setup.environment import default_model
-
-os.environ["LANGCHAIN_TRACING_V2"] = "true"
-os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com"
-os.environ.get("LANGCHAIN_API_KEY")
-os.environ["LANGCHAIN_PROJECT"] = "VELLA"
-
-
-@dataclass
-class DocumentChunk:
-    content: str
-    page_number: int
-    chunk_id: str
-    start_char: int
-    end_char: int
-
-
-class DocumentSummarizer:
-
-    def __init__(
-        self, openai_api_key: str, model, embedding, chunk_config, system_prompt
-    ):
-        self.model = model
-        self.system_prompt = system_prompt
-        self.openai_api_key = openai_api_key
-        self.embeddings = HuggingFaceEmbeddings(model_name=embedding)
-        self.text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=chunk_config["size"], chunk_overlap=chunk_config["overlap"]
-        )
-        self.chunk_metadata = {}  # Store chunk metadata for tracing
-
-    def load_and_split_document(self, pdf_path: str) -> List[DocumentChunk]:
-        """Load PDF and split into chunks with metadata"""
-        loader = PyPDFLoader(pdf_path)
-        pages = loader.load()
-        chunks = []
-        char_count = 0
-
-        for page in pages:
-            text = page.page_content
-            # Split the page content
-            page_chunks = self.text_splitter.split_text(text)
-
-            for chunk in page_chunks:
-                chunk_id = str(uuid.uuid4())
-                start_char = text.find(chunk)
-                end_char = start_char + len(chunk)
-
-                doc_chunk = DocumentChunk(
-                    content=chunk,
-                    page_number=page.metadata.get("page") + 1,  # 1-based page numbering
-                    chunk_id=chunk_id,
-                    start_char=char_count + start_char,
-                    end_char=char_count + end_char,
-                )
-                chunks.append(doc_chunk)
-
-                # Store metadata for later retrieval
-                self.chunk_metadata[chunk_id] = {
-                    "page": doc_chunk.page_number,
-                    "start_char": doc_chunk.start_char,
-                    "end_char": doc_chunk.end_char,
-                }
-
-            char_count += len(text)
-
-        return chunks
-
-    def create_vector_store(self, chunks: List[DocumentChunk]) -> Chroma:
-        """Create vector store with metadata"""
-        texts = [chunk.content for chunk in chunks]
-        metadatas = [
-            {
-                "chunk_id": chunk.chunk_id,
-                "page": chunk.page_number,
-                "start_char": chunk.start_char,
-                "end_char": chunk.end_char,
-            }
-            for chunk in chunks
-        ]
-
-        vector_store = Chroma.from_texts(
-            texts=texts, metadatas=metadatas, embedding=self.embeddings
-        )
-        return vector_store
-
-    def generate_summary_with_sources(
-        self,
-        vector_store: Chroma,
-        query: str = "Summarize the main points of this document",
-    ) -> List[Dict]:
-        """Generate summary with source citations, returning structured JSON data"""
-        # Retrieve relevant chunks with metadata
-        relevant_docs = vector_store.similarity_search_with_score(query, k=5)
-
-        # Prepare context and track sources
-        contexts = []
-        sources = []
-
-        for doc, score in relevant_docs:
-            chunk_id = doc.metadata["chunk_id"]
-            context = doc.page_content
-            contexts.append(context)
-
-            sources.append(
-                {
-                    "content": context,
-                    "page": doc.metadata["page"],
-                    "chunk_id": chunk_id,
-                    "relevance_score": score,
-                }
-            )
-
-        prompt = PromptTemplate(
-            template=self.system_prompt, input_variables=["context"]
-        )
-        llm = ""
-
-        if self.model == default_model:
-            llm = ChatOpenAI(
-                temperature=0, model_name="gpt-4o-mini", api_key=self.openai_api_key
-            )
-        else:
-            llm = HuggingFaceEndpoint(
-                repo_id=self.model,
-                task="text-generation",
-                max_new_tokens=1100,
-                do_sample=False,
-                huggingfacehub_api_token=os.environ.get("HUGGINGFACEHUB_API_TOKEN"),
-            )
-
-        response = llm.invoke(prompt.format(context="\n\n".join(contexts))).content
-
-        # Split the response into paragraphs
-        summaries = [p.strip() for p in response.split("\n\n") if p.strip()]
-
-        # Create structured output
-        structured_output = []
-        for idx, summary in enumerate(summaries):
-            # Associate each summary with the most relevant source
-            structured_output.append(
-                {
-                    "content": summary,
-                    "source": {
-                        "page": sources[min(idx, len(sources) - 1)]["page"],
-                        "text": sources[min(idx, len(sources) - 1)]["content"][:200]
-                        + "...",
-                        "relevance_score": sources[min(idx, len(sources) - 1)][
-                            "relevance_score"
-                        ],
-                    },
-                }
-            )
-
-        return structured_output
-
-    def get_source_context(self, chunk_id: str, window: int = 100) -> Dict:
-        """Get extended context around a specific chunk"""
-        metadata = self.chunk_metadata.get(chunk_id)
-        if not metadata:
-            return None
-
-        return {
-            "page": metadata["page"],
-            "start_char": metadata["start_char"],
-            "end_char": metadata["end_char"],
-        }
-
-
-def get_llm_summary_answer_by_cursor(serializer, listaPDFs):
-    # By Luan
-    allPdfsChunks = []
-
-    # Initialize summarizer
-    summarizer = DocumentSummarizer(
-        openai_api_key=os.environ.get("OPENAI_API_KEY"),
-        embedding=serializer["hf_embedding"],
-        chunk_config={
-            "size": serializer["chunk_size"],
-            "overlap": serializer["chunk_overlap"],
-        },
-        system_prompt=serializer["system_prompt"],
-        model=serializer["model"],
-    )
-
-    # Load and process document
-    for pdf in listaPDFs:
-        pdf_path = pdf
-        chunks = summarizer.load_and_split_document(pdf_path)
-        allPdfsChunks = allPdfsChunks + chunks
-
-    vector_store = summarizer.create_vector_store(allPdfsChunks)
-
-    # Generate structured summary
-    structured_summaries = summarizer.generate_summary_with_sources(vector_store)
-
-    # Print or return the structured data
-    # print(structured_summaries)
-    json_data = json.dumps(structured_summaries)
-    print("\n\n")
-    print(json_data)
-    return structured_summaries
-    # If you need to send to frontend, you can just return structured_summaries
-    # It will be in the format:
-    # [
-    #     {
-    #         "content": "Summary point 1...",
-    #         "source": {
-    #             "page": 1,
-    #             "text": "Source text...",
-    #             "relevance_score": 0.95
-    #         }
-    #     },
-    #     ...
-    # ]
-
-
-if __name__ == "__main__":
-    get_llm_summary_answer_by_cursor()