Remove indexer options + add scope extractor functions + implemented on single document search

app.py

@@ -5,9 +5,15 @@ import requests
 from bs4 import BeautifulSoup
 import json
 import os
+import pymupdf as fitz
+import uuid
+import zipfile
+import io
+import subprocess
+import os
+import re
 import time
 from datetime import datetime
-import traceback
 from dotenv import load_dotenv
 import warnings
 from fastapi import FastAPI, HTTPException
@@ -38,6 +44,63 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+def get_pdf_bytes(specification: str, version: str):
+    doc_id = specification
+    series = doc_id.split(".")[0]
+    response = requests.get(f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version}.zip", verify=False)
+    if response.status_code != 200:
+        raise Exception("Téléchargement du ZIP échoué")
+
+    zip_bytes = io.BytesIO(response.content)
+
+    with zipfile.ZipFile(zip_bytes) as zf:
+        for file_name in zf.namelist():
+            if file_name.endswith("doc") or file_name.endswith("docx"):
+                ext = file_name.split(".")[-1]
+                doc_bytes = zf.read(file_name)
+                temp_id = str(uuid.uuid4())
+                input_path = f"/tmp/{temp_id}.{ext}"
+                output_path = f"/tmp/{temp_id}.pdf"
+
+                with open(input_path, "wb") as f:
+                    f.write(doc_bytes)
+                
+                subprocess.run([
+                    "libreoffice",
+                    "--headless",
+                    "--convert-to", "pdf",
+                    "--outdir", "/tmp",
+                    input_path
+                ], check=True)
+
+                with open(output_path, "rb") as f:
+                    pdf_data = f.read()
+                
+                os.remove(input_path)
+                os.remove(output_path)
+
+                return io.BytesIO(pdf_data)
+        raise Exception("Aucun fichier .doc/.docx trouvé dans le ZIP")
+
+def get_scope(specification: str, version: str):
+    pdf_bytes = get_pdf_bytes(specification, version)
+    doc = fitz.open(stream=pdf_bytes, filetype="pdf")
+
+    for content in doc.get_toc():
+        if "scope" in content[1].lower():
+            page_num = content[2] - 1
+            break
+
+    doc = doc[page_num:]
+
+    pdf_full_text = " ".join(page.get_text("text") for page in doc)
+    pdf_postprocess_text = re.sub(r"\s+", " ", pdf_full_text)
+    pdf_postprocess_text = pdf_postprocess_text.replace("1 Scope", " !-! ")
+    pdf_postprocess_text = pdf_postprocess_text.replace("2 Reference", " !-! ")
+    pdf_postprocess_text = pdf_postprocess_text.replace("", "- ")
+
+    return pdf_postprocess_text.split(" !-! ")[1]
+
 class DocRequest(BaseModel):
     doc_id: str
     release: Optional[int] = None
@@ -45,6 +108,7 @@ class DocRequest(BaseModel):
 class DocResponse(BaseModel):
     doc_id: str
     url: str
+    scope: Optional[str] = None
     search_time: float
 
 class BatchDocRequest(BaseModel):
@@ -294,6 +358,7 @@ def search_spec(request: KeywordRequest):
         )
     else:
         raise HTTPException(status_code=404, detail="Specification not found")
+    
 @app.post("/find", response_model=DocResponse)
 def find_document(request: DocRequest):
     start_time = time.time()
@@ -301,13 +366,18 @@ def find_document(request: DocRequest):
     print(finder)
     
     result = finder.search_document(request.doc_id, request.release)
-    print(result)
 
     if "not found" not in result and "Could not" not in result and "Unable" not in result:
+        version = result.split("/")[-1].replace(".zip", "").split("-")[-1]
         return DocResponse(
             doc_id=request.doc_id,
             url=result,
             search_time=time.time() - start_time
+        ) if isinstance(finder, TsgDocFinder) else DocResponse(
+            doc_id=request.doc_id,
+            url=result,
+            search_time=time.time() - start_time,
+            scope=get_scope(request.doc_id, version)
         )
     else:
         raise HTTPException(status_code=404, detail=result)

static/script.js

@@ -2,12 +2,10 @@
 const singleModeBtn = document.getElementById('single-mode-btn');
 const batchModeBtn = document.getElementById('batch-mode-btn');
 const keywordModeBtn = document.getElementById("keyword-mode-btn");
-const indexerModeBtn = document.getElementById("indexer-mode-btn")
 
 const singleInput = document.querySelector('.single-input');
 const batchInput = document.querySelector('.batch-input');
 const keywordSearchInput = document.querySelector(".keyword-input");
-const indexerButtons = document.querySelector(".indexer-buttons")
 
 const docIdInput = document.getElementById('doc-id');
 const batchIdsInput = document.getElementById('batch-ids');
@@ -28,23 +26,19 @@ singleModeBtn.addEventListener('click', () => {
     singleModeBtn.classList.add('active');
     keywordModeBtn.classList.remove("active");
     batchModeBtn.classList.remove('active');
-    indexerModeBtn.classList.remove("active");
 
     singleInput.style.display = 'block';
     batchInput.style.display = 'none';
     keywordSearchInput.style.display = "none";
-    indexerButtons.style.display = "none";
 });
 
 batchModeBtn.addEventListener('click', () => {
     batchModeBtn.classList.add('active');
     keywordModeBtn.classList.remove("active");
     singleModeBtn.classList.remove('active');
-    indexerModeBtn.classList.remove("active");
 
     batchInput.style.display = 'block';
     keywordSearchInput.style.display = "none";
-    indexerButtons.style.display = "none";
     singleInput.style.display = 'none';
 });
 
@@ -52,26 +46,12 @@ keywordModeBtn.addEventListener('click', () => {
     keywordModeBtn.classList.add("active");
     singleModeBtn.classList.remove('active');
     batchModeBtn.classList.remove("active");
-    indexerModeBtn.classList.remove("active");
 
     singleInput.style.display = "none";
     batchInput.style.display = "none";
-    indexerButtons.style.display = "none";
     keywordSearchInput.style.display = "block";
 })
 
-indexerModeBtn.addEventListener('click', ()=>{
-    keywordModeBtn.classList.remove("active");
-    singleModeBtn.classList.remove('active');
-    batchModeBtn.classList.remove("active");
-    indexerModeBtn.classList.add("active");
-
-    singleInput.style.display = "none";
-    batchInput.style.display = "none";
-    indexerButtons.style.display = "block";
-    keywordSearchInput.style.display = "none";
-})
-
 keywordSearchBtn.addEventListener("click", async ()=>{
     const keywords = keywordInput.value.trim();
     if (!keywords) {
@@ -191,6 +171,7 @@ function displaySingleResult(data) {
     
     const resultItem = document.createElement('div');
     resultItem.className = 'result-item';
+    let scopeItem = data.scope ? `<p>Scope : ${data.scope}</p>` : ""
     resultItem.innerHTML = `
         <div class="result-header">
             <div class="result-id">${data.doc_id}</div>
@@ -198,6 +179,7 @@ function displaySingleResult(data) {
         </div>
         <div class="result-url">
             <a href="${data.url}" target="_blank">${data.url}</a>
+            ${scopeItem}
         </div>
     `;
     

static/style.css

@@ -171,10 +171,6 @@ header {
     display: none;
 }
 
-.indexer-buttons {
-    display: none;
-}
-
 .batch-input textarea {
     width: 100%;
     height: 120px;

templates/index.html

@@ -28,7 +28,6 @@
                 <button id="single-mode-btn" class="active">Single Document</button>
                 <button id="batch-mode-btn">Batch Search</button>
                 <button id="keyword-mode-btn">Keyword Search</button>
-                <button id="indexer-mode-btn">Indexer Options</button>
             </div>
 
             <div class="search-form">
@@ -54,14 +53,6 @@
                         <button id="keyword-search-btn" class="btn">Search</button>
                     </div>
                 </div>
-
-                <div class="input-group indexer-buttons">
-                    <label for="indexerBtns">Actions</label>
-                    <div class="input-field">
-                        <button id="indexing-btn" class="btn">Index all files</button>
-                        <button id="testing-btn" class="btn">Test theory</button>
-                    </div>
-                </div>
             </div>
 
             <div class="error-message" id="error-message"></div>