V2 (Add keyword search)

app.py

@@ -1,96 +1,61 @@
-from io import StringIO
-from dotenv import load_dotenv
-import numpy as np
-import pandas as pd
-import requests
-from bs4 import BeautifulSoup
-import json
-import os
-import traceback
-import uuid
-import zipfile
-import io
-import subprocess
-import os
-import re
 import time
 from datetime import datetime
-import warnings
+import os, warnings, nltk, json, re
+import numpy as np
+from nltk.stem import WordNetLemmatizer
+from dotenv import load_dotenv
+from sklearn.preprocessing import MinMaxScaler
+
+os.environ['CURL_CA_BUNDLE'] = ""
+warnings.filterwarnings('ignore')
+nltk.download('wordnet')
+load_dotenv()
+
+from datasets import load_dataset
+import bm25s
+from bm25s.hf import BM25HF
+
 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
-from pydantic import BaseModel
-from typing import Any, Dict, List, Literal, Optional
-
-load_dotenv()
+from schemas import *
+from bs4 import BeautifulSoup
+import requests
 
-warnings.filterwarnings("ignore")
+lemmatizer = WordNetLemmatizer()
 
-app = FastAPI(title="ETSI Document Finder API", 
-              description="API to find ETSI documents based on document IDs")
+spec_metadatas = load_dataset("OrganizedProgrammers/ETSISpecMetadata", token=os.environ["HF_TOKEN"])
+spec_contents = load_dataset("OrganizedProgrammers/ETSISpecContent", token=os.environ["HF_TOKEN"])
+bm25_index = BM25HF.load_from_hub("OrganizedProgrammers/ETSIBM25IndexSingle", load_corpus=True, token=os.environ["HF_TOKEN"])
 
-app.mount("/static", StaticFiles(directory="static"), name="static")
+spec_metadatas = spec_metadatas["train"].to_list()
+spec_contents = spec_contents["train"].to_list()
 
-origins = [
-    "*",
-]
+def get_document(spec_id: str, spec_title: Optional[str]):
+    text = [f"{spec_id} - {spec_title}" if spec_title else f"{spec_id}"]
+    for section in spec_contents:
+        if spec_id == section["doc_id"]:
+            text.extend([section['section'], section['content']])
+    return text
 
+app = FastAPI(title="3GPP Document Finder Back-End", description="Backend for 3GPPDocFinder - Searching technical documents & specifications from 3GPP FTP server")
+app.mount("/static", StaticFiles(directory="static"), name="static")
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=origins,
+    allow_origins=["*"],
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
-class DocRequest(BaseModel):
-    doc_id: str
-
-class DocResponse(BaseModel):
-    doc_id: str
-    url: str
-    release: Optional[str] = None
-    scope: Optional[str] = None
-    search_time: float
-
-class MultiDocResponse(BaseModel):
-    doc_id: str
-    results: List[DocResponse]
-    search_time: float
-
-class BatchDocRequest(BaseModel):
-    doc_ids: List[str]
-    release: Optional[int] = None
-
-class BatchDocResponse(BaseModel):
-    results: Dict[str, str]
-    missing: List[str]
-    search_time: float
-
 class DocFinder:
     def __init__(self):
         self.main_ftp_url = "https://docbox.etsi.org/SET"
         self.session = requests.Session()
-        self.indexer_file = "indexed_docs.json"
-        self.indexer, self.last_indexer_date = self.load_indexer()
         req = self.session.post("https://portal.etsi.org/ETSIPages/LoginEOL.ashx", verify=False, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"}, data=json.dumps({"username": os.environ.get("EOL_USER"), "password": os.environ.get("EOL_PASSWORD")}))
         print(req.content, req.status_code)
     
-    def load_indexer(self):
-        if os.path.exists(self.indexer_file):
-            with open(self.indexer_file, "r", encoding="utf-8") as f:
-                x = json.load(f)
-                return x["docs"], x["last_indexed_date"]
-        return {}, None
-
-    def save_indexer(self):
-        today = datetime.today()
-        self.last_indexer_date = today.strftime("%d/%m/%Y-%H:%M:%S")
-        with open(self.indexer_file, "w", encoding="utf-8") as f:
-            output = {"docs": self.indexer, "last_indexed_date": self.last_indexer_date}
-            json.dump(output, f, indent=4, ensure_ascii=False)
-    
     def get_workgroup(self, doc: str):
         main_tsg = "SET-WG-R" if any(doc.startswith(kw) for kw in ["SETREQ", "SCPREQ"]) else "SET-WG-T" if any(doc.startswith(kw) for kw in ["SETTEC", "SCPTEC"]) else "SET" if any(doc.startswith(kw) for kw in ["SET", "SCP"]) else None
         if main_tsg is None:
@@ -120,12 +85,6 @@ class DocFinder:
     
     def search_document(self, doc_id: str):
         original = doc_id
-
-        if original in self.indexer:
-            return self.indexer[original]
-        for doc in self.indexer:
-            if doc.startswith(original):
-                return self.indexer[doc]
         
         main_tsg, workgroup, doc = self.get_workgroup(doc_id)
         urls = []
@@ -139,34 +98,14 @@ class DocFinder:
                     if doc in f.lower() or original in f:
                         print(f)
                         doc_url = f"{wg_url}/{f}"
-                        
-                        self.indexer[original] = doc_url
-                        self.save_indexer()
                         urls.append(doc_url)
         return urls[0] if len(urls) == 1 else urls[-2] if len(urls) > 1 else f"Document {doc_id} not found"
 
 class SpecFinder:
     def __init__(self):
         self.main_url = "https://www.etsi.org/deliver/etsi_ts"
-        self.indexer_file = "indexed_specifications.json"
-        self.indexer, self.last_indexer_date = self.load_indexer()
         self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"}
         
-    
-    def load_indexer(self):
-        if os.path.exists(self.indexer_file):
-            with open(self.indexer_file, "r", encoding="utf-8") as f:
-                x = json.load(f)
-                return x["specs"], x["last_indexed_date"]
-        return {}, None
-
-    def save_indexer(self):
-        today = datetime.today()
-        self.last_indexer_date = today.strftime("%d/%m/%Y-%H:%M:%S")
-        with open(self.indexer_file, "w", encoding="utf-8") as f:
-            output = {"specs": self.indexer, "last_indexed_date": self.last_indexer_date}
-            json.dump(output, f, indent=4, ensure_ascii=False)
-
     def get_spec_path(self, doc_id: str):
         if "-" in doc_id:
             position, part = doc_id.split("-")
@@ -194,12 +133,6 @@ class SpecFinder:
         # Example : 103 666[-2 opt]
         original = doc_id
 
-        if original in self.indexer:
-            return self.indexer[original]
-        for doc in self.indexer:
-            if doc.startswith(original):
-                return self.indexer[doc]
-
         url = f"{self.main_url}/{self.get_spec_path(original)}/"
         print(url)
         
@@ -253,4 +186,148 @@ def find_documents_batch(request: BatchDocRequest):
         results=results,
         missing=missing,
         search_time=time.time() - start_time
-    )
+    )
+
+
+@app.post("/search-spec", response_model=KeywordResponse)
+def search_specification_by_keywords(request: KeywordRequest):
+    start_time = time.time()
+    boolSensitiveCase = request.case_sensitive
+    search_mode = request.search_mode
+    spec_type = request.spec_type
+    keywords = [string.lower() if boolSensitiveCase else string for string in request.keywords.split(",")]
+    print(keywords)
+    unique_specs = set()
+    results = []
+
+    if keywords == [""] and search_mode == "deep":
+        raise HTTPException(status_code=400, detail="You must enter keywords in deep search mode !")
+
+    for spec in spec_metadatas:
+        valid = False
+        if spec['id'] in unique_specs: continue
+        if spec.get('type', None) is None or (spec_type is not None and spec["type"] != spec_type): continue
+        
+        if search_mode == "deep":
+            contents = []
+            doc = get_document(spec["id"], spec["title"])
+            docValid = len(doc) > 1
+        
+        if request.mode == "and":
+            string = f"{spec['id']}+-+{spec['title']}+-+{spec['type']}+-+{spec['version']}"
+            if all(keyword in (string.lower() if boolSensitiveCase else string) for keyword in keywords):
+                valid = True
+            if search_mode == "deep":
+                if docValid:
+                    for x in range(1, len(doc) - 1, 2):
+                        section_title = doc[x]
+                        section_content = doc[x+1]
+                        if "reference" not in section_title.lower() and "void" not in section_title.lower() and "annex" not in section_content.lower():
+                            if all(keyword in (section_content.lower() if boolSensitiveCase else section_content) for keyword in keywords):
+                                valid = True
+                                contents.append({section_title: section_content})
+        elif request.mode == "or":
+            string = f"{spec['id']}+-+{spec['title']}+-+{spec['type']}+-+{spec['version']}"
+            if any(keyword in (string.lower() if boolSensitiveCase else string) for keyword in keywords):
+                valid = True
+            if search_mode == "deep":
+                if docValid:
+                    for x in range(1, len(doc) - 1, 2):
+                        section_title = doc[x]
+                        section_content = doc[x+1]
+                        if "reference" not in section_title.lower() and "void" not in section_title.lower() and "annex" not in section_content.lower():
+                            if any(keyword in (section_content.lower() if boolSensitiveCase else section_content) for keyword in keywords):
+                                valid = True
+                                contents.append({section_title: section_content})
+        if valid:
+            spec_content = spec
+            if search_mode == "deep":
+                spec_content["contains"] = {k: v for d in contents for k, v in d.items()}
+            results.append(spec_content)
+        else:
+            unique_specs.add(spec['id'])
+    
+    if len(results) > 0:
+        return KeywordResponse(
+            results=results,
+            search_time=time.time() - start_time
+        )
+    else:
+        raise HTTPException(status_code=404, detail="Specifications not found")
+    
+@app.post("/search-spec/experimental", response_model=KeywordResponse)
+def bm25_search_specification(request: BM25KeywordRequest):
+    start_time = time.time()
+    spec_type = request.spec_type
+    threshold = request.threshold
+    query = request.keywords
+
+    results_out = []
+    query_tokens = bm25s.tokenize(query)
+    results, scores = bm25_index.retrieve(query_tokens, k=len(bm25_index.corpus))
+    print("BM25 raw scores:", scores)
+
+    def calculate_boosted_score(metadata, score, query):
+        title = set(metadata['title'].lower().split())
+        q = set(query.lower().split())
+        spec_id_presence = 0.5 if metadata['id'].lower() in q else 0
+        booster = len(q & title) * 0.5
+        return score + spec_id_presence + booster
+
+    spec_scores = {}
+    spec_indices = {}
+    spec_details = {}
+
+    for i in range(results.shape[1]):
+        doc = results[0, i]
+        score = scores[0, i]
+        spec = doc["metadata"]["id"]
+
+        boosted_score = calculate_boosted_score(doc['metadata'], score, query)
+
+        if spec not in spec_scores or boosted_score > spec_scores[spec]:
+            spec_scores[spec] = boosted_score
+            spec_indices[spec] = i
+            spec_details[spec] = {
+                'original_score': score,
+                'boosted_score': boosted_score,
+                'doc': doc
+            }
+
+    def normalize_scores(scores_dict):
+        if not scores_dict:
+            return {}
+        
+        scores_array = np.array(list(scores_dict.values())).reshape(-1, 1)
+        scaler = MinMaxScaler()
+        normalized_scores = scaler.fit_transform(scores_array).flatten()
+        
+        normalized_dict = {}
+        for i, spec in enumerate(scores_dict.keys()):
+            normalized_dict[spec] = normalized_scores[i]
+        
+        return normalized_dict
+
+    normalized_scores = normalize_scores(spec_scores)
+
+    for spec in spec_details:
+        spec_details[spec]["normalized_score"] = normalized_scores[spec]
+
+    unique_specs = sorted(normalized_scores.keys(), key=lambda x: normalized_scores[x], reverse=True)
+    
+    for rank, spec in enumerate(unique_specs, 1):
+        details = spec_details[spec]
+        metadata = details['doc']['metadata']
+        if metadata.get('type', None) is None or (spec_type is not None and metadata["type"] != spec_type):
+            continue
+        if details['normalized_score'] < threshold / 100:
+            break
+        results_out.append(metadata)
+    
+    if len(results_out) > 0:
+        return KeywordResponse(
+            results=results_out,
+            search_time=time.time() - start_time
+        )
+    else:
+        raise HTTPException(status_code=404, detail="Specifications not found")

indexed_docs.json

@@ -1,6 +0,0 @@
-{
-    "docs": {
-        "SET(25)000002": "https://docbox.etsi.org/SET/SET/05-CONTRIBUTIONS/2025/SET(25)000002_Draft_report_of_SET__116.docx"
-    },
-    "last_indexed_date": "03/06/2025-14:20:45"
-}

schemas.py

@@ -0,0 +1,39 @@
+from pydantic import BaseModel
+from typing import *
+
+class DocRequest(BaseModel):
+    doc_id: str
+
+class DocResponse(BaseModel):
+    doc_id: str
+    url: str
+    release: Optional[str] = None
+    scope: Optional[str] = None
+    search_time: float
+    
+class BatchDocRequest(BaseModel):
+    doc_ids: List[str]
+    release: Optional[int] = None
+
+class BatchDocResponse(BaseModel):
+    results: Dict[str, str]
+    missing: List[str]
+    search_time: float
+
+class BM25KeywordRequest(BaseModel):
+    keywords: Optional[str] = ""
+    threshold: Optional[int] = 60
+    release: Optional[str] = None
+    spec_type: Optional[Literal["TS", "TR"]] = None
+
+class KeywordRequest(BaseModel):
+    keywords: Optional[str] = ""
+    search_mode: Literal["quick", "deep"]
+    case_sensitive: Optional[bool] = False
+    release: Optional[str] = None
+    spec_type: Optional[Literal["TS", "TR"]] = None
+    mode: Optional[Literal["and", "or"]] = "and"
+
+class KeywordResponse(BaseModel):
+    results: List[Dict[str, Any]]
+    search_time: float

static/script.js

@@ -4,19 +4,35 @@ const dynamicTitle = document.getElementById("dynamicTitle");
 
 const singleModeBtn = document.getElementById('single-mode-btn');
 const batchModeBtn = document.getElementById('batch-mode-btn');
-// const keywordModeBtn = document.getElementById("keyword-mode-btn");
+const keywordModeBtn = document.getElementById("keyword-mode-btn");
+const expModeBtn = document.getElementById("exp-mode-btn");
 
-const singleInput = document.querySelector('.single-input');
-const batchInput = document.querySelector('.batch-input');
-// const keywordSearchInput = document.querySelector(".keyword-input");
+const singleInputField = document.querySelector('.single-input');
+const batchInputField = document.querySelector('.batch-input');
+const keywordInputField = document.querySelector(".keyword-input");
+const expKeywordInputField = document.querySelector(".experimental-input");
 
 const docIdInput = document.getElementById('doc-id');
 const batchIdsInput = document.getElementById('batch-ids');
-// const keywordInput = document.getElementById("keywords");
+const keywordInput = document.getElementById("keywords");
+const expKeywordInput = document.getElementById("exp-keywords")
+const thresholdInput = document.getElementById("threshold");
+
+const releaseFilter = document.querySelector("input[name=release]")
+const modeFilter = document.querySelector("select[name=mode]")
+const specTypeFilter = document.querySelector("select[name=spec_type]")
+const workingGroupFilter = document.querySelector("select[name=working_group]")
+const caseSensitiveFilter = document.querySelector("input[name=case_sensitive]")
+const searchMode = document.querySelector("select[name=search_mode]")
+
+const releaseFilter2 = document.querySelector("input[name=release2]")
+const specTypeFilter2 = document.querySelector("select[name=spec_type2]")
+const workingGroupFilter2 = document.querySelector("select[name=working_group2]")
 
 const searchBtn = document.getElementById('search-btn');
 const batchSearchBtn = document.getElementById('batch-search-btn');
-// const keywordSearchBtn = document.getElementById("keyword-search-btn");
+const keywordSearchBtn = document.getElementById("keyword-search-btn");
+const expKeywordSearchBtn = document.getElementById("exp-search-btn");
 
 const loader = document.getElementById('loader');
 const resultsContainer = document.getElementById('results-container');
@@ -27,75 +43,167 @@ const errorMessage = document.getElementById('error-message');
 // Search mode toggle
 singleModeBtn.addEventListener('click', () => {
     dynamicTitle.textContent = "Find ETSI Documents";
-    dynamicDesc.textContent = "Enter a SET/SCP/TS document ID to locate the document in the ETSI DocBox server.";
+    dynamicDesc.textContent = "Enter a SET/SCP/TS document ID to locate the document in the ETSI's server.";
 
     singleModeBtn.classList.add('active');
-    // keywordModeBtn.classList.remove("active");
+    keywordModeBtn.classList.remove("active");
     batchModeBtn.classList.remove('active');
+    expModeBtn.classList.remove('active');
 
-    singleInput.style.display = 'block';
-    batchInput.style.display = 'none';
-    // keywordSearchInput.style.display = "none";
+    singleInputField.style.display = 'block';
+    batchInputField.style.display = 'none';
+    keywordInputField.style.display = "none";
+    expKeywordInputField.style.display = "none";
 });
 
 batchModeBtn.addEventListener('click', () => {
     dynamicTitle.textContent = "Find multiple ETSI Documents";
-    dynamicDesc.textContent = "Enter a list of SET/SCP/TS document ID to locate all specified documents in the ETSI DocBox server.";
+    dynamicDesc.textContent = "Enter a list of SET/SCP/TS document ID to locate all of the specified documents in the ETSI's server.";
 
     batchModeBtn.classList.add('active');
-    //keywordModeBtn.classList.remove("active");
+    keywordModeBtn.classList.remove("active");
     singleModeBtn.classList.remove('active');
+    expModeBtn.classList.remove('active');
 
-    batchInput.style.display = 'block';
-    //keywordSearchInput.style.display = "none";
-    singleInput.style.display = 'none';
+    batchInputField.style.display = 'block';
+    keywordInputField.style.display = "none";
+    singleInputField.style.display = 'none';
+    expKeywordInputField.style.display = "none";
 });
 
-// keywordModeBtn.addEventListener('click', () => {
-//     dynamicTitle.textContent = "Search 3GPP specifications";
-//     dynamicDesc.textContent = "With keywords and filters, find all of 3GPP's specifications that matches your needs (with keywords, specification number, release or even working group (C1, S5, SP, CP: always the first letter of the group followed by the workgroup number)";
-
-//     keywordModeBtn.classList.add("active");
-//     singleModeBtn.classList.remove('active');
-//     batchModeBtn.classList.remove("active");
-
-//     singleInput.style.display = "none";
-//     batchInput.style.display = "none";
-//     keywordSearchInput.style.display = "block";
-// })
-
-// keywordSearchBtn.addEventListener("click", async ()=>{
-//     const keywords = keywordInput.value.trim();
-//     if (!keywords) {
-//         showError("Please enter at least one keyword");
-//         return;
-//     }
-
-//     showLoader();
-//     hideError();
-
-//     try{
-//         const response = await fetch("/search-spec", {
-//             method: "POST",
-//             headers: {
-//                 "Content-Type": "application/json"
-//             },
-//             body: JSON.stringify({ keywords })
-//         });
-
-//         const data = await response.json();
-//         if (response.ok){
-//             displayKeywordResults(data);
-//         } else {
-//             showError('Error processing batch request');
-//         }
-//     } catch (error) {
-//         showError('Error connecting to the server. Please check if the API is running.');
-//         console.error('Error:', error);
-//     } finally {
-//         hideLoader();
-//     }
-// })
+keywordModeBtn.addEventListener('click', () => {
+    dynamicTitle.textContent = "Search ETSI specifications";
+    dynamicDesc.textContent = "With keywords and filters, find all of ETSI's specifications that matches your needs (with keywords, specification number, release";
+
+    keywordModeBtn.classList.add("active");
+    singleModeBtn.classList.remove('active');
+    batchModeBtn.classList.remove("active");
+    expModeBtn.classList.remove('active');
+
+    singleInputField.style.display = "none";
+    batchInputField.style.display = "none";
+    expKeywordInputField.style.display = "none";
+    keywordInputField.style.display = "block";
+})
+
+expModeBtn.addEventListener('click', () => {
+    dynamicTitle.textContent = "[EXPERIMENTAL] Search 3GPP specifications";
+    dynamicDesc.textContent = "With keywords and filters, find all of ETSI's specifications that matches your needs (with keywords, specification number, release";
+
+    keywordModeBtn.classList.remove("active");
+    singleModeBtn.classList.remove('active');
+    batchModeBtn.classList.remove("active");
+    expModeBtn.classList.add('active');
+
+    singleInputField.style.display = "none";
+    batchInputField.style.display = "none";
+    expKeywordInputField.style.display = "block";
+    keywordInputField.style.display = "none";
+})
+
+document.getElementById('toggleFilters').onclick = function() {
+  var target = document.getElementById('filtersForm');
+  target.style.display = (target.style.display === 'none' || target.style.display === '') ? 'flex' : 'none';
+};
+
+document.getElementById('toggleFilters2').onclick = function() {
+  var target = document.getElementById('filtersForm2');
+  target.style.display = (target.style.display === 'none' || target.style.display === '') ? 'flex' : 'none';
+};
+
+expKeywordSearchBtn.addEventListener("click", async ()=>{
+    let keywords = expKeywordInput.value.trim();
+    let release = releaseFilter2.value.trim();
+    let specType = specTypeFilter2.value.trim();
+    let threshold = thresholdInput.value.trim() != '' ? thresholdInput.value.trim() : 60
+
+    if (!keywords){
+        showError("Please enter at least one keyword");
+        return;
+    }
+
+    showLoader();
+    hideError();
+
+    try{
+        let body = {
+            keywords,
+            threshold
+        };
+        if (release != ""){body["release"] = release}
+        if (specType != ""){body["spec_type"] = specType}
+        const response = await fetch("/search-spec/experimental", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify(body)
+        });
+
+        const data = await response.json();
+        if (response.ok){
+            displayKeywordResults(data, "");
+        } else if (response.status == 404) {
+            showError('No specification has been found');
+        } else {
+            showError(`Error processing keyword request: ${data.detail}`)
+        }
+    } catch (error) {
+        showError('Error connecting to the server. Please check if the API is running.');
+        console.error('Error:', error);
+    } finally {
+        hideLoader();
+    }
+})
+
+keywordSearchBtn.addEventListener("click", async ()=>{
+    let keywords = keywordInput.value.trim();
+    let release = releaseFilter.value;
+    let specType = specTypeFilter.value;
+    let search = searchMode.value;
+    let checked = caseSensitiveFilter.checked;
+    let mode = modeFilter.value;
+
+    if (!keywords && searchMode == "deep") {
+        showError("Please enter at least one keyword in deep search mode");
+        return;
+    }
+
+    showLoader();
+    hideError();
+
+    try{
+        let body = {
+            keywords,
+            "search_mode": search,
+            "case_sensitive": checked,
+            "mode": mode
+        };
+        if (release != ""){body.release = release}
+        if (specType != ""){body["spec_type"] = specType}
+        const response = await fetch("/search-spec", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify(body)
+        });
+
+        const data = await response.json();
+        if (response.ok){
+            displayKeywordResults(data, search);
+        } else if (response.status == 404) {
+            showError('No specification has been found');
+        } else {
+            showError(`Error processing keyword request: ${data.detail}`)
+        }
+    } catch (error) {
+        showError('Error connecting to the server. Please check if the API is running.');
+        console.error('Error:', error);
+    } finally {
+        hideLoader();
+    }
+})
 
 
 // Single document search
@@ -115,7 +223,7 @@ searchBtn.addEventListener('click', async () => {
             headers: {
                 'Content-Type': 'application/json'
             },
-            body: JSON.stringify({ doc_id: docId })
+            body: JSON.stringify({ doc_id: docId, release: null })
         });
 
         const data = await response.json();
@@ -166,8 +274,10 @@ batchSearchBtn.addEventListener('click', async () => {
         
         if (response.ok) {
             displayBatchResults(data);
+        } else if (response.status == 404) {
+            showError('No document has been found');
         } else {
-            showError('Error processing batch request');
+            showError('Error processing batch request')
         }
     } catch (error) {
         showError('Error connecting to the server. Please check if the API is running.');
@@ -219,32 +329,96 @@ function displaySingleNotFound(docId, message) {
     resultsContainer.style.display = 'block';
 }
 
-// function displayKeywordResults(data) {
-//     resultsList.innerHTML = '';
+function displayKeywordResults(data, mode) {
+    resultsList.innerHTML = '';
     
-//     data.results.forEach(spec => {
-//         const resultItem = document.createElement("div");
-//         resultItem.className = "result-item"
-//         resultItem.innerHTML = `
-//             <div class="result-header">
-//                 <div class="result-id">${spec.id}</div>
-//                 <div class="result-status status-found">Found</div>
-//             </div>
-//             <div class="result-url">
-//                 <p>Title: ${spec.title}</p>
-//                 <p>Type: ${spec.type}</p>
-//                 <p>Release: ${spec.release}</p>
-//                 <p>Version: ${spec.version}</p>
-//                 <p>WG: ${spec.working_group}</p>
-//                 <p>URL: <a target="_blank" href="${spec.url}">${spec.url}</a></p>
-//                 <p>Scope: ${spec.scope}</p>
-//             </div>
-//         `;
-//         resultsList.appendChild(resultItem);
-//     });
-//     resultsStats.textContent = `Found in ${data.search_time.toFixed(2)} seconds`
-//     resultsContainer.style.display = 'block';
-// }
+    data.results.forEach(spec => {
+        const resultItem = document.createElement("div");
+        resultItem.className = "result-item";
+        
+        resultItem.innerHTML = `
+            <div class="result-header">
+                <div class="result-id">${spec.id}</div>
+                <div class="result-status status-found">Found</div>
+            </div>
+            <div class="result-url">
+                <p>Title: ${spec.title}</p>
+                <p>Type: ${spec.type}</p>
+                <p>Version: ${spec.version}</p>
+                <p>URL: <a target="_blank" href="${spec.url}">${spec.url}</a></p>
+                <p>Scope: ${spec.scope}</p>
+            </div>
+        `;
+
+        if(mode == "deep"){
+            resultItem.innerHTML += `
+                <div class="result-actions">
+                    <button class="get-section-btn btn" data-spec-id="${spec.id}">Get section</button>
+                </div>
+            `
+        }
+        
+        // Ajouter le bouton au DOM
+        resultsList.appendChild(resultItem);
+        
+        // Récupérer le bouton nouvellement créé
+        if(mode == "deep"){
+            const button1 = resultItem.querySelector('.get-section-btn');
+            button1._sections = spec.contains;
+        }
+    });
+
+    document.querySelectorAll('.get-section-btn').forEach(button => {
+      button.addEventListener('click', function() {
+        let specId = this.getAttribute("data-spec-id");
+        let sections = this._sections;
+        openSectionPopup(specId, sections);
+      });
+    });
+
+    resultsStats.textContent = `Found ${data.results.length} in ${data.search_time.toFixed(2)} seconds`
+    resultsContainer.style.display = 'block';
+}
+
+function openSectionPopup(specId, sections) {
+    const newTab = window.open('', '_blank');
+    let htmlContent = 
+    `
+        <!DOCTYPE html>
+        <html lang="fr">
+        <head>
+          <meta charset="UTF-8">
+          <title>Sections of specification number ${specId}</title>
+          <link rel="stylesheet" href="/static/style.css">
+          <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap">
+        </head>
+        <body>
+            <div class="popup-header">
+                <h2 id="popupTitle">Sections of specification number ${specId}</h2>
+            </div>
+            <div id="popupTextareas" class="popup-textareas">
+    `;
+
+    Object.entries(sections).forEach(([sectionTitle, content], index) => {
+        htmlContent += 
+        `
+            <div class="textarea-container">
+                <h2>${sectionTitle}</h2>
+                <p>${content.replace(/\n/g, '<br>')}</p>
+            </div>
+        `;
+    });
+
+    htmlContent += `
+        </div>
+        </body>
+        </html>
+      `;
+
+    newTab.document.open();
+    newTab.document.write(htmlContent);
+    newTab.document.close()
+}
 
 // Display batch results
 function displayBatchResults(data) {
@@ -298,6 +472,8 @@ function hideLoader() {
 
 // Show error message
 function showError(message) {
+    resultsList.innerHTML = "";
+    resultsStats.textContent = `Found 0 documents`;
     errorMessage.textContent = message;
     errorMessage.style.display = 'block';
 }
@@ -314,8 +490,14 @@ docIdInput.addEventListener('keypress', (e) => {
     }
 });
 
-// keywordInput.addEventListener('keypress', (event)=>{
-//     if (event.key === "Enter"){
-//         keywordSearchBtn.click();
-//     }
-// })
+keywordInput.addEventListener('keypress', (event)=>{
+    if (event.key === "Enter"){
+        keywordSearchBtn.click();
+    }
+})
+
+expKeywordInput.addEventListener('keypress', (event)=>{
+    if (event.key === "Enter"){
+        expKeywordSearchBtn.click();
+    }
+})

static/style.css

@@ -10,27 +10,192 @@
     --shadow-color: rgba(0, 0, 0, 0.7);
 }
 
+/* Section Popup */
+.section-popup {
+  display: none;
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-color: rgba(24, 26, 32, 0.85); /* Plus foncé, légèrement opaque */
+  z-index: 1000;
+  overflow: auto;
+}
+.section-popup-content {
+  position: relative;
+  background-color: #23262f;
+  width: 80%;
+  max-width: 800px;
+  margin: 50px auto;
+  padding: 30px;
+  border-radius: 8px;
+  box-shadow: 0 4px 15px var(--shadow-color);
+  animation: popupFadeIn 0.3s;
+}
+@keyframes popupFadeIn {
+  from { opacity: 0; transform: translateY(-20px);}
+  to { opacity: 1; transform: translateY(0);}
+}
+
+/* Popup header */
+.popup-header {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 20px;
+  padding-bottom: 15px;
+  border-bottom: 1px solid var(--border-color);
+}
+.popup-header h2 {
+  font-size: 22px;
+  font-weight: 500;
+  color: var(--text-color);
+}
+.close-popup {
+  font-size: 28px;
+  font-weight: bold;
+  color: var(--light-text);
+  cursor: pointer;
+  transition: color 0.3s;
+}
+.close-popup:hover {
+  color: var(--primary-color);
+}
+
+/* Popup Textareas */
+.popup-textareas {
+  display: flex;
+  flex-direction: column;
+  gap: 20px;
+  overflow-y: auto;
+  padding-right: 10px;
+}
+.textarea-container {
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+}
+.textarea-container p {
+  width: 100%;
+  padding: 12px 16px;
+  border: 1px solid #383b44;
+  border-radius: 4px;
+  font-size: 16px;
+  font-family: 'Roboto', sans-serif;
+  background-color: #181a20;
+  color: var(--text-color);
+  resize: vertical;
+  outline: none;
+}
+
+/* Boutons de copie */
+.copy-btn {
+  align-self: flex-end;
+  background-color: var(--secondary-color);
+  color: var(--primary-color);
+  border: 1px solid var(--border-color);
+  border-radius: 4px;
+  padding: 8px 16px;
+  font-size: 14px;
+  font-weight: 500;
+  cursor: pointer;
+  transition: background-color 0.3s;
+}
+.copy-btn:hover {
+  background-color: #23262f;
+}
+.copy-all-btn {
+  display: block;
+  width: 100%;
+  background-color: var(--primary-color);
+  color: #181a20;
+  border: none;
+  border-radius: 4px;
+  padding: 12px 24px;
+  font-size: 16px;
+  font-weight: 500;
+  cursor: pointer;
+  transition: background-color 0.3s;
+  margin-top: 20px;
+}
+.copy-all-btn:hover {
+  background-color: var(--accent-color);
+  color: #fff;
+}
+
+/* Filtres */
+.filter-tab-container {
+  margin-top: 20px;
+  margin-bottom: 15px;
+}
+.filter-toggle-btn {
+  background: #487bbd;
+  color: #fff;
+  border: none;
+  border-radius: 4px 4px 0 0;
+  padding: 8px 14px;
+  cursor: pointer;
+  font-size: 16px;
+  display: flex;
+  align-items: center;
+  gap: 5px;
+  transition: background 0.3s;
+  box-shadow: 0 2px 4px rgba(64, 122, 212, 0.1);
+}
+.filter-toggle-btn:hover {
+  background: #305070;
+}
+.filters-row {
+  display: flex;
+  gap: 12px;
+  align-items: center;
+  background: #181a20;
+  padding: 13px 14px;
+  border: 1px solid #23262f;
+  border-top: none;
+  border-radius: 0 0 4px 4px;
+  margin: 0;
+  margin-top: -1px;
+  box-shadow: 0 2px 4px rgba(44, 82, 130, 0.10);
+}
+.filter-input, .filter-select {
+  padding: 7px 10px;
+  border: 1px solid #23262f;
+  border-radius: 4px;
+  background: #23262f;
+  color: #e8eaed;
+  font-size: 15px;
+}
+.filter-checkbox-label {
+  display: flex;
+  align-items: center;
+  gap: 7px;
+  font-size: 15px;
+  color: #8ab4f8;
+  margin: 0;
+}
+.filter-checkbox {
+  accent-color: #8ab4f8;
+}
+
+/* Responsive et global */
 * {
     margin: 0;
     padding: 0;
     box-sizing: border-box;
 }
-
 body {
     font-family: 'Roboto', sans-serif;
     background-color: var(--secondary-color);
     color: var(--text-color);
     line-height: 1.6;
-    padding: 0;
-    margin: 0;
 }
-
 .container {
     max-width: 1200px;
     margin: 0 auto;
     padding: 20px;
 }
-
 header {
     background-color: #23262f;
     box-shadow: 0 2px 10px var(--shadow-color);
@@ -39,29 +204,33 @@ header {
     top: 0;
     z-index: 100;
 }
-
+.input-field input.filter-input{
+    flex: none;
+    padding: 7px 10px;
+    border: 1px solid #23262f;
+    border-radius: 4px;
+    background: #23262f;
+    color: #e8eaed;
+    font-size: 15px;
+}
 .header-content {
     display: flex;
     align-items: center;
     justify-content: space-between;
 }
-
 .logo {
     display: flex;
     align-items: center;
 }
-
 .logo img {
     height: 40px;
     margin-right: 10px;
 }
-
 .logo h1 {
     font-size: 24px;
     font-weight: 500;
     color: var(--primary-color);
 }
-
 .search-container {
     background-color: #23262f;
     border-radius: 8px;
@@ -69,46 +238,38 @@ header {
     padding: 30px;
     margin-top: 30px;
 }
-
 .search-header {
     margin-bottom: 20px;
 }
-
 .search-header h2 {
     font-size: 22px;
     font-weight: 500;
     color: var(--text-color);
     margin-bottom: 10px;
 }
-
 .search-header p {
     color: var(--light-text);
     font-size: 16px;
 }
-
 .search-form {
     display: flex;
     flex-direction: column;
     gap: 20px;
 }
-
 .input-group {
     display: flex;
     flex-direction: column;
     gap: 8px;
 }
-
 .input-group label {
     font-size: 14px;
     font-weight: 500;
     color: var(--light-text);
 }
-
 .input-field {
     display: flex;
     gap: 10px;
 }
-
 .input-field input {
     flex: 1;
     padding: 12px 16px;
@@ -120,12 +281,10 @@ header {
     outline: none;
     transition: border-color 0.3s;
 }
-
 .input-field input:focus {
     border-color: var(--primary-color);
     box-shadow: 0 0 0 2px rgba(138, 180, 248, 0.15);
 }
-
 .btn {
     background-color: var(--primary-color);
     color: #181a20;
@@ -137,18 +296,15 @@ header {
     cursor: pointer;
     transition: background-color 0.3s;
 }
-
 .btn:hover {
     background-color: var(--accent-color);
     color: #fff;
 }
-
 .search-mode {
     display: flex;
     gap: 20px;
     margin-bottom: 20px;
 }
-
 .search-mode button {
     background: none;
     border: none;
@@ -160,20 +316,13 @@ header {
     border-bottom: 2px solid transparent;
     transition: all 0.3s;
 }
-
 .search-mode button.active {
     color: var(--primary-color);
     border-bottom: 2px solid var(--primary-color);
 }
-
-.batch-input {
-    display: none;
-}
-
-.keyword-input {
+.batch-input, .keyword-input, .experimental-input {
     display: none;
 }
-
 .batch-input textarea {
     width: 100%;
     height: 120px;
@@ -187,18 +336,15 @@ header {
     background: #181a20;
     color: var(--text-color);
 }
-
 .batch-input textarea:focus {
     border-color: var(--primary-color);
     box-shadow: 0 0 0 2px rgba(138, 180, 248, 0.15);
 }
-
 .batch-input .hint {
     font-size: 14px;
     color: var(--light-text);
     margin-top: 8px;
 }
-
 .results-container {
     margin-top: 30px;
     background-color: #23262f;
@@ -207,7 +353,6 @@ header {
     padding: 30px;
     display: none;
 }
-
 .results-header {
     display: flex;
     justify-content: space-between;
@@ -216,23 +361,19 @@ header {
     padding-bottom: 15px;
     border-bottom: 1px solid var(--border-color);
 }
-
 .results-header h2 {
     font-size: 22px;
     font-weight: 500;
 }
-
 .results-stats {
     color: var(--light-text);
     font-size: 14px;
 }
-
 .results-list {
     display: flex;
     flex-direction: column;
     gap: 15px;
 }
-
 .result-item {
     padding: 15px;
     border: 1px solid var(--border-color);
@@ -240,61 +381,50 @@ header {
     background: #181a20;
     transition: box-shadow 0.3s;
 }
-
 .result-item:hover {
     box-shadow: 0 4px 8px var(--shadow-color);
 }
-
 .result-header {
     display: flex;
     justify-content: space-between;
     align-items: center;
     margin-bottom: 10px;
 }
-
 .result-id {
     font-weight: 500;
     font-size: 18px;
     color: var(--primary-color);
 }
-
 .result-status {
     font-size: 14px;
     padding: 4px 12px;
     border-radius: 12px;
 }
-
 .status-found {
     background-color: rgba(52, 232, 158, 0.1);
     color: var(--success-color);
 }
-
 .status-not-found {
     background-color: rgba(255, 109, 109, 0.1);
     color: var(--error-color);
 }
-
 .result-url {
     word-break: break-all;
     margin-top: 10px;
 }
-
 .result-url a {
     color: var(--primary-color);
     text-decoration: none;
     transition: color 0.3s;
 }
-
 .result-url a:hover {
     text-decoration: underline;
 }
-
 .loader {
     display: none;
     text-align: center;
     padding: 20px;
 }
-
 .spinner {
     border: 4px solid rgba(255, 255, 255, 0.1);
     border-radius: 50%;
@@ -304,12 +434,10 @@ header {
     animation: spin 1s linear infinite;
     margin: 0 auto;
 }
-
 @keyframes spin {
-    0% { transform: rotate(0deg); }
-    100% { transform: rotate(360deg); }
+    0% { transform: rotate(0deg);}
+    100% { transform: rotate(360deg);}
 }
-
 .error-message {
     background-color: rgba(255, 109, 109, 0.1);
     color: var(--error-color);
@@ -318,7 +446,6 @@ header {
     margin-top: 20px;
     display: none;
 }
-
 footer {
     text-align: center;
     padding: 30px 0;
@@ -326,17 +453,14 @@ footer {
     color: var(--light-text);
     font-size: 14px;
 }
-
 @media (max-width: 768px) {
     .header-content {
         flex-direction: column;
         gap: 15px;
     }
-
     .input-field {
         flex-direction: column;
     }
-
     .search-mode {
         overflow-x: auto;
         padding-bottom: 5px;

templates/index.html

@@ -26,7 +26,8 @@
             <div class="search-mode">
                 <button id="single-mode-btn" class="active">Single Document</button>
                 <button id="batch-mode-btn">Batch Search</button>
-                <!--<button id="keyword-mode-btn">Keyword Search</button> -->
+                <button id="keyword-mode-btn">Keyword Search</button>
+                <button id="exp-mode-btn">Experimental Search (Using BM25)</button>
             </div>
 
             <div class="search-form">
@@ -44,14 +45,71 @@
                     <div class="hint">Enter one document ID per line</div>
                     <button id="batch-search-btn" class="btn" style="margin-top: 10px;">Search All</button>
                 </div>
-                <!--
+                
                 <div class="input-group keyword-input">
-                    <label for="keywords">Keywords</label>
                     <div class="input-field">
-                        <input type="text" id="keywords" placeholder="Enter your keywords separated by space">
+                        <select name="search_mode" class="filter-select" id="search-mode">
+                            <option value="quick">Quick Search (only title & scope)</option>
+                            <option value="deep">Deep Search</option>
+                        </select>
+                        <input type="text" id="keywords" placeholder="Enter your keywords separated by comma">
                         <button id="keyword-search-btn" class="btn">Search</button>
                     </div>
-                </div> -->
+                    <div class="filter-tab-container">
+                        <button type="button" id="toggleFilters" class="filter-toggle-btn">
+                            <span>Filters</span>
+                            <svg width="16" height="16" style="vertical-align:middle" fill="currentColor">
+                                <path d="M4 7l4 4 4-4"></path>
+                            </svg>
+                        </button>
+                        <form id="filtersForm" class="filters-row" style="display:none;">
+                            <input type="number" min="0" max="21" name="release" placeholder="Release"
+                                class="filter-input">
+
+                            <select name="mode" class="filter-select">
+                                <option value="and">AND</option>
+                                <option value="or">OR</option>
+                            </select>
+
+                            <select name="spec_type" class="filter-select">
+                                <option value="">All types</option>
+                                <option value="TR">Technical Report (TR)</option>
+                                <option value="TS">Technical Specification (TS)</option>
+                            </select>
+
+                            <label class="filter-checkbox-label">
+                                <input type="checkbox" name="case_sensitive" class="filter-checkbox" />
+                                Case sensitive
+                            </label>
+                        </form>
+                    </div>
+                </div>
+
+                <div class="input-group experimental-input">
+                    <div class="input-field">
+                        <input type="number" name="threshold" id="threshold" class="filter-input" min="30" max="100" placeholder="Min %">
+                        <input type="text" id="exp-keywords" placeholder="Enter keywords separated by spaces">
+                        <button id="exp-search-btn" class="btn">Search</button>
+                    </div>
+                    <div class="filter-tab-container">
+                        <button type="button" id="toggleFilters2" class="filter-toggle-btn">
+                            <span>Filters</span>
+                            <svg width="16" height="16" style="vertical-align:middle" fill="currentColor">
+                                <path d="M4 7l4 4 4-4"></path>
+                            </svg>
+                        </button>
+                        <form id="filtersForm2" class="filters-row" style="display:none;">
+                            <input type="number" min="0" max="21" name="release2" placeholder="Release"
+                                class="filter-input">
+
+                            <select name="spec_type2" class="filter-select">
+                                <option value="">All types</option>
+                                <option value="TR">Technical Report (TR)</option>
+                                <option value="TS">Technical Specification (TS)</option>
+                            </select>
+                        </form>
+                    </div>
+                </div>
             </div>
 
             <div class="error-message" id="error-message"></div>