Add feature: Keyword search (FINALLY) + debug (frontend only)

app.py

@@ -1,3 +1,6 @@
+from io import StringIO
+import numpy as np
+import pandas as pd
 import requests
 from bs4 import BeautifulSoup
 import json
@@ -12,7 +15,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional
 
 load_dotenv()
 
@@ -53,6 +56,18 @@ class BatchDocResponse(BaseModel):
     missing: List[str]
     search_time: float
 
+class KeywordRequest(BaseModel):
+    keywords: str
+    release: Optional[str] = None
+    version: Optional[str] = None
+    wg: Optional[str] = None
+    spec_type: Optional[Literal["TS", "TR"]] = None
+    mode: Optional[Literal["and", "or"]] = "and"
+
+class KeywordResponse(BaseModel):
+    results: List[Dict[str, str]]
+    search_time: float
+
 class TsgDocFinder:
     def __init__(self):
         self.main_ftp_url = "https://www.3gpp.org/ftp"
@@ -166,7 +181,7 @@ class SpecDocFinder:
     def __init__(self):
         self.chars = "0123456789abcdefghijklmnopqrstuvwxyz"
         
-    def search_document(self, doc_id, release):
+    def search_document(self, doc_id, release = None):
         series = doc_id.split(".")[0]
         while len(series) < 2:
             series = "0" + series
@@ -181,7 +196,6 @@ class SpecDocFinder:
             try:
                 item = items[-1].find("a")
             except Exception as e:
-                traceback.print_exc(e)
                 return f"Unable to find specification {doc_id} : {e}"
             a, b, c = [_ for _ in item.get_text().split("-")[1].replace(".zip", "")]
             version = f"{self.chars.index(a)}.{self.chars.index(b)}.{self.chars.index(c)}"
@@ -205,6 +219,67 @@ finder_spec = SpecDocFinder()
 async def main_menu():
     return FileResponse(os.path.join("templates", "index.html"))
 
+@app.post("/search-spec", response_model=KeywordResponse)
+def search_spec(request: KeywordRequest):
+    start_time = time.time()
+    response = requests.get(f'https://www.3gpp.org/dynareport?code=status-report.htm', headers={"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}, verify=False)
+    dfs = pd.read_html(StringIO(response.text), storage_options={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}, encoding="utf-8")
+
+    for x in range(len(dfs)):
+        dfs[x] = dfs[x].replace({np.nan: None})
+
+    columns_needed = [0, 1, 2, 3, 4]
+    extracted_dfs: List[pd.DataFrame] = [df.iloc[:, columns_needed] for df in dfs]
+    columns = [x.replace("\xa0", "_") for x in extracted_dfs[0].columns]
+
+    specifications = []
+
+    for df in extracted_dfs:
+        for index, row in df.iterrows():
+            doc = row.to_list()
+            doc_dict = dict(zip(columns, doc))
+            specifications.append(doc_dict)
+    
+    kws = [_.lower() for _ in request.keywords.split(" ")]
+    results = []
+
+    for spec in specifications:
+        if request.mode == "and":
+            if not all(kw in spec["title"].lower() for kw in kws):
+                continue
+        elif request.mode == "or":
+            if not any(kw in spec["title"].lower() for kw in kws):
+                continue
+        release = request.release
+        version = request.version
+        working_group = request.wg
+        spec_type = request.spec_type
+
+        if spec.get('vers', None) is None or (release is not None and spec["vers"].split(".")[0] != str(release)):
+            continue
+        if spec.get('vers', None) is None or (version is not None and spec["vers"] != version):
+            continue
+        if spec.get('WG', None) is None or (working_group is not None and spec["WG"] != working_group):
+            continue
+        if spec_type is not None and spec["type"] != spec_type:
+            continue
+
+        results.append({
+            "id": str(spec["spec_num"]),
+            "title": spec["title"],
+            "type": "Technical Specification" if spec["type"] == "TS" else "Technical Report",
+            "release": str(spec["vers"].split(".")[0]),
+            "version": str(spec["vers"]),
+            "working_group": spec["WG"]
+        })
+
+    if len(results) > 0:
+        return KeywordResponse(
+            results=results,
+            search_time=time.time() - start_time
+        )
+    else:
+        raise HTTPException(status_code=404, detail="Specification not found")
 @app.post("/find", response_model=DocResponse)
 def find_document(request: DocRequest):
     start_time = time.time()
@@ -214,7 +289,7 @@ def find_document(request: DocRequest):
     result = finder.search_document(request.doc_id, request.release)
     print(result)
 
-    if "not found" not in result and "Could not" not in result:
+    if "not found" not in result and "Could not" not in result and "Unable" not in result:
         return DocResponse(
             doc_id=request.doc_id,
             url=result,
@@ -231,7 +306,7 @@ def find_documents_batch(request: BatchDocRequest):
     missing = []
     
     for doc_id in request.doc_ids:
-        finder = finder_tsg if request.doc_id[0].isalpha() else finder_spec
+        finder = finder_tsg if doc_id[0].isalpha() else finder_spec
         result = finder.search_document(doc_id)
         if "not found" not in result and "Could not" not in result and "Unable" not in result:
             results[doc_id] = result

static/script.js

@@ -1,12 +1,22 @@
 // DOM elements
 const singleModeBtn = document.getElementById('single-mode-btn');
 const batchModeBtn = document.getElementById('batch-mode-btn');
+const keywordModeBtn = document.getElementById("keyword-mode-btn");
+const indexerModeBtn = document.getElementById("indexer-mode-btn")
+
 const singleInput = document.querySelector('.single-input');
 const batchInput = document.querySelector('.batch-input');
+const keywordSearchInput = document.querySelector(".keyword-input");
+const indexerButtons = document.querySelector(".indexer-buttons")
+
 const docIdInput = document.getElementById('doc-id');
 const batchIdsInput = document.getElementById('batch-ids');
+const keywordInput = document.getElementById("keywords");
+
 const searchBtn = document.getElementById('search-btn');
 const batchSearchBtn = document.getElementById('batch-search-btn');
+const keywordSearchBtn = document.getElementById("keyword-search-btn");
+
 const loader = document.getElementById('loader');
 const resultsContainer = document.getElementById('results-container');
 const resultsList = document.getElementById('results-list');
@@ -16,18 +26,86 @@ const errorMessage = document.getElementById('error-message');
 // Search mode toggle
 singleModeBtn.addEventListener('click', () => {
     singleModeBtn.classList.add('active');
+    keywordModeBtn.classList.remove("active");
     batchModeBtn.classList.remove('active');
+    indexerModeBtn.classList.remove("active");
+
     singleInput.style.display = 'block';
     batchInput.style.display = 'none';
+    keywordSearchInput.style.display = "none";
+    indexerButtons.style.display = "none";
 });
 
 batchModeBtn.addEventListener('click', () => {
     batchModeBtn.classList.add('active');
+    keywordModeBtn.classList.remove("active");
     singleModeBtn.classList.remove('active');
+    indexerModeBtn.classList.remove("active");
+
     batchInput.style.display = 'block';
+    keywordSearchInput.style.display = "none";
+    indexerButtons.style.display = "none";
     singleInput.style.display = 'none';
 });
 
+keywordModeBtn.addEventListener('click', () => {
+    keywordModeBtn.classList.add("active");
+    singleModeBtn.classList.remove('active');
+    batchModeBtn.classList.remove("active");
+    indexerModeBtn.classList.remove("active");
+
+    singleInput.style.display = "none";
+    batchInput.style.display = "none";
+    indexerButtons.style.display = "none";
+    keywordSearchInput.style.display = "block";
+})
+
+indexerModeBtn.addEventListener('click', ()=>{
+    keywordModeBtn.classList.remove("active");
+    singleModeBtn.classList.remove('active');
+    batchModeBtn.classList.remove("active");
+    indexerModeBtn.classList.add("active");
+
+    singleInput.style.display = "none";
+    batchInput.style.display = "none";
+    indexerButtons.style.display = "block";
+    keywordSearchInput.style.display = "none";
+})
+
+keywordSearchBtn.addEventListener("click", async ()=>{
+    const keywords = keywordInput.value.trim();
+    if (!keywords) {
+        showError("Please enter at least one keyword");
+        return;
+    }
+
+    showLoader();
+    hideError();
+
+    try{
+        const response = await fetch("/search-spec", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({ keywords })
+        });
+
+        const data = await response.json();
+        if (response.ok){
+            displayKeywordResults(data);
+        } else {
+            showError('Error processing batch request');
+        }
+    } catch (error) {
+        showError('Error connecting to the server. Please check if the API is running.');
+        console.error('Error:', error);
+    } finally {
+        hideLoader();
+    }
+})
+
+
 // Single document search
 searchBtn.addEventListener('click', async () => {
     const docId = docIdInput.value.trim();
@@ -147,6 +225,31 @@ function displaySingleNotFound(docId, message) {
     resultsContainer.style.display = 'block';
 }
 
+function displayKeywordResults(data) {
+    resultsList.innerHTML = '';
+    
+    data.results.forEach(spec => {
+        const resultItem = document.createElement("div");
+        resultItem.className = "result-item"
+        resultItem.innerHTML = `
+            <div class="result-header">
+                <div class="result-id">${spec.id}</div>
+                <div class="result-status status-found">Found</div>
+            </div>
+            <div class="result-url">
+                <p>Title: ${spec.title}</p>
+                <p>Type: ${spec.type}</p>
+                <p>Release: ${spec.release}</p>
+                <p>Version: ${spec.version}</p>
+                <p>WG: ${spec.working_group}</p>
+            </div>
+        `;
+        resultsList.appendChild(resultItem);
+    });
+    resultsStats.textContent = `Found in ${data.search_time.toFixed(2)} seconds`
+    resultsContainer.style.display = 'block';
+}
+
 // Display batch results
 function displayBatchResults(data) {
     resultsList.innerHTML = '';
@@ -213,4 +316,10 @@ docIdInput.addEventListener('keypress', (e) => {
     if (e.key === 'Enter') {
         searchBtn.click();
     }
-});
+});
+
+keywordInput.addEventListener('keypress', (event)=>{
+    if (event.key === "Enter"){
+        keywordSearchBtn.click();
+    }
+})

static/style.css

@@ -167,6 +167,14 @@ header {
     display: none;
 }
 
+.keyword-input {
+    display: none;
+}
+
+.indexer-buttons {
+    display: none;
+}
+
 .batch-input textarea {
     width: 100%;
     height: 120px;

templates/index.html

@@ -27,6 +27,8 @@
             <div class="search-mode">
                 <button id="single-mode-btn" class="active">Single Document</button>
                 <button id="batch-mode-btn">Batch Search</button>
+                <button id="keyword-mode-btn">Keyword Search</button>
+                <button id="indexer-mode-btn">Indexer Options</button>
             </div>
 
             <div class="search-form">
@@ -44,6 +46,22 @@
                     <div class="hint">Enter one document ID per line</div>
                     <button id="batch-search-btn" class="btn" style="margin-top: 10px;">Search All</button>
                 </div>
+
+                <div class="input-group keyword-input">
+                    <label for="keywords">Keywords</label>
+                    <div class="input-field">
+                        <input type="text" id="keywords" placeholder="Enter your keywords separated by space">
+                        <button id="keyword-search-btn" class="btn">Search</button>
+                    </div>
+                </div>
+
+                <div class="input-group indexer-buttons">
+                    <label for="indexerBtns">Actions</label>
+                    <div class="input-field">
+                        <button id="indexing-btn" class="btn">Index all files</button>
+                        <button id="testing-btn" class="btn">Test theory</button>
+                    </div>
+                </div>
             </div>
 
             <div class="error-message" id="error-message"></div>