Create app.py

app.py

@@ -0,0 +1,125 @@
+import gradio as gr
+import spaces
+import torch
+from datasets import load_dataset, concatenate_datasets
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.evaluation import InformationRetrievalEvaluator, SequentialEvaluator
+from sentence_transformers.util import cos_sim
+import pandas as pd
+
+# Check for GPU support and configure appropriately
+device = "cuda" if torch.cuda.is_available() else "cpu"
+zero = torch.Tensor([0]).to(device)  # Ensure that tensor operates on the appropriate device
+print(f"Device being used: {zero.device}")
+
+@spaces.GPU  # Enable GPU support for Gradio Spaces
+def evaluate_model(model_id):
+    # Load the model on the appropriate device
+    model = SentenceTransformer(model_id, device=device)
+    
+    # Define the evaluation parameters
+    matryoshka_dimensions = [768, 512, 256, 128, 64]
+    
+    # Prepare datasets
+    datasets_info = [
+        {
+            "name": "Arabic-financial",
+            "dataset_id": "Omartificial-Intelligence-Space/Arabic-finanical-rag-embedding-dataset",
+            "split": "train",
+            "size": 7000,
+            "columns": ("question", "context"),
+            "sample_size": 100
+        },
+        {
+            "name": "MLQA Arabic",
+            "dataset_id": "google/xtreme",
+            "split": "validation",
+            "subset": "MLQA.ar.ar",
+            "size": 500,
+            "columns": ("question", "context"),
+            "sample_size": 100
+        },
+        {
+            "name": "Custom",
+            "dataset_id": "philschmid/finanical-rag-embedding-dataset",
+            "split": "train",
+            "size": None,
+            "columns": ("question", "context"),
+            "sample_size": 100
+        }
+    ]
+    
+    evaluation_results = []
+    
+    for dataset_info in datasets_info:
+        # Load the dataset
+        dataset = load_dataset(dataset_info["dataset_id"], split=dataset_info["split"])
+        if dataset_info.get("subset"):
+            dataset = dataset.filter(lambda x: x["subset"] == dataset_info["subset"])
+
+        # Sample the dataset
+        dataset = dataset.select(range(min(dataset_info["sample_size"], len(dataset))))
+        
+        # Rename columns
+        dataset = dataset.rename_column(dataset_info["columns"][0], "anchor")
+        dataset = dataset.rename_column(dataset_info["columns"][1], "positive")
+        dataset = dataset.add_column("id", range(len(dataset)))
+        
+        # Prepare queries and corpus
+        corpus = dict(zip(dataset["id"], dataset["positive"]))
+        queries = dict(zip(dataset["id"], dataset["anchor"]))
+        
+        # Create a mapping of relevant documents (1 in our case) for each query
+        relevant_docs = {q_id: [q_id] for q_id in queries}
+        
+        matryoshka_evaluators = []
+        # Iterate over the different dimensions
+        for dim in matryoshka_dimensions:
+            ir_evaluator = InformationRetrievalEvaluator(
+                queries=queries,
+                corpus=corpus,
+                relevant_docs=relevant_docs,
+                name=f"dim_{dim}",
+                truncate_dim=dim,
+                score_functions={"cosine": cos_sim},
+            )
+            matryoshka_evaluators.append(ir_evaluator)
+        
+        # Create a sequential evaluator
+        evaluator = SequentialEvaluator(matryoshka_evaluators)
+        
+        # Evaluate the model
+        results = evaluator(model)
+        
+        # Collect results for each dataset
+        for dim in matryoshka_dimensions:
+            key = f"dim_{dim}_cosine_ndcg@10"
+            score = results[key] if key in results else None
+            evaluation_results.append({
+                "Dataset": dataset_info["name"],
+                "Dimension": dim,
+                "Score": score
+            })
+    
+    # Convert results to DataFrame for display
+    result_df = pd.DataFrame(evaluation_results)
+    return result_df
+
+# Define the Gradio interface
+def display_results(model_name):
+    # Evaluate model and return results
+    result_df = evaluate_model(model_name)
+    return result_df
+
+# Create the Gradio interface
+demo = gr.Interface(
+    fn=display_results, 
+    inputs=gr.Textbox(label="Model ID"), 
+    outputs="dataframe",
+    title="Model Evaluation with GPU Support",
+    description="Enter a Hugging Face Sentence Transformer model ID to evaluate it across datasets, leveraging GPU if available."
+)
+
+# Launch the Gradio app
+if __name__ == "__main__":
+    demo.launch(debug= True)