time profiling for prediction

interfaces/cap.py

@@ -2,6 +2,7 @@ import gradio as gr
 import spaces
 
 import os
+import time
 import torch
 import numpy as np
 import pandas as pd
@@ -34,6 +35,7 @@ domains = {
     "local government agenda": "localgovernment"
 }
 
+
 def check_huggingface_path(checkpoint_path: str):
     try:
         hf_api = HfApi(token=HF_TOKEN)
@@ -41,6 +43,7 @@ def check_huggingface_path(checkpoint_path: str):
         return True
     except:
         return False
+        
 
 def build_huggingface_path(language: str, domain: str):
     language = language.lower()
@@ -82,19 +85,22 @@ def build_huggingface_path(language: str, domain: str):
         return model_path
     else:
         return "poltextlab/xlm-roberta-large-pooled-cap"
+        
 
 def predict(text, model_id, tokenizer_id):
     device = torch.device("cpu")
 
-    # Load JIT-traced model
+    t0 = time.perf_counter()
     jit_model_path = f"/data/jit_models/{model_id.replace('/', '_')}.pt"
     model = torch.jit.load(jit_model_path).to(device)
     model.eval()
+    print(f"Model loading: {time.perf_counter() - t0:.3f}s")
 
-    # Load tokenizer (still regular HF)
+    t1 = time.perf_counter()
     tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
+    print(f"Tokenizer loading: {time.perf_counter() - t1:.3f}s")
 
-    # Tokenize input
+    t2 = time.perf_counter()
     inputs = tokenizer(
         text,
         max_length=256,
@@ -103,18 +109,24 @@ def predict(text, model_id, tokenizer_id):
         return_tensors="pt"
     )
     inputs = {k: v.to(device) for k, v in inputs.items()}
+    print(f"Tokenization: {time.perf_counter() - t2:.3f}s")
 
+    t3 = time.perf_counter()
     with torch.no_grad():
         output = model(inputs["input_ids"], inputs["attention_mask"])
-        print(output) # debug
         logits = output["logits"]
-        
+    print(f"Inference: {time.perf_counter() - t3:.3f}s")
+
     release_model(model, model_id)
 
+    t4 = time.perf_counter()
     probs = torch.nn.functional.softmax(logits, dim=1).cpu().numpy().flatten()
     output_pred = {f"[{CAP_NUM_DICT[i]}] {CAP_LABEL_NAMES[CAP_NUM_DICT[i]]}": probs[i] for i in np.argsort(probs)[::-1]}
     output_info = f'<p style="text-align: center; display: block">Prediction was made using the <a href="https://huggingface.co/{model_id}">{model_id}</a> model.</p>'
+    print(f"Post-processing: {time.perf_counter() - t4:.3f}s")
+
     return output_pred, output_info
+    
 
 def predict_cap(text, language, domain):
     print(domain) # debug statement
@@ -127,6 +139,7 @@ def predict_cap(text, language, domain):
         os.system('rm -r ~/.cache/huggingface/hub')
         
     return predict(text, model_id, tokenizer_id)
+    
 
 demo = gr.Interface(
     title="CAP Babel Demo",