update

app.py

@@ -4,7 +4,9 @@ import torch
 
 # Model loading and setup
 model_name = "jhu-clsp/FollowIR-7B"
-model = AutoModelForCausalLM.from_pretrained(model_name).cuda()
+model = AutoModelForCausalLM.from_pretrained(model_name)
+if torch.cuda.is_available():
+    model = model.cuda()
 tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
 tokenizer.pad_token = tokenizer.eos_token
 tokenizer.padding_side = "left"
@@ -29,8 +31,9 @@ def check_relevance(query, instruction, passage):
         pad_to_multiple_of=None,
     )
 
-    for key in tokens:
-        tokens[key] = tokens[key].cuda()
+    if torch.cuda.is_available():
+        for key in tokens:
+            tokens[key] = tokens[key].cuda()
 
     batch_scores = model(**tokens).logits[:, -1, :]
     true_vector = batch_scores[:, token_true_id]

requirements.txt

@@ -1,3 +1,4 @@
 gradio
+--extra-index-url https://download.pytorch.org/whl/cu117
 torch
 transformers