Will it load?

app.py

@@ -0,0 +1,26 @@
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+# Load model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained("ISTA-DASLab/Meta-Llama-3.1-70B-AQLM-PV-2Bit-1x16")
+model = AutoModelForCausalLM.from_pretrained("ISTA-DASLab/Meta-Llama-3.1-70B-AQLM-PV-2Bit-1x16", torch_dtype=torch.float16)
+model = model.to('cuda')  # Move the model to GPU if available
+
+# Define a function for generating text from a prompt
+def generate_text(prompt):
+    inputs = tokenizer(prompt, return_tensors="pt").to('cuda')  # Tokenize input and move to GPU
+    outputs = model.generate(inputs.input_ids, max_length=100)  # Generate output text
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)  # Decode and return the text
+
+# Create Gradio Interface
+interface = gr.Interface(
+    fn=generate_text,  # Function that handles text generation
+    inputs="text",  # Input is a text box
+    outputs="text",  # Output is a text box
+    title="Meta-Llama-3.1-70B Text Generation",
+    description="Enter a prompt and generate text using Meta-Llama-3.1-70B.",
+)
+
+# Launch the Gradio app
+interface.launch()

requirements.txt

@@ -0,0 +1,3 @@
+transformers
+gradio 
+torch