Update app.py

app.py

@@ -23,22 +23,7 @@ transcriber = pipeline(
     device=0 if torch.cuda.is_available() else "cpu",
 )
 
-# LLaMA Model Optimization
-LLAMA = "meta-llama/Llama-3.2-3B-Instruct"
-llama_quant_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_compute_dtype=torch.bfloat16,
-    bnb_4bit_quant_type="nf4"
-)
 
-tokenizer = AutoTokenizer.from_pretrained(LLAMA)
-tokenizer.pad_token = tokenizer.eos_token
-model = AutoModelForCausalLM.from_pretrained(
-    LLAMA,
-    torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
-    device_map="auto"
-)
 
 # Function to Transcribe & Generate Minutes
 def process_audio(audio_file):
@@ -49,6 +34,22 @@ def process_audio(audio_file):
     transcript = transcriber(audio_file)["text"]
     del transcriber 
     del processor
+    # LLaMA Model Optimization
+    LLAMA = "meta-llama/Llama-3.2-3B-Instruct"
+    llama_quant_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_use_double_quant=True,
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        bnb_4bit_quant_type="nf4"
+    )
+    
+    tokenizer = AutoTokenizer.from_pretrained(LLAMA)
+    tokenizer.pad_token = tokenizer.eos_token
+    model = AutoModelForCausalLM.from_pretrained(
+        LLAMA,
+        torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+        device_map="auto"
+    )
     # Generate meeting minutes
     system_message = "You are an assistant that produces minutes of meetings from transcripts, with summary, key discussion points, takeaways and action items with owners, in markdown."
     user_prompt = f"Below is an extract transcript of a Denver council meeting. Please write minutes in markdown, including a summary with attendees, location and date; discussion points; takeaways; and action items with owners.\n{transcript}"