Update app.py

app.py

@@ -6,8 +6,6 @@ import gradio as gr
 import os
 import logging
 from unsloth import FastLanguageModel
-
-# Set up logging
 logging.basicConfig(
     level=logging.DEBUG,  # Set the logging level to DEBUG to capture all messages
     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
@@ -16,9 +14,7 @@ logging.basicConfig(
     ]
 )
 logger = logging.getLogger(__name__)
-
 READ_HF = os.environ["read_hf"]
-
 alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
 
 ### Instruction:
@@ -29,7 +25,6 @@ alpaca_prompt = """Below is an instruction that describes a task, paired with an
 
 ### Response:
 {}"""
-
 string = '''
 You are an AI assistant tasked with managing inventory based on user instructions. You must meticulously analyze each user request to determine the appropriate action and execute it with the correct parameters.
 
@@ -74,14 +69,11 @@ You are an AI assistant tasked with managing inventory based on user instruction
 
 Category List : ["Dairy & Eggs", "Beverages & Snacks", "Cleaning & Hygiene", "Grains & Staples", "Personal Care", "Other"]
 '''
-
 @spaces.GPU()
-num_elements = (5 * 1024 * 1024) // 4
-# Create a tensor with the calculated number of elements
-tensor = torch.randn(num_elements, dtype=torch.float32)
-# Move the tensor to the GPU
-tensor_gpu = tensor.to('cuda')
 def chunk_it(inventory_list, user_input_text):
+    num_elements = (5 * 1024 * 1024) // 4
+    tensor = torch.randn(num_elements, dtype=torch.float32)
+    tensor_gpu = tensor.to('cuda')
     logger.info("Loading model and tokenizer...")
     try:
         model, tokenizer = FastLanguageModel.from_pretrained(
@@ -95,7 +87,6 @@ def chunk_it(inventory_list, user_input_text):
     except Exception as e:
         logger.error(f"Failed to load model and tokenizer: {e}")
         raise
-
     logger.info("Enabling native 2x faster inference...")
     try:
         FastLanguageModel.for_inference(model)
@@ -103,21 +94,18 @@ def chunk_it(inventory_list, user_input_text):
     except Exception as e:
         logger.error(f"Failed to enable native inference: {e}")
         raise
-
     formatted_prompt = alpaca_prompt.format(
         string + inventory_list,  # instruction
         user_input_text,  # input
         "",  # output - leave this blank for generation!
     )
     logger.debug(f"Formatted prompt: {formatted_prompt}")
-
     try:
         inputs = tokenizer([formatted_prompt], return_tensors="pt").to("cuda")
         logger.debug(f"Tokenized inputs: {inputs}")
     except Exception as e:
         logger.error(f"Failed to tokenize inputs: {e}")
         raise
-
     logger.info("Generating output...")
     try:
         outputs = model.generate(**inputs, max_new_tokens=216, use_cache=True)
@@ -125,22 +113,17 @@ def chunk_it(inventory_list, user_input_text):
     except Exception as e:
         logger.error(f"Failed to generate output: {e}")
         raise
-
     try:
         reply = tokenizer.batch_decode(outputs, skip_special_tokens=True)
         logger.debug(f"Decoded output: {reply}")
     except Exception as e:
         logger.error(f"Failed to decode output: {e}")
         raise
-
-    # Uncomment the following lines if further processing of the reply is needed
     # pattern = r"### Response:\n(.*?)<\|end_of_text\|>"
     # match = re.search(pattern, reply[0], re.DOTALL)
     # reply = match.group(1).strip()
-
     logger.debug(f"Final reply: {reply}")
     return reply
-
 # Interface for inputs
 iface = gr.Interface(
     fn=chunk_it,
@@ -151,10 +134,9 @@ iface = gr.Interface(
     outputs=gr.Textbox(label="output", lines=23),
     title="Testing",
 )
-
 logger.info("Launching Gradio interface...")
 try:
     iface.launch(inline=False)
     logger.info("Gradio interface launched.")
 except Exception as e:
-    logger.error(f"Failed to launch Gradio interface: {e}")
+    logger.error(f"Failed to launch Gradio interface: {e}")