app.py

import torch
import spaces
import re
from transformers import AutoTokenizer, AutoModelForCausalLM
import gradio as gr
import os
READ_HF = os.environ["read_hf"]
from unsloth import FastLanguageModel

alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

### Instruction:
{}

### Input:
{}

### Response:
{}"""

string = '''
You are an AI assistant tasked with managing inventory based on user instructions. You must meticulously analyze each user request to determine the appropriate action and execute it with the correct parameters.

**Here's your step-by-step thought process:**

1. **Identify the Function:** Carefully examine the user's input to determine the primary function they want to perform.  The available functions are:
    - `transaction`: Record a new item transaction.
    - `last n days transactions`: Retrieve transaction records within a specific timeframe.
    - `view inventory`: View inventory details for a specific category and risk level.
    - `generate report`: Generate an inventory report.

2. **Extract Parameters:** Once you've identified the function, carefully extract the necessary parameters from the user's input. Each function requires specific parameters:

    **`transaction`:**
        - `ItemName`: (string) **Must be an exact match from the provided Item List.**
        - `ItemQt`: (integer) The quantity of the item.
        - `Type`: (string) "sale", "purchase", or "return".

    **`last n days transactions`:**
        - `ItemCategory`: (string) **Must be from the provided Item Category List.**
        - `Duration`: (integer) Number of days (convert weeks, months, years to days).

    **`view inventory`:**
        - `ItemCategory`: (string) **Must be from the provided Item Category List.**
        - `RiskType`: (string) "overstock", "understock", or "Null" (if risk inventory is not asked), or "All" for both overstock and understock.

    **`generate report`:**
        - `ItemCategory`: (string) **Must be from the provided Item Category List.**
        - `Duration`: (integer) Number of days (convert weeks, months, years to days).
        - `ReportType`: (string):  "profit", "revenue", "inventory", or "Null" (for all reports).

3. **Validate Inputs:** Before proceeding, validate the extracted parameters:

    - **ItemName:**  Ensure the `ItemName` is an exact match from the provided Item List.
    - **ItemCategory:** Ensure the `ItemCategory` is from the provided Category List.
    - **Data Types:** Verify that all parameters are of the correct data type (string or integer).

4. **Output in JSON:** Always format your response as a JSON object.

**Additional Notes:**
- Pay close attention to the case and spelling of function names and parameters.

Category List : ["Dairy & Eggs", "Beverages & Snacks", "Cleaning & Hygiene", "Grains & Staples", "Personal Care", "Other"]
'''

@spaces.GPU()
def chunk_it(inventory_list, user_input_text):
    print("Loading model and tokenizer...")
    model, tokenizer = FastLanguageModel.from_pretrained(
        model_name = "VanguardAI/CoT_multi_llama_LoRA_4bit",
        max_seq_length = 2048,
        dtype = torch.bfloat16,
        load_in_4bit = True,
        token = READ_HF    
    )
    print("Model and tokenizer loaded.")

    print("Enabling native 2x faster inference...")
    FastLanguageModel.for_inference(model)
    print("Inference enabled.")

    formatted_prompt = alpaca_prompt.format(
        string + inventory_list,  # instruction
        user_input_text,  # input
        "",  # output - leave this blank for generation!
    )
    print("Formatted prompt: ", formatted_prompt)

    inputs = tokenizer([formatted_prompt], return_tensors="pt").to("cuda")
    print("Tokenized inputs: ", inputs)

    print("Generating output...")
    outputs = model.generate(**inputs, max_new_tokens=216, use_cache=True)
    print("Output generated.")

    reply = tokenizer.batch_decode(outputs, skip_special_tokens=True)
    print("Decoded output: ", reply)

    # Uncomment the following lines if further processing of the reply is needed
    # pattern = r"### Response:\n(.*?)<\|end_of_text\|>"
    # match = re.search(pattern, reply[0], re.DOTALL)
    # reply = match.group(1).strip()

    print("Final reply: ", reply)
    return reply

# Interface for inputs
iface = gr.Interface(
    fn=chunk_it,
    inputs=[
        gr.Textbox(label="user_input_text", lines=3),
        gr.Textbox(label="inventory_list", lines=5)
    ],
    outputs=gr.Textbox(label="output", lines=23),
    title="Testing",
)

print("Launching Gradio interface...")
iface.launch(inline=False)
print("Gradio interface launched.")