initial commit

app.py

@@ -0,0 +1,123 @@
+import json
+import streamlit as st
+from whisper_stt import transcribe_audio
+from intent_recognition import get_intent_and_amount
+from generate_response import generate_voice_response
+from test_NLU import get_slots
+
+DATABASE_PATH = "database.json"
+
+
+def load_database():
+    try:
+        with open(DATABASE_PATH, "r") as db_file:
+            return json.load(db_file)
+    except FileNotFoundError:
+        return {"requests": []}
+
+
+def save_to_database(data):
+    with open(DATABASE_PATH, "w") as db_file:
+        json.dump(data, db_file, indent=4)
+
+
+def handle_request(audio_file):
+    while True:
+       
+        text = transcribe_audio(audio_file)
+
+        
+        intent_data = get_intent_and_amount(text)
+        intent = intent_data.get("intent")
+        if intent:
+            intent = intent.replace("_", " ").title()
+        amount_data = intent_data.get("amount_data")
+        amount = amount_data.get("amount") if amount_data else None
+        currency = amount_data.get("currency") if amount_data else ""
+        slots = get_slots(text)
+        project_name = slots.get("project_name")
+        project_id = slots.get("project_id")
+        task_id = slots.get("task_id")
+        status = slots.get("status")
+
+        # Ensure mandatory fields are present
+        if not intent or not amount or not project_id:
+            generate_voice_response(
+                "Mandatory fields are missing. Please provide the required information again."
+            )
+            st.warning("Mandatory fields missing. Please try again.")
+            continue
+
+        
+        st.write("### Extracted Data")
+        st.text(f"Extracted Text: {text}")
+        st.text(f"Intent: {intent}")
+        st.text(f"Project Name: {project_name}")
+        st.text(f"Project ID: {project_id}")
+        st.text(f"Amount: {amount} {currency}")
+        st.text(f"Task ID: {task_id}")
+        st.text(f"Status: {status}")
+
+        response = (
+            f"You have requested for the task: Intent: {intent}, "
+            f"Project: {project_name}. Project ID: {project_id}. "
+            f"Amount: {amount} {currency}. Task ID: {task_id} and Status: {status}. "
+            "Please confirm by typing your response: Yes or No."
+        )
+        generate_voice_response(response)
+
+        # User confirmation
+        # user_input = st.text_input("Type your response (Yes/No):")
+        user_input = st.text_input("Type 'yes' or 'no':").strip().lower()
+        if user_input.lower() == "yes":
+            request_data = {
+                "project": project_name,
+                "project_id": project_id,
+                "amount": amount,
+                "Intent": intent,
+                "task_id": task_id,
+                "status": status,
+            }
+
+            # Save to database
+            database = load_database()
+            database["requests"].append(request_data)
+            save_to_database(database)
+
+            generate_voice_response(
+                "Thank you for your response, Your request has been confirmed successfully."
+            )
+            st.success("Request confirmed and saved successfully.")
+            st.session_state.reset = True
+            break
+        elif user_input.lower() == "no":
+            generate_voice_response(
+                "Thank you for your response, You have denied the confirmation request."
+            )
+            st.warning("Request denied.")
+            st.session_state.reset = True
+            break
+        # else:
+        #     generate_voice_response("You have typed an invalid response.")
+        #     st.error("Invalid response. Please try again.")
+        #     continue
+
+
+# Streamlit App
+st.title("ERP Voice Request Handling AI System-Demo")
+st.write("Upload an audio file and extract information from the request.")
+
+# Upload audio file
+audio_file = st.file_uploader("Upload Audio File", type=["wav", "mp3"])
+
+if audio_file:
+    st.write("### Processing Audio Input")
+    handle_request(audio_file)
+
+# Display database records
+st.write("### Saved Requests in Database")
+database = load_database()
+if database["requests"]:
+    st.json(database)
+else:
+    st.write("No requests found.")

database.json

@@ -0,0 +1,60 @@
+{
+    "requests": [
+        {
+            "project": null,
+            "project_id": "223",
+            "amount": "500",
+            "Intent": "Request Money",
+            "task_id": null,
+            "status": null
+        },
+        {
+            "project": null,
+            "project_id": "223",
+            "amount": "500",
+            "Intent": "Request Money",
+            "task_id": null,
+            "status": null
+        },
+        {
+            "project": null,
+            "project_id": "223",
+            "amount": "500",
+            "Intent": "Request Money",
+            "task_id": null,
+            "status": null
+        },
+        {
+            "project": null,
+            "project_id": "223",
+            "amount": "500",
+            "Intent": "Request Money",
+            "task_id": null,
+            "status": null
+        },
+        {
+            "project": null,
+            "project_id": "223",
+            "amount": "500",
+            "Intent": "Request Money",
+            "task_id": null,
+            "status": null
+        },
+        {
+            "project": null,
+            "project_id": "223",
+            "amount": "500",
+            "Intent": "Request Money",
+            "task_id": null,
+            "status": null
+        },
+        {
+            "project": null,
+            "project_id": "223",
+            "amount": "500",
+            "Intent": "Request Money",
+            "task_id": null,
+            "status": null
+        }
+    ]
+}

fine_tune_nlu.py

@@ -0,0 +1,141 @@
+from transformers import BertTokenizerFast, BertForTokenClassification, Trainer, TrainingArguments, DataCollatorForTokenClassification
+from datasets import DatasetDict, Dataset
+import json
+
+
+def preprocess_data1(json_path, tokenizer):
+    with open(json_path, "r") as f:
+        data = json.load(f)["data"]
+
+    tokenized_data = {"input_ids": [], "attention_mask": [], "labels": []}
+    slot_label_map = {"O": 0}
+    label_id = 1
+
+    for intent_data in data:
+        for utterance in intent_data["utterances"]:
+            text = utterance["text"]
+            encoding = tokenizer(
+                text, 
+                truncation=True, 
+                padding="max_length", 
+                max_length=128,  
+                return_offsets_mapping=True
+            )
+            tokens = tokenizer.convert_ids_to_tokens(encoding["input_ids"])
+
+            # Create slot labels for the tokens
+            slot_labels = ["O"] * len(tokens)
+            for slot, value in utterance["slots"].items():
+                if value != "not specified":  # Skip unspecified slots
+                    slot_tokens = tokenizer.tokenize(value)
+                    for i in range(len(tokens) - len(slot_tokens) + 1):
+                        if tokens[i:i + len(slot_tokens)] == slot_tokens:
+                            slot_labels[i] = f"B-{slot}"
+                            for j in range(1, len(slot_tokens)):
+                                slot_labels[i + j] = f"I-{slot}"
+
+            # Map slot labels to IDs
+            for label in slot_labels:
+                if label not in slot_label_map:
+                    slot_label_map[label] = label_id
+                    label_id += 1
+
+            label_ids = [slot_label_map[label] for label in slot_labels]
+
+            
+            tokenized_data["input_ids"].append(encoding["input_ids"])
+            tokenized_data["attention_mask"].append(encoding["attention_mask"])
+            tokenized_data["labels"].append(label_ids)
+
+    
+    print("Slot Label Map:", slot_label_map)
+    
+    
+    dataset = Dataset.from_dict(tokenized_data)
+    return DatasetDict({"train": dataset, "validation": dataset}), slot_label_map
+
+
+# Update training preprocessing to handle multi-token amount
+def preprocess_data(json_path, tokenizer):
+    with open(json_path, "r") as f:
+        data = json.load(f)["data"]
+
+    tokenized_data = {"input_ids": [], "attention_mask": [], "labels": []}
+    slot_label_map = {"O": 0}  
+
+    for intent_data in data:
+        for utterance in intent_data["utterances"]:
+            text = utterance["text"]
+            encoding = tokenizer(
+                text, 
+                truncation=True, 
+                padding="max_length", 
+                max_length=128, 
+                return_offsets_mapping=True
+            )
+            tokens = tokenizer.convert_ids_to_tokens(encoding["input_ids"])
+
+            
+            slot_labels = ["O"] * len(tokens)
+            for slot, value in utterance["slots"].items():
+                if value != "not specified": 
+                   
+                    slot_tokens = tokenizer.tokenize(value)
+                    for i in range(len(tokens) - len(slot_tokens) + 1):
+                        if tokens[i:i + len(slot_tokens)] == slot_tokens:
+                            slot_labels[i] = f"B-{slot}"
+                            for j in range(1, len(slot_tokens)):
+                                slot_labels[i + j] = f"I-{slot}"
+
+            # Map slot labels to IDs
+            for label in slot_labels:
+                if label not in slot_label_map:
+                    slot_label_map[label] = label_id
+                    label_id += 1
+
+            label_ids = [slot_label_map[label] for label in slot_labels]
+
+           
+            tokenized_data["input_ids"].append(encoding["input_ids"])
+            tokenized_data["attention_mask"].append(encoding["attention_mask"])
+            tokenized_data["labels"].append(label_ids)
+
+   
+    dataset = Dataset.from_dict(tokenized_data)
+    return DatasetDict({"train": dataset, "validation": dataset}), slot_label_map
+
+
+tokenizer = BertTokenizerFast.from_pretrained("bert-base-multilingual-cased")
+
+
+json_path = "nlu_dataset.json"  
+dataset, slot_label_map = preprocess_data(json_path, tokenizer)
+
+
+model = BertForTokenClassification.from_pretrained(
+    "bert-base-multilingual-cased", 
+    num_labels=len(slot_label_map)  
+)
+
+
+data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)
+
+
+training_args = TrainingArguments(
+    output_dir="./results",
+    num_train_epochs=100,
+    per_device_train_batch_size=16,
+    per_device_eval_batch_size=16,
+    evaluation_strategy="epoch",
+    logging_dir="./logs",
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=dataset["train"],
+    eval_dataset=dataset["validation"],
+    data_collator=data_collator
+)
+
+trainer.train()

generate_response.py

@@ -0,0 +1,8 @@
+import pyttsx3
+
+def generate_voice_response(text: str, lang="en"):
+    engine = pyttsx3.init()
+    engine.setProperty('rate', 150)  # Speed
+    engine.setProperty('volume', 1)  # (0.0 to 1.0)
+    engine.say(text)
+    engine.runAndWait()

intent_recognition.py

@@ -0,0 +1,87 @@
+
+import re
+from sentence_transformers import SentenceTransformer, util
+from typing import Dict, Any
+
+
+model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')
+
+# Define the Intent Rules
+dataset = {
+    "data": [
+        {
+            "intent": "request_money",
+            "utterances": [
+                {"text": "I need to request money for project 223 to buy some tools, the amount I need is 500 riyals"},
+                {"text": "Please add a money request for the project Abha University for 300 riyals"},
+                {"text": "I need 1000 riyals for project 445 to purchase some equipment"},
+                {"text": "Can you initiate a money request for project 678 with an amount of 250 riyals for team activities?"},
+                {"text": "Requesting 800 riyals for the project Green Energy for office supplies"}
+            ]
+        },
+        {
+            "intent": "submit_task",
+            "utterances": [
+                {"text": "I have completed the task 1025, please mark it as done"},
+                {"text": "Mark task 3054 as finished in the system"},
+                {"text": "Task 8899 has been completed, update its status"},
+                {"text": "Please mark task 1122 as done, I just finished it"},
+                {"text": "Set the status of task 4500 to finished"}
+            ]
+        },
+        {
+            "intent": "get_project_status",
+            "utterances": [
+                {"text": "Can you tell me the status of project 223?"},
+                {"text": "What is the current progress on project Abha University?"},
+                {"text": "I need an update on project 445. What is its status?"},
+                {"text": "Could you check and let me know the status of the Smart City project?"},
+                {"text": "What’s the progress on the renewable energy project?"}
+            ]
+        }
+    ]
+}
+
+def extract_amount_with_context(text: str) -> Dict[str, Any]:
+    """Extract the amount (in currency) along with the currency term and context using regex."""
+    
+    # Adjust the regex to capture the amount and surrounding words
+    match = re.search(r'(\d+)\s*(riyals?|reels?|rils?|reel?|dollars?|money|amount|usd|euro|pounds?)\s*(\w{1,20})?(\w{1,20})?', text.lower())
+    
+    if match:
+        # Extract the amount and the currency type
+        amount = match.group(1)
+        currency = match.group(2)
+        additional_info = f"{match.group(3)} {match.group(4)}".strip() if match.group(3) or match.group(4) else None
+        return {"amount": amount, "currency": currency, "context": additional_info}
+    return None
+
+def get_intent_and_amount(text: str) -> Dict[str, Any]:
+    """
+    Extract intent and amount (if present) from a given text using a similarity model.
+    """
+    best_match = None
+    best_score = 0
+    intent = "unknown"
+    amount_data = extract_amount_with_context(text)  
+
+    # Now, let's detect the intent from the dataset
+    for intent_data in dataset["data"]:
+        for utterance in intent_data["utterances"]:
+            # Compute similarity
+            similarity_score = util.pytorch_cos_sim(
+                model.encode(text, convert_to_tensor=True),
+                model.encode(utterance["text"], convert_to_tensor=True)
+            ).item()
+
+            if similarity_score > best_score:
+                best_score = similarity_score
+                best_match = utterance
+                intent = intent_data["intent"]
+
+    return {"intent": intent, "amount_data": amount_data, "score": best_score}
+
+# Example test
+user_text = "Hey, I need to request money for a project name Abha University and id is 123 and the amount is 500 riyals"
+result = get_intent_and_amount(user_text)
+print(result)

main.py

@@ -0,0 +1,67 @@
+import json
+from whisper_stt import transcribe_audio_raw
+from intent_recognition import get_intent_and_amount
+from generate_response import generate_voice_response
+from test_NLU import get_slots
+
+DATABASE_PATH = "database.json"
+
+def load_database():
+    with open(DATABASE_PATH, "r") as db_file:
+        return json.load(db_file)
+
+def save_to_database(data):
+    with open(DATABASE_PATH, "w") as db_file:
+        json.dump(data, db_file, indent=4)
+
+def handle_request(audio_file):
+    
+    text = transcribe_audio_raw(audio_file)
+    
+    
+    intent_data = get_intent_and_amount(text)
+    intent=intent_data.get('intent')
+    intent=intent.replace("_", " ").title()
+    amount_data=intent_data.get('amount_data')
+    amount=amount_data.get('amount')
+    currency=amount_data.get('currency')
+    slots=get_slots(text)
+    slots['amount']=amount+' '+currency
+    if intent is not None:
+        response=f"You have requested for the task: Intent: {intent}, Project: {slots.get('project_name')}. Project ID: {slots.get('project_id')}.  Amount: {slots.get('amount')}. Task ID: {slots.get('task_id')} and Status: {slots.get('status')}. Please Confirm by typing your response: Yes or No: "
+        generate_voice_response(response) 
+        user_input=input("Please type your response: Yes or No: ")
+    
+        if user_input.lower()=="yes":
+            # Prepare the data to save
+            request_data = {
+                "project": slots.get("project_name"),
+                "project_id": slots.get("project_id"),
+                "amount": amount,
+                "Intent": intent,
+                "task_id": slots.get("task_id"),
+                "status": slots.get("status"),
+            }
+
+           
+            database = load_database()
+            database["requests"].append(request_data)
+            save_to_database(database)
+            generate_voice_response("Thank you for your response, Your request has been confirmed successfully.")
+            
+        elif user_input.lower()=="no":
+            generate_voice_response("Thank you for your response, You have denied the confirmation request.")
+        else:
+            generate_voice_response("You have typed an invalid response.")
+    else:
+        response = "Sorry, I did not understand your request."
+        generate_voice_response(response)
+    return response
+    
+
+if __name__ == "__main__":
+    
+    user_audio = "input_audio.wav"
+    # audio_file = open(user_audio, "rb")
+    
+    print(handle_request(user_audio))

nlu_dataset.json

@@ -0,0 +1,124 @@
+{
+    "data": [
+        {
+            "intent": "request_money",
+            "utterances": [
+                {
+                    "text": "I need to request money for project 223 to buy some tools, the amount I need is 500 riyals",
+                    "slots": {
+                        "project_id": "223",
+                        "reason": "buy some tools",
+                        "amount": "500 riyals"
+                    }
+                },
+                {
+                    "text": "Please add a money request for the project Abha University for 300 riyals",
+                    "slots": {
+                        "project_name": "Abha University",
+                        "reason": "not specified",
+                        "amount": "300 riyals"
+                    }
+                },
+                {
+                    "text": "I need 1000 riyals for project 445 to purchase some equipment",
+                    "slots": {
+                        "project_id": "445",
+                        "reason": "purchase some equipment",
+                        "amount": "1000 riyals"
+                    }
+                },
+                {
+                    "text": "Can you initiate a money request for project 678 with an amount of 250 riyals for team activities?",
+                    "slots": {
+                        "project_id": "678",
+                        "reason": "team activities",
+                        "amount": "250 riyals"
+                    }
+                },
+                {
+                    "text": "Requesting 800 riyals for the project Green Energy for office supplies",
+                    "slots": {
+                        "project_name": "Green Energy",
+                        "reason": "office supplies",
+                        "amount": "800 riyals"
+                    }
+                }
+            ]
+        },
+        {
+            "intent": "submit_task",
+            "utterances": [
+                {
+                    "text": "I have completed the task 1025, please mark it as done",
+                    "slots": {
+                        "task_id": "1025",
+                        "status": "completed"
+                    }
+                },
+                {
+                    "text": "Mark task 3054 as finished in the system",
+                    "slots": {
+                        "task_id": "3054",
+                        "status": "finished"
+                    }
+                },
+                {
+                    "text": "Task 8899 has been completed, update its status",
+                    "slots": {
+                        "task_id": "8899",
+                        "status": "completed"
+                    }
+                },
+                {
+                    "text": "Please mark task 1122 as done, I just finished it",
+                    "slots": {
+                        "task_id": "1122",
+                        "status": "done"
+                    }
+                },
+                {
+                    "text": "Set the status of task 4500 to finished",
+                    "slots": {
+                        "task_id": "4500",
+                        "status": "finished"
+                    }
+                }
+            ]
+        },
+        {
+            "intent": "get_project_status",
+            "utterances": [
+                {
+                    "text": "Can you tell me the status of project 223?",
+                    "slots": {
+                        "project_id": "223"
+                    }
+                },
+                {
+                    "text": "What is the current progress on project Abha University?",
+                    "slots": {
+                        "project_name": "Abha University"
+                    }
+                },
+                {
+                    "text": "I need an update on project 445. What is its status?",
+                    "slots": {
+                        "project_id": "445"
+                    }
+                },
+                {
+                    "text": "Could you check and let me know the status of the Smart City project?",
+                    "slots": {
+                        "project_name": "Smart City"
+                    }
+                },
+                {
+                    "text": "What’s the progress on the renewable energy project?",
+                    "slots": {
+                        "project_name": "renewable energy"
+                    }
+                }
+            ]
+        }
+    ]
+}

requirements.txt

@@ -0,0 +1,171 @@
+absl-py==2.1.0
+accelerate==1.2.1
+aiohappyeyeballs==2.4.4
+aiohttp==3.11.11
+aiosignal==1.3.2
+altair==5.5.0
+annotated-types==0.7.0
+anyascii==0.3.2
+attrs==24.3.0
+audioread==3.0.1
+babel==2.16.0
+bangla==0.0.2
+blinker==1.9.0
+blis==1.1.0
+bnnumerizer==0.0.2
+bnunicodenormalizer==0.1.7
+cachetools==5.5.0
+catalogue==2.0.10
+certifi==2024.12.14
+cffi==1.17.1
+charset-normalizer==3.4.1
+click==8.1.8
+cloudpathlib==0.20.0
+colorama==0.4.6
+comtypes==1.4.8
+confection==0.1.5
+contourpy==1.3.1
+coqpit==0.0.17
+cycler==0.12.1
+cymem==2.0.10
+Cython==3.0.11
+datasets==3.2.0
+dateparser==1.1.8
+decorator==5.1.1
+dill==0.3.8
+docopt==0.6.2
+einops==0.8.0
+encodec==0.1.1
+filelock==3.16.1
+Flask==3.1.0
+fonttools==4.55.3
+frozenlist==1.5.0
+fsspec==2024.9.0
+g2pkk==0.1.2
+gitdb==4.0.11
+GitPython==3.1.43
+grpcio==1.68.1
+gruut==2.2.3
+gruut-ipa==0.13.0
+gruut_lang_de==2.0.1
+gruut_lang_en==2.0.1
+gruut_lang_es==2.0.1
+gruut_lang_fr==2.0.2
+hangul-romanize==0.1.0
+huggingface-hub==0.27.0
+idna==3.10
+inflect==7.4.0
+itsdangerous==2.2.0
+jamo==0.4.1
+jieba==0.42.1
+Jinja2==3.1.5
+joblib==1.4.2
+jsonlines==1.2.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+kiwisolver==1.4.8
+langcodes==3.5.0
+language_data==1.3.0
+lazy_loader==0.4
+librosa==0.10.2.post1
+llvmlite==0.43.0
+marisa-trie==1.2.1
+Markdown==3.7
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+matplotlib==3.10.0
+mdurl==0.1.2
+more-itertools==10.5.0
+mpmath==1.3.0
+msgpack==1.1.0
+multidict==6.1.0
+multiprocess==0.70.16
+murmurhash==1.0.11
+narwhals==1.19.1
+networkx==2.8.8
+nltk==3.9.1
+num2words==0.5.14
+numba==0.60.0
+numpy==1.26.4
+openai-whisper==20240930
+packaging==24.2
+pandas==1.5.3
+pillow==11.0.0
+platformdirs==4.3.6
+pooch==1.8.2
+preshed==3.0.9
+propcache==0.2.1
+protobuf==5.29.2
+psutil==6.1.1
+pyarrow==18.1.0
+pycparser==2.22
+pydantic==2.10.4
+pydantic_core==2.27.2
+pydeck==0.9.1
+pydub==0.25.1
+Pygments==2.18.0
+pynndescent==0.5.13
+pyparsing==3.2.0
+pypinyin==0.53.0
+pypiwin32==223
+pysbd==0.3.4
+python-crfsuite==0.9.11
+python-dateutil==2.9.0.post0
+pyttsx3==2.98
+pytz==2024.2
+pywin32==308
+PyYAML==6.0.2
+referencing==0.35.1
+regex==2024.11.6
+requests==2.32.3
+rich==13.9.4
+rpds-py==0.22.3
+safetensors==0.4.5
+scikit-learn==1.6.0
+scipy==1.14.1
+sentence-transformers==3.3.1
+shellingham==1.5.4
+six==1.17.0
+smart-open==7.1.0
+smmap==5.0.1
+soundfile==0.12.1
+soxr==0.5.0.post1
+spacy==3.8.3
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+srsly==2.5.0
+streamlit==1.41.1
+SudachiDict-core==20241021
+SudachiPy==0.6.9
+sympy==1.13.1
+tenacity==9.0.0
+tensorboard==2.18.0
+tensorboard-data-server==0.7.2
+thinc==8.3.3
+threadpoolctl==3.5.0
+tiktoken==0.8.0
+tokenizers==0.21.0
+toml==0.10.2
+torch==2.5.1
+torchaudio==2.5.1
+tornado==6.4.2
+tqdm==4.67.1
+trainer==0.0.36
+transformers==4.47.1
+TTS==0.22.0
+typeguard==4.4.1
+typer==0.15.1
+typing_extensions==4.12.2
+tzdata==2024.2
+tzlocal==5.2
+umap-learn==0.5.7
+Unidecode==1.3.8
+urllib3==2.3.0
+wasabi==1.1.3
+watchdog==6.0.0
+weasel==0.4.1
+Werkzeug==3.1.3
+whisper==1.1.10
+wrapt==1.17.0
+xxhash==3.5.0
+yarl==1.18.3

results/checkpoint-100/config.json

@@ -0,0 +1,57 @@
+{
+  "_name_or_path": "bert-base-multilingual-cased",
+  "architectures": [
+    "BertForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2",
+    "3": "LABEL_3",
+    "4": "LABEL_4",
+    "5": "LABEL_5",
+    "6": "LABEL_6",
+    "7": "LABEL_7",
+    "8": "LABEL_8",
+    "9": "LABEL_9",
+    "10": "LABEL_10"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_10": 10,
+    "LABEL_2": 2,
+    "LABEL_3": 3,
+    "LABEL_4": 4,
+    "LABEL_5": 5,
+    "LABEL_6": 6,
+    "LABEL_7": 7,
+    "LABEL_8": 8,
+    "LABEL_9": 9
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 119547
+}

results/checkpoint-100/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3fc34a5672fc68bec86ba7ac93ec8e1f3c5c5c524673048361d6feef7221237
+size 709108588

results/checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f5db8afdfb30a4f607648049882a8525861563526751d88d52cf6941d75e21b
+size 13990

results/checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b86863d3b3ac3245ff3a6b271bc0e11992d0ed1afe7e2015bafd85562b8ce01
+size 1064

results/checkpoint-100/trainer_state.json

@@ -0,0 +1,825 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 100.0,
+  "eval_steps": 500,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.7868185043334961,
+      "eval_runtime": 1.2903,
+      "eval_samples_per_second": 11.626,
+      "eval_steps_per_second": 0.775,
+      "step": 1
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.28457802534103394,
+      "eval_runtime": 1.5464,
+      "eval_samples_per_second": 9.7,
+      "eval_steps_per_second": 0.647,
+      "step": 2
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.21589069068431854,
+      "eval_runtime": 1.4746,
+      "eval_samples_per_second": 10.172,
+      "eval_steps_per_second": 0.678,
+      "step": 3
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 0.2054772973060608,
+      "eval_runtime": 1.6254,
+      "eval_samples_per_second": 9.229,
+      "eval_steps_per_second": 0.615,
+      "step": 4
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 0.19293639063835144,
+      "eval_runtime": 1.6474,
+      "eval_samples_per_second": 9.105,
+      "eval_steps_per_second": 0.607,
+      "step": 5
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 0.17430831491947174,
+      "eval_runtime": 1.8466,
+      "eval_samples_per_second": 8.123,
+      "eval_steps_per_second": 0.542,
+      "step": 6
+    },
+    {
+      "epoch": 7.0,
+      "eval_loss": 0.1566132754087448,
+      "eval_runtime": 1.6633,
+      "eval_samples_per_second": 9.018,
+      "eval_steps_per_second": 0.601,
+      "step": 7
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 0.15266196429729462,
+      "eval_runtime": 1.8344,
+      "eval_samples_per_second": 8.177,
+      "eval_steps_per_second": 0.545,
+      "step": 8
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 0.13321934640407562,
+      "eval_runtime": 1.7114,
+      "eval_samples_per_second": 8.765,
+      "eval_steps_per_second": 0.584,
+      "step": 9
+    },
+    {
+      "epoch": 10.0,
+      "eval_loss": 0.12868323922157288,
+      "eval_runtime": 1.6882,
+      "eval_samples_per_second": 8.885,
+      "eval_steps_per_second": 0.592,
+      "step": 10
+    },
+    {
+      "epoch": 11.0,
+      "eval_loss": 0.1147986426949501,
+      "eval_runtime": 2.0407,
+      "eval_samples_per_second": 7.35,
+      "eval_steps_per_second": 0.49,
+      "step": 11
+    },
+    {
+      "epoch": 12.0,
+      "eval_loss": 0.11238791793584824,
+      "eval_runtime": 1.8299,
+      "eval_samples_per_second": 8.197,
+      "eval_steps_per_second": 0.546,
+      "step": 12
+    },
+    {
+      "epoch": 13.0,
+      "eval_loss": 0.09630943089723587,
+      "eval_runtime": 1.9623,
+      "eval_samples_per_second": 7.644,
+      "eval_steps_per_second": 0.51,
+      "step": 13
+    },
+    {
+      "epoch": 14.0,
+      "eval_loss": 0.0899113267660141,
+      "eval_runtime": 2.0807,
+      "eval_samples_per_second": 7.209,
+      "eval_steps_per_second": 0.481,
+      "step": 14
+    },
+    {
+      "epoch": 15.0,
+      "eval_loss": 0.0796389952301979,
+      "eval_runtime": 2.043,
+      "eval_samples_per_second": 7.342,
+      "eval_steps_per_second": 0.489,
+      "step": 15
+    },
+    {
+      "epoch": 16.0,
+      "eval_loss": 0.07456444948911667,
+      "eval_runtime": 2.0972,
+      "eval_samples_per_second": 7.152,
+      "eval_steps_per_second": 0.477,
+      "step": 16
+    },
+    {
+      "epoch": 17.0,
+      "eval_loss": 0.0698675587773323,
+      "eval_runtime": 2.3192,
+      "eval_samples_per_second": 6.468,
+      "eval_steps_per_second": 0.431,
+      "step": 17
+    },
+    {
+      "epoch": 18.0,
+      "eval_loss": 0.06313543021678925,
+      "eval_runtime": 2.0522,
+      "eval_samples_per_second": 7.309,
+      "eval_steps_per_second": 0.487,
+      "step": 18
+    },
+    {
+      "epoch": 19.0,
+      "eval_loss": 0.05887909233570099,
+      "eval_runtime": 1.9907,
+      "eval_samples_per_second": 7.535,
+      "eval_steps_per_second": 0.502,
+      "step": 19
+    },
+    {
+      "epoch": 20.0,
+      "eval_loss": 0.0551617294549942,
+      "eval_runtime": 2.0302,
+      "eval_samples_per_second": 7.388,
+      "eval_steps_per_second": 0.493,
+      "step": 20
+    },
+    {
+      "epoch": 21.0,
+      "eval_loss": 0.0511007234454155,
+      "eval_runtime": 2.2652,
+      "eval_samples_per_second": 6.622,
+      "eval_steps_per_second": 0.441,
+      "step": 21
+    },
+    {
+      "epoch": 22.0,
+      "eval_loss": 0.04705721512436867,
+      "eval_runtime": 2.3722,
+      "eval_samples_per_second": 6.323,
+      "eval_steps_per_second": 0.422,
+      "step": 22
+    },
+    {
+      "epoch": 23.0,
+      "eval_loss": 0.0431244932115078,
+      "eval_runtime": 2.2034,
+      "eval_samples_per_second": 6.808,
+      "eval_steps_per_second": 0.454,
+      "step": 23
+    },
+    {
+      "epoch": 24.0,
+      "eval_loss": 0.039149921387434006,
+      "eval_runtime": 2.1518,
+      "eval_samples_per_second": 6.971,
+      "eval_steps_per_second": 0.465,
+      "step": 24
+    },
+    {
+      "epoch": 25.0,
+      "eval_loss": 0.03541847690939903,
+      "eval_runtime": 2.111,
+      "eval_samples_per_second": 7.106,
+      "eval_steps_per_second": 0.474,
+      "step": 25
+    },
+    {
+      "epoch": 26.0,
+      "eval_loss": 0.03274580463767052,
+      "eval_runtime": 2.0862,
+      "eval_samples_per_second": 7.19,
+      "eval_steps_per_second": 0.479,
+      "step": 26
+    },
+    {
+      "epoch": 27.0,
+      "eval_loss": 0.030214540660381317,
+      "eval_runtime": 2.1022,
+      "eval_samples_per_second": 7.135,
+      "eval_steps_per_second": 0.476,
+      "step": 27
+    },
+    {
+      "epoch": 28.0,
+      "eval_loss": 0.02784493751823902,
+      "eval_runtime": 2.1074,
+      "eval_samples_per_second": 7.118,
+      "eval_steps_per_second": 0.475,
+      "step": 28
+    },
+    {
+      "epoch": 29.0,
+      "eval_loss": 0.02551179751753807,
+      "eval_runtime": 2.2566,
+      "eval_samples_per_second": 6.647,
+      "eval_steps_per_second": 0.443,
+      "step": 29
+    },
+    {
+      "epoch": 30.0,
+      "eval_loss": 0.02329176291823387,
+      "eval_runtime": 2.3476,
+      "eval_samples_per_second": 6.39,
+      "eval_steps_per_second": 0.426,
+      "step": 30
+    },
+    {
+      "epoch": 31.0,
+      "eval_loss": 0.02115248702466488,
+      "eval_runtime": 2.3423,
+      "eval_samples_per_second": 6.404,
+      "eval_steps_per_second": 0.427,
+      "step": 31
+    },
+    {
+      "epoch": 32.0,
+      "eval_loss": 0.01917836256325245,
+      "eval_runtime": 2.44,
+      "eval_samples_per_second": 6.148,
+      "eval_steps_per_second": 0.41,
+      "step": 32
+    },
+    {
+      "epoch": 33.0,
+      "eval_loss": 0.017496634274721146,
+      "eval_runtime": 2.3798,
+      "eval_samples_per_second": 6.303,
+      "eval_steps_per_second": 0.42,
+      "step": 33
+    },
+    {
+      "epoch": 34.0,
+      "eval_loss": 0.016098586842417717,
+      "eval_runtime": 2.8157,
+      "eval_samples_per_second": 5.327,
+      "eval_steps_per_second": 0.355,
+      "step": 34
+    },
+    {
+      "epoch": 35.0,
+      "eval_loss": 0.014923782087862492,
+      "eval_runtime": 2.455,
+      "eval_samples_per_second": 6.11,
+      "eval_steps_per_second": 0.407,
+      "step": 35
+    },
+    {
+      "epoch": 36.0,
+      "eval_loss": 0.013880550861358643,
+      "eval_runtime": 2.6382,
+      "eval_samples_per_second": 5.686,
+      "eval_steps_per_second": 0.379,
+      "step": 36
+    },
+    {
+      "epoch": 37.0,
+      "eval_loss": 0.012886795215308666,
+      "eval_runtime": 2.8315,
+      "eval_samples_per_second": 5.298,
+      "eval_steps_per_second": 0.353,
+      "step": 37
+    },
+    {
+      "epoch": 38.0,
+      "eval_loss": 0.012055573984980583,
+      "eval_runtime": 2.8786,
+      "eval_samples_per_second": 5.211,
+      "eval_steps_per_second": 0.347,
+      "step": 38
+    },
+    {
+      "epoch": 39.0,
+      "eval_loss": 0.011289969086647034,
+      "eval_runtime": 3.6692,
+      "eval_samples_per_second": 4.088,
+      "eval_steps_per_second": 0.273,
+      "step": 39
+    },
+    {
+      "epoch": 40.0,
+      "eval_loss": 0.010605846531689167,
+      "eval_runtime": 3.3063,
+      "eval_samples_per_second": 4.537,
+      "eval_steps_per_second": 0.302,
+      "step": 40
+    },
+    {
+      "epoch": 41.0,
+      "eval_loss": 0.00994051992893219,
+      "eval_runtime": 3.0948,
+      "eval_samples_per_second": 4.847,
+      "eval_steps_per_second": 0.323,
+      "step": 41
+    },
+    {
+      "epoch": 42.0,
+      "eval_loss": 0.009244030341506004,
+      "eval_runtime": 3.2056,
+      "eval_samples_per_second": 4.679,
+      "eval_steps_per_second": 0.312,
+      "step": 42
+    },
+    {
+      "epoch": 43.0,
+      "eval_loss": 0.008599556051194668,
+      "eval_runtime": 3.1486,
+      "eval_samples_per_second": 4.764,
+      "eval_steps_per_second": 0.318,
+      "step": 43
+    },
+    {
+      "epoch": 44.0,
+      "eval_loss": 0.007965602912008762,
+      "eval_runtime": 3.2206,
+      "eval_samples_per_second": 4.658,
+      "eval_steps_per_second": 0.311,
+      "step": 44
+    },
+    {
+      "epoch": 45.0,
+      "eval_loss": 0.007378988899290562,
+      "eval_runtime": 3.3572,
+      "eval_samples_per_second": 4.468,
+      "eval_steps_per_second": 0.298,
+      "step": 45
+    },
+    {
+      "epoch": 46.0,
+      "eval_loss": 0.0068125114776194096,
+      "eval_runtime": 3.2586,
+      "eval_samples_per_second": 4.603,
+      "eval_steps_per_second": 0.307,
+      "step": 46
+    },
+    {
+      "epoch": 47.0,
+      "eval_loss": 0.006272478960454464,
+      "eval_runtime": 3.2573,
+      "eval_samples_per_second": 4.605,
+      "eval_steps_per_second": 0.307,
+      "step": 47
+    },
+    {
+      "epoch": 48.0,
+      "eval_loss": 0.005782509222626686,
+      "eval_runtime": 3.5423,
+      "eval_samples_per_second": 4.235,
+      "eval_steps_per_second": 0.282,
+      "step": 48
+    },
+    {
+      "epoch": 49.0,
+      "eval_loss": 0.005347794853150845,
+      "eval_runtime": 3.2104,
+      "eval_samples_per_second": 4.672,
+      "eval_steps_per_second": 0.311,
+      "step": 49
+    },
+    {
+      "epoch": 50.0,
+      "eval_loss": 0.004948154091835022,
+      "eval_runtime": 3.2715,
+      "eval_samples_per_second": 4.585,
+      "eval_steps_per_second": 0.306,
+      "step": 50
+    },
+    {
+      "epoch": 51.0,
+      "eval_loss": 0.004586088005453348,
+      "eval_runtime": 3.2838,
+      "eval_samples_per_second": 4.568,
+      "eval_steps_per_second": 0.305,
+      "step": 51
+    },
+    {
+      "epoch": 52.0,
+      "eval_loss": 0.004253142047673464,
+      "eval_runtime": 3.0694,
+      "eval_samples_per_second": 4.887,
+      "eval_steps_per_second": 0.326,
+      "step": 52
+    },
+    {
+      "epoch": 53.0,
+      "eval_loss": 0.003955014981329441,
+      "eval_runtime": 3.3957,
+      "eval_samples_per_second": 4.417,
+      "eval_steps_per_second": 0.294,
+      "step": 53
+    },
+    {
+      "epoch": 54.0,
+      "eval_loss": 0.0036877128295600414,
+      "eval_runtime": 3.0922,
+      "eval_samples_per_second": 4.851,
+      "eval_steps_per_second": 0.323,
+      "step": 54
+    },
+    {
+      "epoch": 55.0,
+      "eval_loss": 0.003441128646954894,
+      "eval_runtime": 3.0702,
+      "eval_samples_per_second": 4.886,
+      "eval_steps_per_second": 0.326,
+      "step": 55
+    },
+    {
+      "epoch": 56.0,
+      "eval_loss": 0.00322518078610301,
+      "eval_runtime": 3.2255,
+      "eval_samples_per_second": 4.65,
+      "eval_steps_per_second": 0.31,
+      "step": 56
+    },
+    {
+      "epoch": 57.0,
+      "eval_loss": 0.003025263315066695,
+      "eval_runtime": 3.2191,
+      "eval_samples_per_second": 4.66,
+      "eval_steps_per_second": 0.311,
+      "step": 57
+    },
+    {
+      "epoch": 58.0,
+      "eval_loss": 0.002828031312674284,
+      "eval_runtime": 3.2217,
+      "eval_samples_per_second": 4.656,
+      "eval_steps_per_second": 0.31,
+      "step": 58
+    },
+    {
+      "epoch": 59.0,
+      "eval_loss": 0.002643935615196824,
+      "eval_runtime": 3.2818,
+      "eval_samples_per_second": 4.571,
+      "eval_steps_per_second": 0.305,
+      "step": 59
+    },
+    {
+      "epoch": 60.0,
+      "eval_loss": 0.0024718190543353558,
+      "eval_runtime": 3.1539,
+      "eval_samples_per_second": 4.756,
+      "eval_steps_per_second": 0.317,
+      "step": 60
+    },
+    {
+      "epoch": 61.0,
+      "eval_loss": 0.0023161820136010647,
+      "eval_runtime": 3.0914,
+      "eval_samples_per_second": 4.852,
+      "eval_steps_per_second": 0.323,
+      "step": 61
+    },
+    {
+      "epoch": 62.0,
+      "eval_loss": 0.0021715254988521338,
+      "eval_runtime": 3.9473,
+      "eval_samples_per_second": 3.8,
+      "eval_steps_per_second": 0.253,
+      "step": 62
+    },
+    {
+      "epoch": 63.0,
+      "eval_loss": 0.002039993414655328,
+      "eval_runtime": 3.0608,
+      "eval_samples_per_second": 4.901,
+      "eval_steps_per_second": 0.327,
+      "step": 63
+    },
+    {
+      "epoch": 64.0,
+      "eval_loss": 0.0019214763306081295,
+      "eval_runtime": 3.5577,
+      "eval_samples_per_second": 4.216,
+      "eval_steps_per_second": 0.281,
+      "step": 64
+    },
+    {
+      "epoch": 65.0,
+      "eval_loss": 0.0018131383694708347,
+      "eval_runtime": 3.6577,
+      "eval_samples_per_second": 4.101,
+      "eval_steps_per_second": 0.273,
+      "step": 65
+    },
+    {
+      "epoch": 66.0,
+      "eval_loss": 0.001715756836347282,
+      "eval_runtime": 3.2888,
+      "eval_samples_per_second": 4.561,
+      "eval_steps_per_second": 0.304,
+      "step": 66
+    },
+    {
+      "epoch": 67.0,
+      "eval_loss": 0.0016270950436592102,
+      "eval_runtime": 3.1551,
+      "eval_samples_per_second": 4.754,
+      "eval_steps_per_second": 0.317,
+      "step": 67
+    },
+    {
+      "epoch": 68.0,
+      "eval_loss": 0.0015468894271180034,
+      "eval_runtime": 3.1791,
+      "eval_samples_per_second": 4.718,
+      "eval_steps_per_second": 0.315,
+      "step": 68
+    },
+    {
+      "epoch": 69.0,
+      "eval_loss": 0.001476285862736404,
+      "eval_runtime": 3.1149,
+      "eval_samples_per_second": 4.816,
+      "eval_steps_per_second": 0.321,
+      "step": 69
+    },
+    {
+      "epoch": 70.0,
+      "eval_loss": 0.0014136920217424631,
+      "eval_runtime": 3.181,
+      "eval_samples_per_second": 4.716,
+      "eval_steps_per_second": 0.314,
+      "step": 70
+    },
+    {
+      "epoch": 71.0,
+      "eval_loss": 0.0013573016040027142,
+      "eval_runtime": 3.9126,
+      "eval_samples_per_second": 3.834,
+      "eval_steps_per_second": 0.256,
+      "step": 71
+    },
+    {
+      "epoch": 72.0,
+      "eval_loss": 0.0013067907420918345,
+      "eval_runtime": 3.1187,
+      "eval_samples_per_second": 4.81,
+      "eval_steps_per_second": 0.321,
+      "step": 72
+    },
+    {
+      "epoch": 73.0,
+      "eval_loss": 0.0012619098415598273,
+      "eval_runtime": 3.3641,
+      "eval_samples_per_second": 4.459,
+      "eval_steps_per_second": 0.297,
+      "step": 73
+    },
+    {
+      "epoch": 74.0,
+      "eval_loss": 0.0012214797316119075,
+      "eval_runtime": 3.3495,
+      "eval_samples_per_second": 4.478,
+      "eval_steps_per_second": 0.299,
+      "step": 74
+    },
+    {
+      "epoch": 75.0,
+      "eval_loss": 0.0011843887623399496,
+      "eval_runtime": 3.1759,
+      "eval_samples_per_second": 4.723,
+      "eval_steps_per_second": 0.315,
+      "step": 75
+    },
+    {
+      "epoch": 76.0,
+      "eval_loss": 0.0011517057428136468,
+      "eval_runtime": 3.8153,
+      "eval_samples_per_second": 3.931,
+      "eval_steps_per_second": 0.262,
+      "step": 76
+    },
+    {
+      "epoch": 77.0,
+      "eval_loss": 0.0011234048288315535,
+      "eval_runtime": 3.3084,
+      "eval_samples_per_second": 4.534,
+      "eval_steps_per_second": 0.302,
+      "step": 77
+    },
+    {
+      "epoch": 78.0,
+      "eval_loss": 0.0010975470067933202,
+      "eval_runtime": 3.1214,
+      "eval_samples_per_second": 4.806,
+      "eval_steps_per_second": 0.32,
+      "step": 78
+    },
+    {
+      "epoch": 79.0,
+      "eval_loss": 0.0010739320423454046,
+      "eval_runtime": 3.145,
+      "eval_samples_per_second": 4.769,
+      "eval_steps_per_second": 0.318,
+      "step": 79
+    },
+    {
+      "epoch": 80.0,
+      "eval_loss": 0.0010527895065024495,
+      "eval_runtime": 3.2234,
+      "eval_samples_per_second": 4.654,
+      "eval_steps_per_second": 0.31,
+      "step": 80
+    },
+    {
+      "epoch": 81.0,
+      "eval_loss": 0.0010336448904126883,
+      "eval_runtime": 3.6565,
+      "eval_samples_per_second": 4.102,
+      "eval_steps_per_second": 0.273,
+      "step": 81
+    },
+    {
+      "epoch": 82.0,
+      "eval_loss": 0.001016051392070949,
+      "eval_runtime": 3.2992,
+      "eval_samples_per_second": 4.547,
+      "eval_steps_per_second": 0.303,
+      "step": 82
+    },
+    {
+      "epoch": 83.0,
+      "eval_loss": 0.0010000885231420398,
+      "eval_runtime": 3.2485,
+      "eval_samples_per_second": 4.618,
+      "eval_steps_per_second": 0.308,
+      "step": 83
+    },
+    {
+      "epoch": 84.0,
+      "eval_loss": 0.000985819729976356,
+      "eval_runtime": 3.4147,
+      "eval_samples_per_second": 4.393,
+      "eval_steps_per_second": 0.293,
+      "step": 84
+    },
+    {
+      "epoch": 85.0,
+      "eval_loss": 0.0009730439051054418,
+      "eval_runtime": 3.3375,
+      "eval_samples_per_second": 4.494,
+      "eval_steps_per_second": 0.3,
+      "step": 85
+    },
+    {
+      "epoch": 86.0,
+      "eval_loss": 0.0009613920701667666,
+      "eval_runtime": 3.2532,
+      "eval_samples_per_second": 4.611,
+      "eval_steps_per_second": 0.307,
+      "step": 86
+    },
+    {
+      "epoch": 87.0,
+      "eval_loss": 0.0009508637012913823,
+      "eval_runtime": 3.3259,
+      "eval_samples_per_second": 4.51,
+      "eval_steps_per_second": 0.301,
+      "step": 87
+    },
+    {
+      "epoch": 88.0,
+      "eval_loss": 0.0009414219530299306,
+      "eval_runtime": 3.1885,
+      "eval_samples_per_second": 4.704,
+      "eval_steps_per_second": 0.314,
+      "step": 88
+    },
+    {
+      "epoch": 89.0,
+      "eval_loss": 0.0009328797459602356,
+      "eval_runtime": 3.9468,
+      "eval_samples_per_second": 3.801,
+      "eval_steps_per_second": 0.253,
+      "step": 89
+    },
+    {
+      "epoch": 90.0,
+      "eval_loss": 0.0009253285243175924,
+      "eval_runtime": 3.344,
+      "eval_samples_per_second": 4.486,
+      "eval_steps_per_second": 0.299,
+      "step": 90
+    },
+    {
+      "epoch": 91.0,
+      "eval_loss": 0.0009186835959553719,
+      "eval_runtime": 3.403,
+      "eval_samples_per_second": 4.408,
+      "eval_steps_per_second": 0.294,
+      "step": 91
+    },
+    {
+      "epoch": 92.0,
+      "eval_loss": 0.0009127946686930954,
+      "eval_runtime": 3.2406,
+      "eval_samples_per_second": 4.629,
+      "eval_steps_per_second": 0.309,
+      "step": 92
+    },
+    {
+      "epoch": 93.0,
+      "eval_loss": 0.0009078615694306791,
+      "eval_runtime": 3.2285,
+      "eval_samples_per_second": 4.646,
+      "eval_steps_per_second": 0.31,
+      "step": 93
+    },
+    {
+      "epoch": 94.0,
+      "eval_loss": 0.0009037015843205154,
+      "eval_runtime": 2.9444,
+      "eval_samples_per_second": 5.094,
+      "eval_steps_per_second": 0.34,
+      "step": 94
+    },
+    {
+      "epoch": 95.0,
+      "eval_loss": 0.0009001877042464912,
+      "eval_runtime": 3.459,
+      "eval_samples_per_second": 4.337,
+      "eval_steps_per_second": 0.289,
+      "step": 95
+    },
+    {
+      "epoch": 96.0,
+      "eval_loss": 0.0008972398354671896,
+      "eval_runtime": 3.2953,
+      "eval_samples_per_second": 4.552,
+      "eval_steps_per_second": 0.303,
+      "step": 96
+    },
+    {
+      "epoch": 97.0,
+      "eval_loss": 0.0008948465110734105,
+      "eval_runtime": 3.238,
+      "eval_samples_per_second": 4.633,
+      "eval_steps_per_second": 0.309,
+      "step": 97
+    },
+    {
+      "epoch": 98.0,
+      "eval_loss": 0.0008930906769819558,
+      "eval_runtime": 3.5433,
+      "eval_samples_per_second": 4.233,
+      "eval_steps_per_second": 0.282,
+      "step": 98
+    },
+    {
+      "epoch": 99.0,
+      "eval_loss": 0.0008919287356548011,
+      "eval_runtime": 3.3619,
+      "eval_samples_per_second": 4.462,
+      "eval_steps_per_second": 0.297,
+      "step": 99
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 100,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 100,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 97994256768000.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

results/checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4165414f4aaed45dd5f68a45321417b874e253512feaa0d080315683f181b0aa
+size 5240

test_NLU.py

@@ -0,0 +1,87 @@
+
+
+import torch
+from transformers import BertTokenizerFast,BertForTokenClassification
+import numpy as np
+
+
+tokenizer = BertTokenizerFast.from_pretrained("bert-base-multilingual-cased")
+model = BertForTokenClassification.from_pretrained("./results/checkpoint-100")  
+
+
+slot_label_map = {
+    0: "O", 1: "B-project_id", 2: "I-project_id", 3: "B-reason", 4: "I-reason",
+    5: "B-amount", 6: "I-amount", 7: "B-project_name", 8: "I-project_name",
+    9: "B-status", 10: "I-status",11: "B-riyals", 12: "I-riyals" 
+}
+
+
+
+def decode_slots(tokens, predictions, slot_label_map):
+    slots = {}
+    current_slot = None
+    current_value = []
+
+    for token, pred_id in zip(tokens, predictions):
+        label = slot_label_map[pred_id]
+
+        # Handle B- and I- slots
+        if label.startswith("B-"):  # Beginning of a new slot
+            if current_slot:
+                
+                slots[current_slot] = tokenizer.convert_tokens_to_string(current_value)
+            current_slot = label[2:]  # Extract slot name
+            current_value = [token]  # Start a new slot
+        elif label.startswith("I-") and current_slot == label[2:]:  # Continuation of the current slot
+            current_value.append(token)
+        else:  # No slot or "O"
+            if current_slot:
+                
+                slots[current_slot] = tokenizer.convert_tokens_to_string(current_value)
+                current_slot = None
+                current_value = []
+
+    if current_slot: 
+        slots[current_slot] = tokenizer.convert_tokens_to_string(current_value)
+
+    return slots
+
+
+def predict_intent_and_slots(text, model, tokenizer, slot_label_map):
+    encoding = tokenizer(
+        text,
+        truncation=True,
+        padding="max_length",
+        max_length=128,  # Same as during training
+        return_tensors="pt"
+    )
+    input_ids = encoding["input_ids"]
+    attention_mask = encoding["attention_mask"]
+
+    with torch.no_grad():
+        outputs = model(input_ids, attention_mask=attention_mask)
+        logits = outputs.logits
+        predictions = torch.argmax(logits, dim=2).squeeze().tolist()
+
+    tokens = tokenizer.convert_ids_to_tokens(input_ids.squeeze().tolist())
+    predictions = predictions[:len(tokens)]  
+
+   
+    slots = decode_slots(tokens, predictions, slot_label_map)
+
+    
+    intent = "mock_intent" 
+
+    return {"utterance": text, "slots": slots}
+
+def get_slots(text):
+    result = predict_intent_and_slots(text, model, tokenizer, slot_label_map)
+    slots=result['slots']
+    return slots
+
+# Test the model
+test_text = "Hey, I need to request money for a project name Abha University and id is 123 and the amount is 500 riyals"
+result = predict_intent_and_slots(test_text, model, tokenizer, slot_label_map)
+
+print("Prediction Result:")
+print(result)

whisper_stt.py

@@ -0,0 +1,37 @@
+import whisper
+import os
+import librosa
+import torch
+from transformers import pipeline
+
+def transcribe_audio_raw(file_path: str) -> str:
+    # file_path = "C:/Users/Lenovo/ML Notebooks/ERP Assistant/example.wav"
+    # if not os.path.exists(file_path):
+    #     print(f"File not found: {file_path}")
+    # else:
+    #     print("File found!")
+    # audio_data, sr = librosa.load(file_path, sr=None)
+    whisper_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-tiny.en", device="cpu")
+    transcription = whisper_pipe(file_path)
+    print(transcription)
+  
+    return transcription['text']
+
+
+import tempfile
+def transcribe_audio(uploaded_file):
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_file:
+        temp_file.write(uploaded_file.read())
+        file_path = temp_file.name
+
+    whisper_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-tiny.en", device="cpu")
+    transcription = whisper_pipe(file_path)
+    return transcription['text']
+
+
+
+
+
+
+
+