Second Commit

app.py

@@ -2,6 +2,7 @@ import os
 import json
 import gradio as gr
 import torch
+import pandas as pd
 from transformers import (
     TrainingArguments,
     Trainer,
@@ -29,6 +30,15 @@ def save_uploaded_file(file):
         f.write(file.read())
     return file_path
 
+def prepare_training_data(df):
+    """Convert DataFrame into Q&A format"""
+    formatted_data = []
+    for _, row in df.iterrows():
+        # Format each conversation in the required structure
+        formatted_text = f"User: {row['chunk_id']}\nAssistant: {row['text']}"
+        formatted_data.append({"text": formatted_text})
+    return formatted_data
+
 def prepare_training_components(
     data_path,
     learning_rate,
@@ -45,6 +55,10 @@ def prepare_training_components(
     os.makedirs(specific_output_dir, exist_ok=True)
     os.makedirs(LOGS_DIR, exist_ok=True)
 
+    # Load data and convert to Q&A format
+    df = pd.read_csv(data_path)
+    formatted_data = prepare_training_data(df)
+
     # Load tokenizer and model
     tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
     model = AutoModelForCausalLM.from_pretrained(
@@ -87,13 +101,9 @@ def prepare_training_components(
         save_total_limit=2,
     )
 
-    # Load and prepare dataset
-    with open(data_path, 'r') as f:
-        raw_data = json.load(f)
-    
     # Convert to datasets format
     dataset = Dataset.from_dict({
-        'text': [item['text'] for item in raw_data]
+        'text': [item['text'] for item in formatted_data]
     })
 
     # Create data collator
@@ -164,9 +174,9 @@ def create_interface():
         with gr.Row():
             with gr.Column():
                 file_input = gr.File(
-                    label="Upload Training Data (JSON)",
+                    label="Upload Training Data (CSV)",
                     type="binary",
-                    file_types=[".json"]
+                    file_types=[".csv"]
                 )
                 
                 learning_rate = gr.Slider(
@@ -205,13 +215,9 @@ def create_interface():
         
         gr.Markdown("""
         ## Instructions
-        1. Upload your training data in JSON format:
-        ```json
-        [
-            {"text": "User: Question\nAssistant: Answer"},
-            {"text": "User: Another question\nAssistant: Another answer"}
-        ]
-        ```
+        1. Upload your training data in CSV format with columns:
+           - chunk_id (questions)
+           - text (answers)
         2. Adjust training parameters if needed
         3. Click 'Start Training'
         4. Wait for training to complete