update

app.py

@@ -5,6 +5,7 @@ import os
 import string
 import logging
 from datetime import datetime
+from huggingface_hub import login, HfApi, hf_hub_download
 
 # Set up logging
 logging.basicConfig(level=logging.INFO,
@@ -15,144 +16,129 @@ logging.basicConfig(level=logging.INFO,
                     ])
 logger = logging.getLogger(__name__)
 
+# Use the Hugging Face token from environment variables
+hf_token = os.environ.get("HF_TOKEN")
+if hf_token:
+    login(token=hf_token)
+else:
+    logger.error("HF_TOKEN not found in environment variables")
+
 app = Flask(__name__)
 app.config['SECRET_KEY'] = 'supersecretkey'  # Change this to a random secret key
 
 # Directories for visualizations
-VISUALIZATION_DIRS_PLAN_OF_SQLS = {
-    "TP": "htmls_POS/TP",
-    "TN": "htmls_POS/TN",
-    "FP": "htmls_POS/FP",
-    "FN": "htmls_POS/FN"
-}
-
-VISUALIZATION_DIRS_CHAIN_OF_TABLE = {
-    "TP": "htmls_COT/TP",
-    "TN": "htmls_COT/TN",
-    "FP": "htmls_COT/FP",
-    "FN": "htmls_COT/FN"
+VISUALIZATION_DIRS = {
+    "No-XAI": "htmls_NO_XAI",
+    "Dater": "htmls_DATER",
+    "Chain-of-Table": "htmls_COT",
+    "Plan-of-SQLs": "htmls_POS"
 }
 
+METHODS = ["No-XAI", "Dater", "Chain-of-Table", "Plan-of-SQLs"]
 
 def save_session_data(username, data):
     try:
-        base_dir = os.path.dirname(os.path.abspath(__file__))
-        session_dir = os.path.join(base_dir, 'session_data')
-        os.makedirs(session_dir, exist_ok=True)
-
-        file_path = os.path.join(session_dir, f'{username}_session.json')
-
-        with open(file_path, 'w') as f:
-            json.dump(data, f, indent=4)
-
-        logger.info(f"Session data saved for user {username} at {file_path}")
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        file_name = f'{username}_{timestamp}_session.json'
+        json_data = json.dumps(data, indent=4)
+        temp_file_path = f"/tmp/{file_name}"
+        with open(temp_file_path, 'w') as f:
+            f.write(json_data)
+        api = HfApi()
+        api.upload_file(
+            path_or_fileobj=temp_file_path,
+            path_in_repo=f"session_data_foward_simulation/{file_name}",
+            repo_id="luulinh90s/Tabular-LLM-Study-Data",
+            repo_type="space",
+        )
+        os.remove(temp_file_path)
+        logger.info(f"Session data saved for user {username} in Hugging Face Data Space")
     except Exception as e:
         logger.exception(f"Error saving session data for user {username}: {e}")
 
-
-# Similarly, update the load_session_data function
 def load_session_data(username):
     try:
-        base_dir = os.path.dirname(os.path.abspath(__file__))
-        file_path = os.path.join(base_dir, 'session_data', f'{username}_session.json')
-
+        api = HfApi()
+        files = api.list_repo_files(repo_id="luulinh90s/Tabular-LLM-Study-Data", repo_type="space")
+        user_files = [f for f in files if f.startswith(f'session_data_graded/{username}_') and f.endswith('_session.json')]
+        if not user_files:
+            logger.warning(f"No session data found for user {username}")
+            return None
+        latest_file = sorted(user_files, reverse=True)[0]
+        file_path = hf_hub_download(repo_id="luulinh90s/Tabular-LLM-Study-Data", repo_type="space", filename=latest_file)
         with open(file_path, 'r') as f:
             data = json.load(f)
-
-        logger.info(f"Session data loaded for user {username} from {file_path}")
+        logger.info(f"Session data loaded for user {username} from Hugging Face Data Space")
         return data
-    except FileNotFoundError:
-        logger.warning(f"No session data found for user {username}")
-        return None
     except Exception as e:
         logger.exception(f"Error loading session data for user {username}: {e}")
         return None
 
-# Load all sample files from the directories based on the selected method
-def load_samples(method):
-    logger.info(f"Loading samples for method: {method}")
-    if method == "Chain-of-Table":
-        visualization_dirs = VISUALIZATION_DIRS_CHAIN_OF_TABLE
-    else:
-        visualization_dirs = VISUALIZATION_DIRS_PLAN_OF_SQLS
+def load_samples():
+    common_samples = []
+    categories = ["TP", "TN", "FP", "FN"]
 
-    samples = {"TP": [], "TN": [], "FP": [], "FN": []}
-    for category, dir_path in visualization_dirs.items():
-        try:
-            for filename in os.listdir(dir_path):
-                if filename.endswith(".html"):
-                    samples[category].append(filename)
-            logger.info(f"Loaded {len(samples[category])} samples for category {category}")
-        except Exception as e:
-            logger.exception(f"Error loading samples from {dir_path}: {e}")
-    return samples
+    for category in categories:
+        files = set(os.listdir(f'htmls_NO_XAI/{category}'))
+        for method in ["Dater", "Chain-of-Table", "Plan-of-SQLs"]:
+            method_dir = VISUALIZATION_DIRS[method]
+            files &= set(os.listdir(f'{method_dir}/{category}'))
+
+        for file in files:
+            common_samples.append({'category': category, 'file': file})
+
+    logger.info(f"Found {len(common_samples)} common samples across all methods")
+    return common_samples
 
-# Randomly select balanced samples
 def select_balanced_samples(samples):
     try:
-        tp_fp_samples = random.sample(samples["TP"] + samples["FP"], 5)
-        tn_fn_samples = random.sample(samples["TN"] + samples["FN"], 5)
-        logger.info(f"Selected balanced samples: {len(tp_fp_samples + tn_fn_samples)}")
-        return tp_fp_samples + tn_fn_samples
+        if len(samples) < 10:
+            logger.warning(f"Not enough common samples. Only {len(samples)} available.")
+            return samples
+
+        selected_samples = random.sample(samples, 10)
+        logger.info(f"Selected 10 unique samples")
+        return selected_samples
     except Exception as e:
         logger.exception("Error selecting balanced samples")
         return []
 
-def generate_random_string(length=8):
-    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))
-
-
 @app.route('/', methods=['GET', 'POST'])
 def index():
-    logger.info("Rendering index page.")
     if request.method == 'POST':
         username = request.form.get('username')
         seed = request.form.get('seed')
-
-        if not username or not seed:
-            logger.error("Missing username or seed.")
-            return "Missing username or seed", 400
-
+        method = request.form.get('method')
+        if not username or not seed or not method:
+            return "Please fill in all fields and select a method.", 400
         try:
             seed = int(seed)
             random.seed(seed)
-
-            # Use only one method (e.g., "Chain-of-Table")
-            method = "Chain-of-Table"
-            all_samples = load_samples(method)
+            all_samples = load_samples()
             selected_samples = select_balanced_samples(all_samples)
-            logger.info(f"Number of selected samples: {len(selected_samples)}")
-
             if len(selected_samples) == 0:
-                logger.error("No samples were selected.")
-                return "No samples were selected", 500
-
-            filename = f'{username}_{seed}_{method}_{generate_random_string()}.json'
-            logger.info(f"Generated filename: {filename}")
-
-            # Save session data
+                return "No common samples were found", 500
             session_data = {
-                'responses': [],
                 'username': username,
-                'selected_samples': selected_samples,
+                'seed': seed,
                 'method': method,
-                'filename': filename,
-                'current_index': 0
+                'selected_samples': selected_samples,
+                'current_index': 0,
+                'responses': [],
+                'start_time': datetime.now().isoformat()
             }
             save_session_data(username, session_data)
-            logger.info(f"Session data saved for user: {username}")
-
             return redirect(url_for('experiment', username=username))
         except Exception as e:
             logger.exception(f"Error in index route: {e}")
             return "An error occurred", 500
     return render_template('index.html')
+
 @app.route('/experiment/<username>', methods=['GET', 'POST'])
 def experiment(username):
     try:
         session_data = load_session_data(username)
         if not session_data:
-            logger.error(f"No session data found for user: {username}")
             return redirect(url_for('index'))
 
         selected_samples = session_data['selected_samples']
@@ -162,20 +148,9 @@ def experiment(username):
         if current_index >= len(selected_samples):
             return redirect(url_for('completed', username=username))
 
-        visualization_file = selected_samples[current_index]
-
-        vis_dir = 'htmls_COT' if method == "Chain-of-Table" else 'htmls_POS'
-
-        # Determine the correct visualization directory based on the category
-        for category, dir_path in VISUALIZATION_DIRS_CHAIN_OF_TABLE.items():
-            if visualization_file in os.listdir(dir_path):
-                visualization_path = os.path.join(vis_dir, category, visualization_file)
-                break
-        else:
-            logger.error(f"Visualization file {visualization_file} not found.")
-            return "Visualization file not found", 404
-
-        logger.info(f"Rendering experiment page with visualization: {visualization_path}")
+        sample = selected_samples[current_index]
+        visualization_dir = VISUALIZATION_DIRS[method]
+        visualization_path = f"{visualization_dir}/{sample['category']}/{sample['file']}"
 
         statement = """
 Based on the explanation provided, what do you think the AI model will predict?
@@ -186,11 +161,13 @@ Will it predict the statement as TRUE or FALSE?
                                sample_id=current_index,
                                statement=statement,
                                visualization=url_for('send_visualization', filename=visualization_path),
-                               username=username)
+                               username=username,
+                               method=method)
     except Exception as e:
         logger.exception(f"An error occurred in the experiment route: {e}")
         return "An error occurred", 500
 
+
 @app.route('/feedback', methods=['POST'])
 def feedback():
     try:
@@ -202,16 +179,12 @@ def feedback():
             logger.error(f"No session data found for user: {username}")
             return redirect(url_for('index'))
 
-        # Store the user's prediction
         session_data['responses'].append({
             'sample_id': session_data['current_index'],
             'user_prediction': prediction
         })
 
-        # Move to the next sample
         session_data['current_index'] += 1
-
-        # Save updated session data
         save_session_data(username, session_data)
         logger.info(f"Prediction saved for user {username}, sample {session_data['current_index'] - 1}")
 
@@ -223,7 +196,6 @@ def feedback():
         logger.exception(f"Error in feedback route: {e}")
         return "An error occurred", 500
 
-
 @app.route('/completed/<username>')
 def completed(username):
     try:
@@ -232,10 +204,11 @@ def completed(username):
             logger.error(f"No session data found for user: {username}")
             return redirect(url_for('index'))
 
+        session_data['end_time'] = datetime.now().isoformat()
         responses = session_data['responses']
         method = session_data['method']
 
-        json_file = 'Tabular_LLMs_human_study_vis_6_COT.json' if method == "Chain-of-Table" else 'Tabular_LLMs_human_study_vis_6_POS.json'
+        json_file = f'Tabular_LLMs_human_study_vis_6_{method.upper().replace("-", "_")}.json'
 
         with open(json_file, 'r') as f:
             ground_truth = json.load(f)
@@ -247,10 +220,10 @@ def completed(username):
         for response in responses:
             sample_id = response['sample_id']
             user_prediction = response['user_prediction']
-            visualization_file = session_data['selected_samples'][sample_id]
-            index = visualization_file.split('-')[1].split('.')[0]  # Extract index from filename
+            visualization_file = session_data['selected_samples'][sample_id]['file']
+            index = visualization_file.split('-')[1].split('.')[0]
 
-            ground_truth_key = f"COT_test-{index}.html" if method == "Chain-of-Table" else f"POS_test-{index}.html"
+            ground_truth_key = f"{method.upper().replace('-', '_')}_test-{index}.html"
 
             if ground_truth_key in ground_truth:
                 model_prediction = ground_truth[ground_truth_key]['answer'].upper()
@@ -273,6 +246,12 @@ def completed(username):
         true_percentage = round(true_percentage, 2)
         false_percentage = round(false_percentage, 2)
 
+        session_data['accuracy'] = accuracy
+        session_data['true_percentage'] = true_percentage
+        session_data['false_percentage'] = false_percentage
+
+        save_session_data(username, session_data)
+
         return render_template('completed.html',
                                accuracy=accuracy,
                                true_percentage=true_percentage,
@@ -281,11 +260,9 @@ def completed(username):
         logger.exception(f"An error occurred in the completed route: {e}")
         return "An error occurred", 500
 
-
 @app.route('/visualizations/<path:filename>')
 def send_visualization(filename):
     logger.info(f"Attempting to serve file: {filename}")
-    # Ensure the path is safe and doesn't allow access to files outside the intended directory
     base_dir = os.getcwd()
     file_path = os.path.normpath(os.path.join(base_dir, filename))
     if not file_path.startswith(base_dir):
@@ -299,7 +276,5 @@ def send_visualization(filename):
     logger.info(f"Serving file from directory: {directory}, filename: {file_name}")
     return send_from_directory(directory, file_name)
 
-
 if __name__ == "__main__":
-    os.makedirs('session_data', exist_ok=True)  # Ensure the directory for session files exists
     app.run(host="0.0.0.0", port=7860, debug=True)

templates/index.html

@@ -43,12 +43,12 @@
         .method-buttons {
             display: flex;
             flex-wrap: wrap;
-            justify-content: space-between;
+            justify-content: center;
             margin-bottom: 20px;
             gap: 20px;
         }
         .method-button {
-            width: calc(50% - 10px);
+            width: calc(45% - 10px);
             padding: 15px;
             font-size: 20px;
             border-radius: 10px;
@@ -66,11 +66,11 @@
             background-color: #ffcc80;
             color: #e65100;
         }
-        .method-button.cot-ext {
+        .method-button.dater {
             background-color: #e8f5e9;
             color: #4caf50;
         }
-        .method-button.pos-ext {
+        .method-button.no-xai {
             background-color: #fff3e0;
             color: #ff9800;
         }
@@ -139,11 +139,11 @@
                 <div class="method-button pos" onclick="selectMethod('Plan-of-SQLs')">
                     Plan-of-SQLs
                 </div>
-                <div class="method-button cot-ext" onclick="selectMethod('Chain-of-Table-Ext')">
-                    Chain-of-Table-Ext
+                <div class="method-button dater" onclick="selectMethod('Dater')">
+                    Dater
                 </div>
-                <div class="method-button pos-ext" onclick="selectMethod('Plan-of-SQLs-Ext')">
-                    Plan-of-SQLs-Ext
+                <div class="method-button no-xai" onclick="selectMethod('No-XAI')">
+                    No-XAI
                 </div>
             </div>