Create app.py

app.py

@@ -0,0 +1,272 @@
+import pandas as pd
+from openai import OpenAI
+import json
+import gradio as gr
+
+client = OpenAI()
+
+def generate_questions(category):
+  if category == "":
+    category = "general knowledge"
+  print(category)
+  response = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Break the category \""+category+"\" into 6 subcategories, and for each subcategory create 5 True/False questions ranging from a question a Beginner would know to a question only an Expert would know. There should be as many True as False, and the structure of the questions should not make it obvious which is the answer. The harder questions should be tricky and trick non-experts into saying the wrong thing. Provide the correct answers and a field with a 1 sentence explanation. This will total out to 30 questions. Output just a JSON, nothing else. Here's an example JSON output for \"nutrition\":\n\n```json\n{\n  \"Macronutrients\": [\n    {\n      \"question\": \"Protein is one of the three primary macronutrients.\",\n      \"answer\": True,\n      \"explanation\": \"Protein is one of the three primary macronutrients, along with carbohydrates and fats.\"\n    },\n    {\n      \"question\": \"Carbohydrates are the body's main source of energy.\",\n      \"answer\": True,\n      \"explanation\": \"Carbohydrates are typically the body's preferred energy source.\"\n    },\n    {\n      \"question\": \"Fats have the same caloric content per gram as carbohydrates.\",\n      \"answer\": False,\n      \"explanation\": \"Fats have 9 calories per gram, while carbohydrates have 4 calories per gram.\"\n    },\n    {\n      \"question\": \"All proteins are equally effective for muscle growth.\",\n      \"answer\": False,\n      \"explanation\": \"Different proteins have varying amino acid profiles and bioavailability, affecting their effectiveness.\"\n    },\n    {\n      \"question\": \"Omega-3 fatty acids are a type of fat that can reduce inflammation.\",\n      \"answer\": True,\n      \"explanation\": \"Omega-3 fatty acids, found in foods like fish, are known to have anti-inflammatory properties.\"\n    }\n  ],\n  \"Micronutrients\": [\n    { ..."
+          }
+        ]
+      }
+    ],
+    response_format={ "type": "json_object" },
+    temperature=1,
+    max_tokens=4071,
+    top_p=1,
+    frequency_penalty=0,
+    presence_penalty=0
+  )
+  return json.loads(response.choices[0].message.content)
+
+# Function to calculate MetaCog score
+def calculate_meta_cog_score(df):
+    df['Correct'] = df['User Answer'] == df['Correct Answer']
+    df['C'] = df['Correct'].apply(lambda x: 1 if x else -1)
+    n = len(df)
+    sum_C_Conf = (df['C'] * df['Confidence']).sum()
+    meta_cog_ratio = 0.5 + (sum_C_Conf / (2 * n))
+    return meta_cog_ratio
+
+def display_current_question(questions, index):
+    if index < len(questions):
+        question = questions[index]
+        return (
+            f"**Question {index + 1}:** {question['question']}",
+            None, None, True
+        )
+    else:
+        return ("", None, None, False)
+
+def calculate_scores(df):
+    df['Correct'] = df['User Answer'] == df['Correct Answer']
+    df['C'] = df['Correct'].apply(lambda x: 1 if x else 0)
+
+    # Expected score based on confidence
+    df['Expected Score'] = df['Confidence']
+    df['Actual Score'] = df['C']
+
+    # Difference between expected and actual scores
+    df['Overconfidence'] = (df['Expected Score'] > df['Actual Score']).astype(float) * (df['Expected Score'] - df['Actual Score'])
+    df['Underconfidence'] = (df['Expected Score'] < df['Actual Score']).astype(float) * (df['Actual Score'] - df['Expected Score'])
+
+    n = len(df)
+    sum_C_Conf = (df['C'] * df['Confidence']).sum()
+    meta_cog_ratio = 0.5 + (sum_C_Conf / (2 * n))
+
+    accuracy = df['Correct'].mean()
+    overconfidence = df['Overconfidence'].sum() / n
+    underconfidence = df['Underconfidence'].sum() / n
+
+    return {
+        'MetaCog Score': f"{round(meta_cog_ratio * 100, 0)}%",
+        'Accuracy': f"{round(accuracy * 100, 0)}%",
+        'Overconfidence': f"{round(overconfidence * 100, 0)}%",
+        'Underconfidence': f"{round(underconfidence * 100, 0)}%"
+    }
+
+# Function to analyze results using GPT-4o-mini
+def analyze_results(df, overall_scores, subcategory_scores):
+    # Prepare the data for analysis
+    questions = df['Question'].tolist()
+    correct_answers = df['Correct Answer'].tolist()
+    user_answers = df['User Answer'].tolist()
+    explanations = df['Explanation'].tolist()
+    confidence = df['Confidence'].tolist()
+    subcategories = df['Subcategory'].tolist()
+
+    # Generate a summary of the results
+    response = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {
+                "role": "user",
+                "content": f"""
+                Analyze the following quiz results:
+                - Overall MetaCog Score: {overall_scores['MetaCog Score']}
+                - Overall Accuracy: {overall_scores['Accuracy']}
+                - Overall Overconfidence: {overall_scores['Overconfidence']}
+                - Overall Underconfidence: {overall_scores['Underconfidence']}
+
+                Subcategory scores:
+                {subcategory_scores}
+
+                The following is a list of my answers and confidence levels for each question, with the correct answers and subcategory:
+                {list(zip(questions, user_answers, correct_answers, explanations, confidence, subcategories))}
+
+                Provide an analysis of what I got wrong in terms of overall sections and specific questions, as well as what I was overconfident and underconfident in. Don't use numbers, as they're already displayed elsewhere.
+                The analysis should be only about 2 paragraphs. Write the subcategory names in bold when you use them.
+                """
+            }
+        ],
+        # response_format={ "type": "json_object" },
+        temperature=0.7,
+        max_tokens=1024,
+        top_p=1,
+        frequency_penalty=0,
+        presence_penalty=0
+    )
+
+    analysis = response.choices[0].message.content
+
+    # Start the table with larger column titles using <b> for bold and <span> for custom styling
+    question_details = (
+        "<table><thead><tr>"
+        "<th><b><span style='font-size:16px'>Question</span></b></th>"
+        "<th><b><span style='font-size:16px'>User Answer</span></b></th>"
+        "<th><b><span style='font-size:16px'>Correct Answer</span></b></th>"
+        "<th><b><span style='font-size:16px'>Explanation</span></b></th>"
+        "</tr></thead><tbody>"
+    )
+
+    for q, ua, ca, subcategory, e in zip(questions, user_answers, correct_answers, subcategories, explanations):
+        user_answer_str = 'True' if ua else 'False'
+        correct_answer_str = 'True' if ca else 'False'
+
+        # Check if the answer is incorrect
+        if ua != ca:
+            question_details += (
+                f"<tr><td><b>{q}</b></td><td><b>{user_answer_str}</b></td>"
+                f"<td><b>{correct_answer_str}</b></td><td><b>{e}</b></td></tr>"
+            )
+        else:
+            question_details += (
+                f"<tr><td>{q}</td><td>{user_answer_str}</td>"
+                f"<td>{correct_answer_str}</td><td>{e}</td></tr>"
+            )
+
+    question_details += "</tbody></table>"
+
+    return f"## Analysis of Results\n\n{analysis}\n\n## Detailed Questions and Answers\n\n{question_details}"
+
+
+
+# Modify the submit_answer function to include analysis
+def submit_answer(category, questions, index, user_answer, confidence, user_answers):
+    question_data = questions[index]
+    subcategory = question_data["subcategory"]
+
+    user_answers.append({
+        "Question": question_data["question"],
+        "Explanation": question_data["explanation"],
+        "User Answer": user_answer == "True",
+        "Correct Answer": question_data["answer"],
+        "Confidence": confidence,
+        "Subcategory": subcategory
+    })
+    index += 1
+
+    if index >= len(questions):
+        df = pd.DataFrame(user_answers)
+        overall_scores = calculate_scores(df)
+        subcategory_scores = df.groupby('Subcategory').apply(calculate_scores).to_dict()
+        analysis = analyze_results(df, overall_scores, subcategory_scores)
+
+        overall_score_df = pd.DataFrame([["Overall", *overall_scores.values()]], columns=['Subcategory', 'Accuracy', 'MetaCog Score', 'Overconfidence', 'Underconfidence'])
+        subcategory_scores_df = pd.DataFrame([(subcategory, *score.values()) for subcategory, score in subcategory_scores.items()], columns=['Subcategory', 'MetaCog Score', 'Accuracy', 'Overconfidence', 'Underconfidence'])
+        results_df = pd.concat([overall_score_df, subcategory_scores_df], ignore_index=True)
+        results_df = gr.DataFrame(label="Results", value=results_df, visible=True)
+        return "", index, gr.update(visible=False), user_answers, results_df, gr.update(visible=False), gr.update(visible=False), gr.update(value=analysis, visible=True)
+    else:
+        question_text, _, _, visible = display_current_question(questions, index)
+        return question_text, index, gr.update(visible=True), user_answers, gr.update(visible=False), gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)
+
+# Gradio UI setup
+with gr.Blocks(theme="soft") as app:
+    gr.Markdown("""## Metacognition Test
+    See how well you know what you know. Enter a category and a 30-question quiz will be generated. Answer the True/False questions about it, and scores will then be calculated on your knowledge of the category, and on your ***knowledge of your knowledge*** of the category.""")
+    category_input = gr.Textbox(label="Category", placeholder="general knowledge")
+    submit_category = gr.Button("Generate Quiz")
+    question_area = gr.Markdown(visible=False)
+    answer_area = gr.Radio(["True", "False"], label="Your Answer", visible=False)
+    confidence_slider = gr.Slider(0, 1, label="Confidence Level", value=0.5, visible=False)
+    submit_answer_btn = gr.Button("Submit Answer", visible=False)
+    result_area = gr.DataFrame(label="Results", visible=False)
+    loading_text = gr.Textbox(label="Generating Test...", visible=False)
+    analysis_area = gr.Markdown(visible=False)  # Add this line for analysis area
+    questions_state = gr.State()
+    index_state = gr.State(0)
+    user_answers_state = gr.State([])
+
+    def on_generate_quiz(category):
+        questions_data = generate_questions(category)
+
+        questions = []
+        for subcategory, qs in questions_data.items():
+            for q in qs:
+                q["subcategory"] = subcategory
+                questions.append(q)
+
+        import random
+        random.shuffle(questions)
+
+        index = 0
+        question_text, _, _, visible = display_current_question(questions, index)
+        return (
+            gr.update(value=question_text, visible=visible),
+            questions,
+            index,
+            [],
+            gr.update(visible=visible),
+            gr.update(visible=True),
+            gr.update(visible=True),
+            gr.update(visible=True),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False)
+        )
+
+    def remove_button():
+        return gr.update(visible=False)
+
+    def display_loading():
+        return gr.update(visible=True)
+
+    def display_results(index, questions):
+        if index >= len(questions):
+          return gr.update(visible=True)
+
+    submit_category.click(remove_button, inputs=[], outputs=[submit_category])
+    submit_category.click(display_loading, inputs=[], outputs=[loading_text])
+
+    submit_category.click(
+        on_generate_quiz,
+        inputs=[category_input],
+        outputs=[
+            question_area,
+            questions_state,
+            index_state,
+            user_answers_state,
+            question_area,
+            answer_area,
+            confidence_slider,
+            submit_answer_btn,
+            result_area,
+            submit_category,
+            loading_text
+        ]
+    )
+
+
+    submit_answer_btn.click(
+        submit_answer,
+        inputs=[category_input, questions_state, index_state, answer_area, confidence_slider, user_answers_state],
+        outputs=[question_area, index_state, submit_answer_btn, user_answers_state, result_area, confidence_slider, answer_area, analysis_area]  # Add analysis_area here
+    )
+
+    submit_answer_btn.click(display_results, inputs=[index_state, questions_state], outputs=[result_area]
+
+# Launch the app
+app.launch(share=False)