xMerge branch 'master'

.gitignore

@@ -0,0 +1,3 @@
+.venv
+.env
+*_pycache_*

README.md

@@ -0,0 +1,20 @@
+---
+title: PixDiet
+emoji: 🚀
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: 4.44.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+hf_oauth: true
+tags:
+  - mistral
+  - pixtral
+  - nutrition
+  - diet
+  - chatbot
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,208 @@
+import torch
+from transformers import (
+    AutoProcessor,
+    BitsAndBytesConfig,
+    LlavaForConditionalGeneration,
+)
+from PIL import Image
+import gradio as gr
+from threading import Thread
+from transformers import TextIteratorStreamer, AutoModelForCausalLM, CodeGenTokenizerFast as Tokenizer
+from dotenv import load_dotenv
+import os
+# Import Supabase functions
+from db_client import get_user_history, update_user_history, delete_user_history
+
+load_dotenv()
+
+# Add TESTING variable
+TESTING = False  # You can change this to False when not testing
+
+IS_LOGGED_IN = True
+USER_ID = "[email protected]"
+
+# Hugging Face model id
+model_id = "blanchon/pixtral-nutrition-2"
+
+# BitsAndBytesConfig int-4 config
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+# Modify the model and processor initialization
+if TESTING:
+    model_id = "vikhyatk/moondream1"
+    model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True)
+    processor = Tokenizer.from_pretrained(model_id)
+else:
+    model = LlavaForConditionalGeneration.from_pretrained(
+        model_id,
+        device_map="auto",
+        torch_dtype=torch.bfloat16,
+        quantization_config=bnb_config,
+    )
+    processor = AutoProcessor.from_pretrained(model_id)
+
+# Set the chat template for the tokenizer
+processor.chat_template = """
+{%- for message in messages %}
+    {%- if message.role == "user" %}
+        <s>[INST]
+        {%- for item in message.content %}
+            {%- if item.type == "text" %}
+                {{ item.text }}
+            {%- elif item.type == "image" %}
+                \n[IMG]
+            {%- endif %}
+        {%- endfor %}
+        [/INST]
+    {%- elif message.role == "assistant" %}
+        {%- for item in message.content %}
+            {%- if item.type == "text" %}
+                {{ item.text }}
+            {%- endif %}
+        {%- endfor %}
+        </s>
+    {%- endif %}
+{%- endfor %}
+""".replace('    ', "")
+
+processor.tokenizer.pad_token = processor.tokenizer.eos_token
+
+
+def bot_streaming(chatbot, image_input, max_new_tokens=250):
+    # Preprocess inputs
+    messages = get_user_history(USER_ID)
+    images = []
+    text_input = chatbot[-1][0]
+
+    if text_input != "":
+        text_input = "you are a nutrition expert. Identify the food/ingredients in this image. Is this a healthy meal? Can you think of how to improve it?"
+
+    # Add current message
+    if image_input is not None:
+        # Check if image_input is already a PIL Image
+        if isinstance(image_input, Image.Image):
+            image = image_input.convert("RGB")
+        else:
+            image = Image.fromarray(image_input).convert("RGB")
+        images.append(image)
+        messages.append({
+            "role": "user",
+            "content": [{"type": "text", "text": text_input}, {"type": "image"}]
+        })
+    else:
+        messages.append({
+            "role": "user",
+            "content": [{"type": "text", "text": text_input}]
+        })
+
+    # Apply chat template
+    texts = processor.apply_chat_template(messages)
+
+    # Process inputs
+    if not images:
+        inputs = processor(text=texts, return_tensors="pt").to("cuda")
+    else:
+        inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda")
+
+    streamer = TextIteratorStreamer(
+        processor.tokenizer, skip_special_tokens=True, skip_prompt=True
+    )
+
+    generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens)
+
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
+    
+    response = ""
+    for new_text in streamer:
+        response += new_text
+        chatbot[-1][1] = response
+        yield chatbot
+
+    thread.join()
+
+    # Debug output
+    print('*'*60)
+    print('*'*60)
+    print('BOT_STREAMING_CONV_START')
+    for i, (request, answer) in enumerate(chatbot[:-1], 1):
+        print(f'Q{i}:\n {request}')
+        print(f'A{i}:\n {answer}')
+    print('New_Q:\n', text_input)
+    print('New_A:\n', response)
+    print('BOT_STREAMING_CONV_END')
+
+    
+    if IS_LOGGED_IN:
+        new_history = messages + [{"role": "assistant", "content": [{"type": "text", "text": response}]}]
+        update_user_history(USER_ID, new_history)
+
+# Define the HTML content for the header
+html = f"""
+<p align="center" style="font-size: 2.5em; line-height: 1;">
+    <span style="display: inline-block; vertical-align: middle;">🍽️</span>
+    <span style="display: inline-block; vertical-align: middle;">PixDiet</span>
+</p>
+<center><font size=3><b>PixDiet</b> is your AI nutrition expert. Upload an image of your meal and chat with our AI to get personalized advice on your diet, meal composition, and ways to improve your nutrition.</font></center>
+<div style="display: flex; justify-content: center; align-items: center; margin-top: 20px;">
+    <img src="https://static.alan.com/murray/e5830dc1cc164ef2b054023851e32206_Alan_Lockup_Horizontal_Black_RGB_Large.png" alt="Alan AI Logo" style="height: 50px; margin-right: 20px;">
+    <img src="https://seeklogo.com/images/M/mistral-ai-icon-logo-B3319DCA6B-seeklogo.com.png" alt="Mistral AI Logo" style="height: 50px;">
+</div>
+"""
+
+# Define LaTeX delimiters
+latex_delimiters_set = [
+    {"left": "\\(", "right": "\\)", "display": False},
+    {"left": "\\begin{equation}", "right": "\\end{equation}", "display": True},
+    {"left": "\\begin{align}", "right": "\\end{align}", "display": True},
+    {"left": "\\begin{alignat}", "right": "\\end{alignat}", "display": True},
+    {"left": "\\begin{gather}", "right": "\\end{gather}", "display": True},
+    {"left": "\\begin{CD}", "right": "\\end{CD}", "display": True},
+    {"left": "\\[", "right": "\\]", "display": True}
+]
+
+# Create the Gradio interface
+with gr.Blocks(title="PixDiet", theme=gr.themes.Ocean()) as demo:
+    gr.HTML(html)
+    with gr.Row():
+        with gr.Column(scale=3):
+            image_input = gr.Image(label="Upload your meal image", height=350, type="pil")
+            gr.Examples(
+                examples=[
+                    ["./examples/mistral_breakfast.jpeg", ""],
+                    ["./examples/pate_carbo.jpg", ""],
+                    ["./examples/mc_do.jpeg", ""],
+                ],
+                inputs=[image_input, gr.Textbox(visible=False)]
+            )
+        with gr.Column(scale=7):
+            chatbot = gr.Chatbot(label="Chat with PixDiet", layout="panel", height=600, show_copy_button=True, latex_delimiters=latex_delimiters_set)
+            text_input = gr.Textbox(label="Ask about your meal", placeholder="Enter your question here...", lines=1, container=False)
+            with gr.Row():
+                send_btn = gr.Button("Send", variant="primary")
+                clear_btn = gr.Button("Delete my historic", variant="secondary")
+
+    def submit_chat(chatbot, text_input):
+        response = ''
+        chatbot.append((text_input, response))
+        return chatbot, ''
+
+    def clear_chat():
+        delete_user_history(USER_ID)
+        return [], None, ""
+
+    send_click_event = send_btn.click(submit_chat, [chatbot, text_input], [chatbot, text_input]).then(
+        bot_streaming, [chatbot, image_input], chatbot
+    )
+    # submit_event = text_input.submit(submit_chat, [chatbot, text_input], [chatbot, text_input]).then(
+    #     bot_streaming, [chatbot, image_input], chatbot
+    # )
+    clear_btn.click(clear_chat, outputs=[chatbot, image_input, text_input])
+
+if __name__ == "__main__":
+    demo.launch(debug=False, share=False, show_api=False)

db_client.py

@@ -0,0 +1,31 @@
+from supabase import create_client, Client
+import os
+import json
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Initialize Supabase client
+supabase: Client = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY"))
+
+def get_user_history(user_id):
+    response = supabase.table("history").select("messages").eq("user", user_id).order("created_at", desc=True).limit(1).execute()
+    if response.data:
+        return response.data[0]['messages']
+    return []
+
+def update_user_history(user_id, new_history):
+    # Filter out messages with image type
+    filtered_history = [
+        message for message in new_history
+        if not (message["role"] == "user" and
+                isinstance(message["content"], list) and
+                any(item.get("type") == "image" for item in message["content"]))
+    ]
+    
+    # Insert the filtered history into the database
+    supabase.table("history").insert({"user": user_id, "messages": filtered_history}).execute()
+
+def delete_user_history(user_id):
+    response = supabase.table("history").delete().eq("user", user_id).execute()
+    return response.data

publish.py

@@ -0,0 +1,7 @@
+from huggingface_hub import upload_folder
+
+upload_folder(
+    folder_path="/mnt/sda/jeremy/PixDiet",
+    repo_id="eBoreal/PixDiet",
+    repo_type="space",
+)

requirements.txt

@@ -0,0 +1,12 @@
+torch>=2.0.0
+transformers==4.45.2
+bitsandbytes>=0.39.0
+accelerate>=0.20.0
+pillow
+gradio
+torchvision 
+einops 
+timm
+gradio[oauth]
+python-dotenv
+supabase

scenario.py

@@ -0,0 +1,162 @@
+
+
+renal_failure = {
+    "context": {
+        "patient": "Matthieu, 27 years old, 65kg, diagnosed with stage 3 chronic kidney disease. Requires a low-sodium, low-potassium, and low-phosphorus diet. Advised to limit fluid intake.",
+        "chat_history": [
+            {
+                "day": 1,
+                "meal": "Breakfast",
+                "image_url": "https://example.com/breakfast_day1.jpg",
+                "bot_response": "Good morning, Matthieu! I see you have oatmeal with berries and a glass of water. This is a good start to your day. The oatmeal is low in sodium and phosphorus, which is great for your kidneys. The berries add some natural sweetness and vitamins. Just be mindful of the portion size of berries, as they contain some potassium. The water is good, but remember to stick to your recommended fluid intake. Overall, this is a kidney-friendly breakfast!",
+            },
+            {
+                "day": 1,
+                "meal": "Lunch",
+                "image_url": "https://example.com/lunch_day1.jpg",
+                "bot_response": "I see a sandwich with turkey, lettuce, and tomato on whole wheat bread. Good job on trying to reduce sodium! Turkey is a lean protein, which is good. However, deli meats are often high in sodium, so try to use fresh, unseasoned turkey breast next time. The lettuce is great, but tomatoes are high in potassium. Consider replacing them with cucumber or bell peppers. The whole wheat bread is okay in moderation, but watch your portion sizes as it contains phosphorus. Don't forget to pair this with a low-potassium fruit like apple slices or grapes.",
+            },
+            {
+                "day": 2,
+                "meal": "Breakfast",
+                "image_url": "https://example.com/breakfast_day2.jpg",
+                "bot_response": "Great job, Matthieu! I see scrambled egg whites with herbs, a small portion of low-sodium toast, and some sliced apples. This is an excellent kidney-friendly breakfast. Egg whites are a great source of protein without the phosphorus found in egg yolks. The herbs add flavor without sodium. The toast in moderation is fine, and apples are a perfect low-potassium fruit choice. Remember to measure your fluid intake with that glass of water. Keep up the good work!",
+            },
+        ],
+    },
+    "message": {
+        "day": 2,
+        "meal": "Lunch",
+        "image_url": "https://example.com/lunch_day2.jpg",
+        "bot_response": """Hello Matthieu! I'm glad to see you're focusing on incorporating more vegetables. Let's analyze your lunch:
+
+1. Grilled chicken breast: Excellent choice! It's a lean protein source that's low in sodium and phosphorus.
+
+2. Steamed vegetables: I see broccoli, carrots, and green beans. These are great low-potassium options. Broccoli is especially good as it's also low in phosphorus.
+
+3. Brown rice: While brown rice is nutritious, it's higher in phosphorus and potassium than white rice. For your condition, it's better to have white rice in moderation. Consider switching to bulgur wheat or couscous as alternatives.
+
+4. Lemon wedge: Great idea! Using lemon for flavor instead of salt is a smart move for reducing sodium intake.
+
+5. Glass of water: Good to see you staying hydrated, but remember to track your fluid intake as per your doctor's recommendations.
+
+Overall, this is a much-improved meal compared to yesterday's lunch. Here are some tips for next time:
+- Replace the brown rice with a lower phosphorus grain or reduce the portion size.
+- Add some low-potassium fruits like strawberries or blueberries for dessert.
+- Consider adding herbs like basil or oregano to your vegetables for extra flavor without sodium.
+
+Keep up the great work in making these positive changes to your diet!
+""",
+    },
+}
+
+marathon_runner = {
+    "context": {
+        "patient": "Jeremie, 27 years old, 1.92m, 75kg, lactose intolerant. Training for a marathon. Wants to optimize energy levels and reduce injury chances. Has difficulty gaining weight.",
+        "chat_history": [
+            {
+                "day": 1,
+                "meal": "Breakfast",
+                "image_url": "https://example.com/jeremie_breakfast_day1.jpg",
+                "bot_response": "Good morning, Jeremie! Your breakfast of oatmeal with banana and almond milk is a great start. Aim for 600-700 calories here. Add 2 tbsp of almond butter for healthy fats and extra calories. Consider 1 scoop of vegan protein powder for muscle recovery. Don't forget 500ml of water to start your hydration routine.",
+            },
+            {
+                "day": 1,
+                "meal": "Post-run Snack",
+                "image_url": "https://example.com/jeremie_snack_day1.jpg",
+                "bot_response": "Good choice with the apple and peanut butter. Aim for a 3:1 carb to protein ratio within 30 minutes post-run. Add a slice of whole grain toast to hit about 30g of carbs. Consider 10g of branched-chain amino acids (BCAAs) to reduce muscle soreness and improve recovery.",
+            },
+            {
+                "day": 2,
+                "meal": "Lunch",
+                "image_url": "https://example.com/jeremie_lunch_day2.jpg",
+                "bot_response": "Great quinoa and vegetable bowl! For your needs, increase portion to 150g of quinoa (dry weight) and 150g of chicken. Add 1 tbsp olive oil for healthy fats. Aim for 800-900 calories here. Include a kiwi for vitamin C to aid iron absorption. Remember to drink 500ml of water with electrolytes during this meal.",
+            },
+        ],
+    },
+    "message": {
+        "day": 2,
+        "meal": "Dinner",
+        "image_url": "https://example.com/jeremie_dinner_day2.jpg",
+        "bot_response": """Hi Jeremie! Let's analyze your dinner:
+
+1. Salmon (200g): Excellent! Rich in omega-3s (about 4g) for reducing inflammation. Provides about 44g of protein.
+
+2. Sweet potato (150g): Good complex carbs, about 30g. Rich in beta-carotene for immune function.
+
+3. Broccoli (100g): Great for fiber and micronutrients. Only 6g of carbs but packed with vitamins.
+
+4. Olive oil drizzle: Good for healthy fats. Aim for 1 tbsp (14g) for about 120 calories.
+
+This meal is a good start but could use some adjustments:
+
+- Increase sweet potato to 250g for more carbs (50g total) to support your training.
+- Add 1 cup of cooked quinoa (185g) for extra 220 calories and 8g of protein.
+- Include 1 tbsp of chia seeds for extra 70 calories and 2g of omega-3s.
+
+Total calories should be around 900-1000 for this meal.
+
+Supplements to consider:
+- 1000mg of Vitamin D3 daily for bone health
+- 500mg of Vitamin C to support your immune system
+- 5g of creatine monohydrate daily for improved performance and recovery
+
+Remember to drink at least 500ml of water with this meal. Aim for a total daily fluid intake of 3-4 liters, increasing on heavy training days.
+
+Keep up the great work, Jeremie!""",
+    },
+}
+
+startup_coder = {
+    "context": {
+        "patient": "Julien, 25 years old, 70kg. Hardcore coder participating in a hackathon. Wants to optimize energy, reduce stress, and improve sleep.",
+        "chat_history": [
+            {
+                "day": 1,
+                "meal": "Breakfast",
+                "image_url": "https://example.com/julien_breakfast_day1.jpg",
+                "bot_response": "Good morning, Julien! Your breakfast of fruits, pain au raisin, and coffee is a quick start, but let's improve it. The fruits provide vitamins and fiber, which is great. However, the pain au raisin is high in sugar and low in protein. Try adding some Greek yogurt or eggs for protein to stabilize your energy. Limit coffee to 1-2 cups to avoid jitters and sleep disruption.",
+            },
+            {
+                "day": 1,
+                "meal": "Lunch",
+                "image_url": "https://example.com/julien_lunch_day1.jpg",
+                "bot_response": "Your lunch of a chicken sandwich with salad is a good base. The chicken provides protein for sustained energy, and the salad offers nutrients. However, the brownie is a sugar spike waiting to happen. Next time, opt for a piece of dark chocolate if you need something sweet. Add some whole grain bread and avocado to your sandwich for complex carbs and healthy fats.",
+            },
+            {
+                "day": 1,
+                "meal": "Afternoon Snack",
+                "image_url": "https://example.com/julien_snack_day1.jpg",
+                "bot_response": "I see you've been snacking on madeleines throughout the afternoon. While they're tasty, they're not the best for sustained energy. Try switching to nuts, seeds, or fresh fruit with nut butter for a better balance of protein, healthy fats, and complex carbs. This will help maintain your energy levels without the sugar crashes.",
+            },
+        ],
+    },
+    "message": {
+        "day": 1,
+        "meal": "Dinner",
+        "image_url": "https://example.com/julien_dinner_day1.jpg",
+        "bot_response": """Hey Julien! Let's take a look at your dinner and plan for tomorrow:
+
+1. Bowl of pasta: While pasta provides quick energy, it's not the best choice for sustained focus and good sleep. Here's how to improve it:
+   - Switch to whole grain pasta for more fiber and nutrients.
+   - Add grilled chicken or tofu for protein.
+   - Include plenty of vegetables like spinach, broccoli, and cherry tomatoes.
+   - Use olive oil and herbs instead of heavy sauces.
+
+2. For tomorrow, I suggest:
+   - Breakfast: Overnight oats with chia seeds, berries, and a scoop of protein powder. Green tea instead of coffee.
+   - Morning snack: Apple slices with almond butter.
+   - Lunch: Quinoa bowl with grilled chicken, mixed vegetables, and avocado. Add a small dark chocolate square for a treat.
+   - Afternoon snack: Greek yogurt with walnuts and a drizzle of honey.
+   - Dinner: Grilled salmon, sweet potato, and steamed broccoli. Add a small side salad with olive oil and lemon dressing.
+
+3. General tips:
+   - Stay hydrated! Aim for at least 2-3 liters of water throughout the day.
+   - Take short breaks every hour to stretch and rest your eyes.
+   - Consider supplements like Omega-3, Vitamin D, and Magnesium for brain function and stress reduction.
+   - Limit caffeine after 2 PM to improve sleep quality.
+   - Try meditation or deep breathing exercises to manage stress.
+
+Remember, good nutrition, hydration, and regular breaks are key to maintaining high performance during your hackathon. Good luck, Julien!""",
+    },
+}