chroma

app.py

@@ -1,222 +1,53 @@
+import os
 import torch
 import gradio as gr
-import spaces
+from diffusers import FluxTransformer2DModel, FluxPipeline, BitsAndBytesConfig
+from transformers import T5EncoderModel, BitsAndBytesConfig as BitsAndBytesConfigTF
 
-# Create Gradio UI without loading models first
-title = """<h1 align="center">AI Video Prompt Generator</h1>
-<p align="center">Generate creative video prompts with technical specifications</p>
-<p align="center">You can use prompts with Kling, MiniMax, Hunyuan, Haiper, CogVideoX, Luma, LTX, Runway, PixVerse. </p>"""
-
-# Import these at global scope but don't instantiate yet
-from vlm_captions import VLMCaptioning
-from llm_inference_video import VideoLLMInferenceNode
-
-# Global singleton instances - we'll initialize them only when needed
-vlm_captioner = None
-llm_node = None
-
-# Initialize only once on first use
-def get_vlm_captioner():
-    global vlm_captioner
-    if vlm_captioner is None:
-        print("Initializing Video Prompt Generator...")
-        vlm_captioner = VLMCaptioning()
-        print("Video Prompt Generator initialized successfully!")
-    return vlm_captioner
-
-def get_llm_node():
-    global llm_node
-    if llm_node is None:
-        llm_node = VideoLLMInferenceNode()
-    return llm_node
-
-# Wrapper functions that avoid passing the model between processes
-@spaces.GPU()
-def describe_image_wrapper(image, question="Describe this image in detail."):
-    """GPU-decorated function for image description"""
-    if image is None:
-        return "Please upload an image."
+def generate_image(prompt, negative_prompt="", num_inference_steps=30, guidance_scale=7.5):
+    # Initialize Flux pipeline
+    device = "cuda" if torch.cuda.is_available() else "cpu"
     
-    if not question or question.strip() == "":
-        question = "Describe this image in detail."
+    dtype = torch.bfloat16
+    single_file_base_model = "camenduru/FLUX.1-dev-diffusers"
+    file_url = "https://huggingface.co/lodestones/Chroma/resolve/main/chroma-unlocked-v31.safetensors"
     
-    # Get the captioner inside this GPU-decorated function
-    vlm = get_vlm_captioner()
-    return vlm.describe_image(image=image, question=question)
-
-@spaces.GPU()
-def describe_video_wrapper(video, frame_interval=30):
-    """GPU-decorated function for video description"""
-    if video is None:
-        return "Please upload a video."
+    quantization_config_tf = BitsAndBytesConfigTF(load_in_8bit=True, bnb_8bit_compute_dtype=torch.bfloat16)
+    text_encoder_2 = T5EncoderModel.from_pretrained(single_file_base_model, subfolder="text_encoder_2", torch_dtype=dtype, config=single_file_base_model, quantization_config=quantization_config_tf)
     
-    # Get the captioner inside this GPU-decorated function
-    vlm = get_vlm_captioner()
-    return vlm.describe_video(video_path=video, frame_interval=frame_interval)
-
-def generate_video_prompt_wrapper(
-    concept, style, camera_style, camera_direction, 
-    pacing, special_effects, custom_elements, 
-    provider, model, prompt_length
-):
-    """Wrapper for LLM prompt generation"""
-    node = get_llm_node()
-    return node.generate_video_prompt(
-        concept, style, camera_style, camera_direction,
-        pacing, special_effects, custom_elements,
-        provider, model, prompt_length
-    )
-
-def create_video_interface():
-    with gr.Blocks(theme='bethecloud/storj_theme') as demo:
-        gr.HTML(title)
-        
-        with gr.Tab("Video Prompt Generator"):
-            with gr.Row():
-                with gr.Column(scale=1):
-                    input_concept = gr.Textbox(label="Core Concept/Thematic Input", lines=3)
-                    style = gr.Dropdown(
-                        choices=["Minimalist", "Simple", "Detailed", "Descriptive", "Dynamic", 
-                                "Cinematic", "Documentary", "Animation", "Action", "Experimental"],
-                        value="Simple",
-                        label="Video Style"
-                    )
-                    custom_elements = gr.Textbox(label="Custom Technical Elements", 
-                                               placeholder="e.g., Infrared hybrid, Datamosh transitions")
-                    prompt_length = gr.Dropdown(
-                        choices=["Short", "Medium", "Long"],
-                        value="Medium",
-                        label="Prompt Length"
-                    ) 
-                    
-                with gr.Column(scale=1):
-                    camera_direction = gr.Dropdown(
-                        choices=[
-                            "None",
-                            "Zoom in", "Zoom out", "Pan left", "Pan right",
-                            "Tilt up", "Tilt down", "Orbital rotation",
-                            "Push in", "Pull out", "Track forward", "Track backward",
-                            "Spiral in", "Spiral out", "Arc movement",
-                            "Diagonal traverse", "Vertical rise", "Vertical descent"
-                        ],
-                        value="None",
-                        label="Camera Direction"
-                    )
-
-                    camera_style = gr.Dropdown(
-                        choices=[
-                            "None",
-                            "Steadicam flow", "Drone aerials", "Handheld urgency", "Crane elegance",
-                            "Dolly precision", "VR 360", "Multi-angle rig", "Static tripod",
-                            "Gimbal smoothness", "Slider motion", "Jib sweep", "POV immersion",
-                            "Time-slice array", "Macro extreme", "Tilt-shift miniature",
-                            "Snorricam character", "Whip pan dynamics", "Dutch angle tension",
-                            "Underwater housing", "Periscope lens"
-                        ],
-                        value="None",
-                        label="Camera Movement Style"
-                    )
-
-                    pacing = gr.Dropdown(
-                        choices=[
-                            "None",
-                            "Slow burn", "Rhythmic pulse", "Frantic energy", "Ebb and flow",
-                            "Hypnotic drift", "Time-lapse rush", "Stop-motion staccato",
-                            "Gradual build", "Quick cut rhythm", "Long take meditation",
-                            "Jump cut energy", "Match cut flow", "Cross-dissolve dreamscape",
-                            "Parallel action", "Slow motion impact", "Ramping dynamics",
-                            "Montage tempo", "Continuous flow", "Episodic breaks"
-                        ],
-                        value="None",
-                        label="Pacing Rhythm"
-                    )
-                    special_effects = gr.Dropdown(
-                        choices=[
-                            "None",
-                            "Practical effects", "CGI enhancement", "Analog glitches",
-                            "Light painting", "Projection mapping", "Nanosecond exposures",
-                            "Double exposure", "Smoke diffusion", "Lens flare artistry",
-                            "Particle systems", "Holographic overlay", "Chromatic aberration",
-                            "Digital distortion", "Wire removal", "Motion capture",
-                            "Miniature integration", "Weather simulation", "Color grading",
-                            "Mixed media composite", "Neural style transfer"
-                        ],
-                        value="None",
-                        label="SFX Approach"
-                    )
-                    
-                with gr.Column(scale=1):
-                    provider = gr.Dropdown(
-                        choices=["SambaNova", "Groq"],
-                        value="SambaNova",
-                        label="LLM Provider"
-                    )
-                    model = gr.Dropdown(
-                        choices=[
-                            "Meta-Llama-3.1-70B-Instruct",
-                            "Meta-Llama-3.1-405B-Instruct",
-                            "Meta-Llama-3.1-8B-Instruct"
-                        ],
-                        value="Meta-Llama-3.1-70B-Instruct",
-                        label="Model"
-                    )
-                    
-                    generate_btn = gr.Button("Generate Video Prompt", variant="primary")
-                    output = gr.Textbox(label="Generated Prompt", lines=12, show_copy_button=True)
-
-            def update_models(provider):
-                models = {
-                    "Groq": ["llama-3.3-70b-versatile"],
-                    "SambaNova": [
-                        "Meta-Llama-3.1-70B-Instruct",
-                        "Meta-Llama-3.1-405B-Instruct",
-                        "Meta-Llama-3.1-8B-Instruct"
-                    ]
-                }
-                return gr.Dropdown(choices=models[provider], value=models[provider][0])
-
-            provider.change(update_models, inputs=provider, outputs=model)
-
-            generate_btn.click(
-                generate_video_prompt_wrapper,
-                inputs=[input_concept, style, camera_style, camera_direction, pacing, special_effects, 
-                       custom_elements, provider, model, prompt_length],
-                outputs=output
-            )
-
-        with gr.Tab("Visual Analysis"):
-            with gr.Row():
-                with gr.Column():
-                    image_input = gr.Image(label="Upload Image", type="filepath")
-                    image_question = gr.Textbox(
-                        label="Question (optional)", 
-                        placeholder="What is in this image?"
-                    )
-
-                    analyze_image_btn = gr.Button("Analyze Image")
-                    image_output = gr.Textbox(label="Analysis Result", lines=5)
-
-                with gr.Column():
-                    video_input = gr.Video(label="Upload Video")
-                    analyze_video_btn = gr.Button("Analyze Video")
-                    video_output = gr.Textbox(label="Video Analysis", lines=10)
-
-            # Use GPU-decorated wrapper functions directly
-            analyze_image_btn.click(
-                describe_image_wrapper,
-                inputs=[image_input, image_question],
-                outputs=image_output
-            )
-
-            analyze_video_btn.click(
-                describe_video_wrapper,
-                inputs=video_input,
-                outputs=video_output
-            )
-
-    return demo
+    quantization_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16)
+    transformer = FluxTransformer2DModel.from_single_file(file_url, subfolder="transformer", torch_dtype=dtype, config=single_file_base_model, quantization_config=quantization_config)
+    
+    flux_pipeline = FluxPipeline.from_pretrained(single_file_base_model, transformer=transformer, text_encoder_2=text_encoder_2, torch_dtype=dtype)
+    flux_pipeline.to(device)
+    
+    # Generate image
+    image = flux_pipeline(
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_steps,
+        guidance_scale=guidance_scale
+    ).images[0]
+    
+    return image
+
+# Create Gradio interface
+iface = gr.Interface(
+    fn=generate_image,
+    inputs=[
+        gr.Textbox(label="Prompt", placeholder="Enter your prompt here..."),
+        gr.Textbox(label="Negative Prompt", placeholder="Enter negative prompt here...", value=""),
+        gr.Slider(minimum=1, maximum=100, value=30, step=1, label="Number of Inference Steps"),
+        gr.Slider(minimum=1.0, maximum=20.0, value=7.5, step=0.1, label="Guidance Scale")
+    ],
+    outputs=gr.Image(label="Generated Image"),
+    title="Chroma Image Generator",
+    description="Generate images using the Chroma model with FLUX pipeline",
+    examples=[
+        ["A beautiful sunset over mountains, photorealistic, 8k", "blurry, low quality, distorted", 30, 7.5],
+        ["A futuristic cityscape at night, neon lights, cyberpunk style", "ugly, deformed, low resolution", 30, 7.5]
+    ]
+)
 
 if __name__ == "__main__":
-    demo = create_video_interface()
-    # Don't use share=True on Hugging Face Spaces
-    demo.launch()
+    iface.launch()

llm_inference_video.py

@@ -1,351 +0,0 @@
-import os
-import time
-import requests
-from typing import Optional, Dict, Any, List
-import json
-import tempfile
-from PIL import Image
-from groq import Groq
-from openai import OpenAI
-import spaces
-
-class VideoLLMInferenceNode:
-    def __init__(self):
-        """
-        Initialize the VideoLLMInferenceNode without VLM captioning dependency
-        """
-        self.sambanova_api_key = os.environ.get("SAMBANOVA_API_KEY", "")
-        self.groq_api_key = os.environ.get("GROQ_API_KEY", "")
-        
-        # Initialize API clients if keys are available
-        if self.groq_api_key:
-            self.groq_client = Groq(api_key=self.groq_api_key)
-        else:
-            self.groq_client = None
-            
-        if self.sambanova_api_key:
-            self.sambanova_client = OpenAI(
-                api_key=self.sambanova_api_key,
-                base_url="https://api.sambanova.ai/v1",
-            )
-        else:
-            self.sambanova_client = None
-
-    @spaces.GPU()
-    def analyze_image(self, image_path: str, question: Optional[str] = None) -> str:
-        """
-        Analyze an image using VLM model directly
-        
-        Args:
-            image_path: Path to the image file
-            question: Optional question to ask about the image
-            
-        Returns:
-            str: Analysis result
-        """
-        if not image_path:
-            return "Please upload an image."
-            
-        if not question or question.strip() == "":
-            question = "Describe this image in detail."
-            
-        try:
-            # Import and use VLMCaptioning within this GPU-scoped function
-            from app import get_vlm_captioner
-            vlm = get_vlm_captioner()
-            return vlm.describe_image(image_path, question)
-        except Exception as e:
-            return f"Error analyzing image: {str(e)}"
-    
-    @spaces.GPU()
-    def analyze_video(self, video_path: str) -> str:
-        """
-        Analyze a video using VLM model directly
-        
-        Args:
-            video_path: Path to the video file
-            
-        Returns:
-            str: Analysis result
-        """
-        if not video_path:
-            return "Please upload a video."
-            
-        try:
-            # Import and use VLMCaptioning within this GPU-scoped function
-            from app import get_vlm_captioner
-            vlm = get_vlm_captioner()
-            return vlm.describe_video(video_path)
-        except Exception as e:
-            return f"Error analyzing video: {str(e)}"
-    
-    def generate_video_prompt(
-        self,
-        concept: str,
-        style: str = "Simple",
-        camera_style: str = "None",
-        camera_direction: str = "None",
-        pacing: str = "None",
-        special_effects: str = "None",
-        custom_elements: str = "",
-        provider: str = "SambaNova",
-        model: str = "Meta-Llama-3.1-70B-Instruct",
-        prompt_length: str = "Medium",
-        image_path: str = "",
-        video_path: str = ""
-    ) -> str:
-        """
-        Generate a video prompt using the specified LLM provider
-        
-        Args:
-            concept: Core concept for the video
-            style: Video style
-            camera_style: Camera style
-            camera_direction: Camera direction
-            pacing: Pacing rhythm
-            special_effects: Special effects approach
-            custom_elements: Custom technical elements
-            provider: LLM provider (SambaNova or Groq)
-            model: Model name
-            prompt_length: Desired prompt length
-            image_path: Optional path to an image for VLM description
-            video_path: Optional path to a video for VLM description
-            
-        Returns:
-            str: Generated video prompt
-        """
-        if not concept:
-            return "Please enter a concept for the video."
-            
-        try:
-            # Get VLM descriptions if image or video paths are provided
-            image_description = ""
-            video_description = ""
-            
-            if image_path:
-                try:
-                    image_description = self.analyze_image(image_path, "Describe this image in detail for a video creator.")
-                    print(f"Generated image description: {image_description}")
-                except Exception as e:
-                    print(f"Error generating image description: {str(e)}")
-            
-            if video_path:
-                try:
-                    video_description = self.analyze_video(video_path)
-                    print(f"Generated video description: {video_description}")
-                except Exception as e:
-                    print(f"Error generating video description: {str(e)}")
-            
-            # Helper function to format optional elements
-            def format_element(element, element_type):
-                if element == "None" or not element:
-                    return ""
-                
-                element_prefixes = {
-                    "camera": "utilizing",
-                    "direction": "with",
-                    "pacing": "with",
-                    "effects": "incorporating"
-                }
-                
-                return f" {element_prefixes.get(element_type, '')} {element}"
-
-            # Format camera movement combination
-            camera_movement = ""
-            if camera_style != "None" and camera_direction != "None":
-                camera_movement = f"{camera_style} {camera_direction}"
-            elif camera_style != "None":
-                camera_movement = camera_style
-            elif camera_direction != "None":
-                camera_movement = camera_direction
-
-            # Video prompt templates
-            default_style = "simple"  # Changed from "cinematic" to "simple" as default
-            
-            prompt_templates = {
-                "minimalist": f"""Create an elegantly sparse video description focusing on {concept}.
-                    {format_element(camera_movement, 'camera')}
-                    {format_element(pacing, 'pacing')}
-                    {format_element(special_effects, 'effects')}
-                    {' with ' + custom_elements if custom_elements else ''}.""",
-
-                "dynamic": f"""Craft an energetic, fast-paced paragraph showcasing {concept} in constant motion. Utilize bold {camera_style} movements and {pacing} rhythm to create momentum. Layer {special_effects} effects and {custom_elements if custom_elements else 'powerful visual elements'} to maintain high energy throughout.""",
-
-                "simple": f"""Create a straightforward, easy-to-understand paragraph describing a video about {concept}. Use {camera_style} camera work and {pacing} pacing. Keep the visuals clear and uncomplicated, incorporating {special_effects} effects and {custom_elements if custom_elements else 'basic visual elements'} in an accessible way.""",
-
-                "detailed": f"""Construct a meticulous, technically precise paragraph outlining a video about {concept}. Incorporate specific details about {camera_style} cinematography, {pacing} timing, and {special_effects} effects. Include {custom_elements if custom_elements else 'precise technical elements'} while maintaining clarity and depth.""",
-
-                "descriptive": f"""Write a richly descriptive paragraph for a video exploring {concept}. Paint a vivid picture using sensory details, incorporating {camera_style} movement, {pacing} flow, and {special_effects} effects. Emphasize texture, color, and atmosphere, enhanced by {custom_elements if custom_elements else 'evocative visual elements'}.""",
-
-                "cinematic": f"""Create a single, detailed paragraph describing a cinematic video that captures {concept}. Focus on creating a cohesive narrative that incorporates {style} visual aesthetics, {camera_style} camera work, {pacing} pacing, and {special_effects} effects. Include atmospheric elements like {custom_elements if custom_elements else 'mood lighting and environmental details'} to enhance the storytelling. Describe the visual journey without technical timestamps or shot lists.""",
-
-                "documentary": f"""Write a comprehensive paragraph for a documentary-style video exploring {concept}. Blend observational footage with {camera_style} cinematography, incorporating {pacing} editorial rhythm and {special_effects} visual treatments. Focus on creating an immersive narrative that educates and engages, enhanced by {custom_elements if custom_elements else 'authentic moments and natural lighting'}.""",
-
-                "animation": f"""Compose a vivid paragraph describing a {style} animated video showcasing {concept}. Detail the unique visual style, character movements, and world-building elements, incorporating {camera_style} perspectives and {pacing} story flow. Include {special_effects} animation effects and {custom_elements if custom_elements else 'signature artistic elements'} to create a memorable visual experience.""",
-
-                "action": f"""Craft an energetic paragraph describing an action sequence centered on {concept}. Emphasize the dynamic flow of action using {camera_style} cinematography, {pacing} rhythm, and {special_effects} visual effects. Incorporate {style} stylistic choices and {custom_elements if custom_elements else 'impactful moments'} to create an adrenaline-pumping experience.""",
-
-                "experimental": f"""Create an avant-garde paragraph describing an experimental video exploring {concept}. Embrace unconventional storytelling through {style} aesthetics, {camera_style} techniques, and {pacing} temporal flow. Incorporate {special_effects} digital manipulations and {custom_elements if custom_elements else 'abstract visual metaphors'} to challenge traditional narrative structures."""
-            }
-
-            # Get the template with a more neutral default
-            selected_style = style.lower()
-            if selected_style not in prompt_templates:
-                print(f"Warning: Style '{style}' not found, using '{default_style}' template")
-                selected_style = default_style
-                
-            base_prompt = prompt_templates[selected_style]
-
-            # Configure length requirements
-            length_config = {
-                "Short": {
-                    "guidance": "Create exactly very short, ONE impactful sentence that captures the essence of the video. Be concise but descriptive.",
-                    "structure": "Combine all elements into a single, powerful sentence."
-                },
-                "Medium": {
-                    "guidance": "Create 2-3 flowing sentences that paint a picture of the video.",
-                    "structure": "First sentence should set the scene, followed by 1-2 sentences developing the concept."
-                },
-                "Long": {
-                    "guidance": "Create 4-5 detailed sentences that thoroughly describe the video.",
-                    "structure": "Begin with the setting, develop the action/movement, and conclude with impact."
-                }
-            }
-            
-            config = length_config[prompt_length]
-
-            system_message = f"""You are a visionary video director and creative storyteller. {config['guidance']}
-
-Structure: {config['structure']}
-
-Focus on these elements while maintaining the specified sentence count:
-1. Visual atmosphere and mood
-2. Camera movement and cinematography
-3. Narrative flow
-4. Style and aesthetic choices
-5. Key moments
-6. Emotional impact
-{'' if not image_description and not video_description else '7. Elements from the provided image/video descriptions'}
-
-{'' if not image_description and not video_description else 'If image or video descriptions are provided, incorporate their key visual elements and content into your description to ensure accuracy and relevance.'}
-
-IMPORTANT REQUIREMENTS:
-- Deliver exactly the specified number of sentences
-- Short: ONE sentence
-- Medium: TWO to THREE sentences
-- Long: FOUR to FIVE sentences
-- If camera movements are specified, you MUST incorporate them into the description
-- Keep everything in a single paragraph format
-- Avoid technical specifications or shot lists
-- Avoid talking about 'video' or 'videos'. Do not start with 'The video opens with...' or 'The video starts with...' and do not include 'in this video' or 'focus of this video'. kind of terms"""
-
-            # Format the user prompt with style guidance and camera movement
-            user_message = f"""Style Guide: {selected_style.capitalize()} Style
-{prompt_templates[selected_style]}
-
-Camera Movement: {camera_movement if camera_movement else 'No specific camera movement'}
-Core Concept: {concept}
-{f'Reference Image Description: {image_description}' if image_description else ''}
-{f'Reference Video Description: {video_description}' if video_description else ''}
-
-Please create a {prompt_length.lower()}-length description incorporating these elements into a cohesive narrative.
-{'' if not image_description and not video_description else 'Use the provided image/video descriptions as reference to inform your prompt creation.'}
-Avoid talking about 'video' or 'videos'. Do not start with 'The video opens with...' or 'The video starts with...' and do not include 'in this video' or 'focus of this video'. kind of terms. Do not say "Here is your video prompt" or "Here is your video description" or anything like that. Just give the prompt."""
-
-            # Call the appropriate API based on provider
-            if provider == "SambaNova":
-                if self.sambanova_client:
-                    return self._call_sambanova_client(system_message, user_message, model)
-                else:
-                    return self._call_sambanova_api(system_message, user_message, model)
-            elif provider == "Groq":
-                if self.groq_client:
-                    return self._call_groq_client(system_message, user_message, model) 
-                else:
-                    return self._call_groq_api(system_message, user_message, model)
-            else:
-                return "Unsupported provider. Please select SambaNova or Groq."
-        except Exception as e:
-            return f"Error generating prompt: {str(e)}"
-    
-    def _call_sambanova_client(self, system_message: str, user_message: str, model: str) -> str:
-        """Call the SambaNova API using the client library"""
-        try:
-            chat_completion = self.sambanova_client.chat.completions.create(
-                model=model,
-                messages=[
-                    {"role": "system", "content": system_message},
-                    {"role": "user", "content": user_message}
-                ]
-            )
-            return chat_completion.choices[0].message.content
-        except Exception as e:
-            return f"Error from SambaNova API: {str(e)}"
-    
-    def _call_sambanova_api(self, system_message: str, user_message: str, model: str) -> str:
-        """Call the SambaNova API using direct HTTP requests"""
-        if not self.sambanova_api_key:
-            return "SambaNova API key not configured. Please set the SAMBANOVA_API_KEY environment variable."
-        
-        api_url = "https://api.sambanova.ai/api/v1/chat/completions"
-        headers = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer {self.sambanova_api_key}"
-        }
-        
-        payload = {
-            "model": model,
-            "messages": [
-                {"role": "system", "content": system_message},
-                {"role": "user", "content": user_message}
-            ]
-        }
-        
-        response = requests.post(api_url, headers=headers, json=payload)
-        
-        if response.status_code == 200:
-            result = response.json()
-            return result.get("choices", [{}])[0].get("message", {}).get("content", "No content returned")
-        else:
-            return f"Error from SambaNova API: {response.status_code} - {response.text}"
-    
-    def _call_groq_client(self, system_message: str, user_message: str, model: str) -> str:
-        """Call the Groq API using the client library"""
-        try:
-            chat_completion = self.groq_client.chat.completions.create(
-                model=model,
-                messages=[
-                    {"role": "system", "content": system_message},
-                    {"role": "user", "content": user_message}
-                ]
-            )
-            return chat_completion.choices[0].message.content
-        except Exception as e:
-            return f"Error from Groq API: {str(e)}"
-    
-    def _call_groq_api(self, system_message: str, user_message: str, model: str) -> str:
-        """Call the Groq API using direct HTTP requests"""
-        if not self.groq_api_key:
-            return "Groq API key not configured. Please set the GROQ_API_KEY environment variable."
-        
-        api_url = "https://api.groq.com/openai/v1/chat/completions"
-        headers = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer {self.groq_api_key}"
-        }
-        
-        payload = {
-            "model": model,
-            "messages": [
-                {"role": "system", "content": system_message},
-                {"role": "user", "content": user_message}
-            ]
-        }
-        
-        response = requests.post(api_url, headers=headers, json=payload)
-        
-        if response.status_code == 200:
-            result = response.json()
-            return result.get("choices", [{}])[0].get("message", {}).get("content", "No content returned")
-        else:
-            return f"Error from Groq API: {response.status_code} - {response.text}"

requirements.txt

@@ -1,17 +1,8 @@
-openai
-groq
-numpy==1.26.4
-Pillow==10.1.0
-torch==2.1.2
-torchaudio==2.1.2
-torchvision==0.16.2
-transformers==4.44.2
-sentencepiece==0.1.99
-https://github.com/Dao-AILab/flash-attention/releases/download/v2.6.2/flash_attn-2.6.2+cu123torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
-decord
-librosa==0.9.0
-soundfile==0.12.1
-vector-quantize-pytorch==1.18.5
-vocos==0.1.0
-moviepy
-gradio
+accelerate
+git+https://github.com/huggingface/diffusers.git
+torch
+gradio
+transformers
+xformers
+sentencepiece
+peft

vlm_captions.py

@@ -1,129 +0,0 @@
-import torch
-from PIL import Image
-from transformers import AutoModel, AutoTokenizer
-from decord import VideoReader, cpu
-import spaces
-
-class VLMCaptioning:
-    def __init__(self):
-        print("Loading MiniCPM-O model...")
-        self.model = AutoModel.from_pretrained(
-            'openbmb/MiniCPM-o-2_6', 
-            trust_remote_code=True,
-            attn_implementation='sdpa', 
-            torch_dtype=torch.bfloat16,
-            init_vision=True,
-        )
-        self.model = self.model.eval().cuda()
-        self.tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-o-2_6', trust_remote_code=True)
-
-    @spaces.GPU()
-    def describe_image(
-        self,
-        image: str,
-        question: str = "Describe this image in detail.",
-        temperature: float = 0.7,
-        top_p: float = 0.9,
-        top_k: int = 40,
-        max_new_tokens: int = 512,
-        stream=False,
-        sampling=False
-    ) -> str:
-        """
-        Generate description for a single image
-
-        Args:
-            image (str): Path to image file
-            question (str): Question to ask about the image
-            temperature (float): Sampling temperature
-            top_p (float): Nucleus sampling parameter
-            top_k (int): Top-k sampling parameter
-            max_new_tokens (int): Maximum new tokens to generate
-
-        Returns:
-            str: Generated description
-        """
-        try:
-            if not image:
-                return "Please provide an image."
-
-            # Convert image to RGB
-            image = Image.open(image).convert('RGB')
-            
-            # Prepare message
-            msgs = [{'role': 'user', 'content': [image, question]}]
-            
-            # Generate response
-            response = self.model.chat(
-                image=None,
-                msgs=msgs,
-                tokenizer=self.tokenizer,
-                temperature=temperature,
-                top_p=top_p,
-                top_k=top_k,
-                max_new_tokens=max_new_tokens,
-                stream=stream,
-                sampling=sampling
-            )
-            return response
-        except Exception as e:
-            return f"Error analyzing image: {str(e)}"
-
-    @spaces.GPU()
-    def describe_video(
-        self,
-        video_path: str,
-        frame_interval: int = 30,
-        temperature: float = 0.7,
-        top_p: float = 0.9,
-        top_k: int = 40,
-        max_new_tokens: int = 512,
-        stream=False,
-        sampling=False
-    ) -> str:
-        """
-        Generate description for video frames
-
-        Args:
-            video_path (str): Path to video file
-            frame_interval (int): Interval between frames to analyze
-            temperature (float): Sampling temperature
-            top_p (float): Nucleus sampling parameter
-            top_k (int): Top-k sampling parameter
-            max_new_tokens (int): Maximum new tokens to generate
-
-        Returns:
-            str: Generated description
-        """
-        try:
-            # Load video and extract frames
-            vr = VideoReader(video_path, ctx=cpu(0))
-            total_frames = len(vr)
-            frame_indices = list(range(0, total_frames, frame_interval))
-            frames = vr.get_batch(frame_indices).asnumpy()
-
-            # Convert frames to PIL Images
-            frame_images = [Image.fromarray(frame) for frame in frames]
-            
-            # Prepare messages for all frames
-            msgs = [
-                {'role': 'user', 'content': [frame, "Describe the main action in this scene."]}
-                for frame in frame_images
-            ]
-            
-            # Generate response for all frames at once
-            response = self.model.chat(
-                image=None,
-                msgs=msgs,
-                tokenizer=self.tokenizer,
-                temperature=temperature,
-                top_p=top_p,
-                top_k=top_k,
-                max_new_tokens=max_new_tokens,
-                stream=stream,
-                sampling=sampling
-            )
-            return response
-            
-        except Exception as e:
-            return f"Error processing video: {str(e)}"