add init

app.py

@@ -1,19 +1,22 @@
 import torch
 import gradio as gr
 from vlm_captions import VLMCaptioning
-from llm_inference_video import VideoLLMInferenceNode
 
 # Initialize the VLMCaptioning model once at startup
 print("Initializing Video Prompt Generator...")
 vlm_captioner = VLMCaptioning()
 print("Video Prompt Generator initialized successfully!")
 
+# Import VideoLLMInferenceNode after VLMCaptioning initialization
+from llm_inference_video import VideoLLMInferenceNode
+
 title = """<h1 align="center">AI Video Prompt Generator</h1>
 <p align="center">Generate creative video prompts with technical specifications</p>
 <p align="center">You can use prompts with Kling, MiniMax, Hunyuan, Haiper, CogVideoX, Luma, LTX, Runway, PixVerse. </p>"""
 
 def create_video_interface():
-    llm_node = VideoLLMInferenceNode()
+    # Pass the already initialized vlm_captioner to avoid serialization issues
+    llm_node = VideoLLMInferenceNode(vlm_captioner)
 
     with gr.Blocks(theme='bethecloud/storj_theme') as demo:
         gr.HTML(title)
@@ -108,7 +111,6 @@ def create_video_interface():
                         label="Model"
                     )
                     
-
                     generate_btn = gr.Button("Generate Video Prompt", variant="primary")
                     output = gr.Textbox(label="Generated Prompt", lines=12, show_copy_button=True)
 
@@ -149,46 +151,43 @@ def create_video_interface():
                     analyze_video_btn = gr.Button("Analyze Video")
                     video_output = gr.Textbox(label="Video Analysis", lines=10)
 
+            # Use direct function calls to avoid serialization issues
             analyze_image_btn.click(
-                llm_node.analyze_image,
+                describe_image_interface,
                 inputs=[image_input, image_question],
                 outputs=image_output
             )
 
             analyze_video_btn.click(
-                llm_node.analyze_video,
+                describe_video_interface,
                 inputs=video_input,
                 outputs=video_output
             )
 
     return demo
 
-def describe_image_interface(image, question="Describe this image in detail.", temperature=0.7, top_p=0.9, top_k=40, max_new_tokens=512):
+# Define these functions at the module level to avoid pickling issues
+def describe_image_interface(image, question="Describe this image in detail."):
     """Interface function for image description"""
     if image is None:
         return "Please upload an image."
     
+    if not question or question.strip() == "":
+        question = "Describe this image in detail."
+    
     return vlm_captioner.describe_image(
         image=image,
-        question=question,
-        temperature=temperature,
-        top_p=top_p,
-        top_k=top_k,
-        max_new_tokens=max_new_tokens
+        question=question
     )
 
-def describe_video_interface(video, frame_interval=30, temperature=0.7, top_p=0.9, top_k=40, max_new_tokens=512):
+def describe_video_interface(video, frame_interval=30):
     """Interface function for video description"""
     if video is None:
         return "Please upload a video."
     
     return vlm_captioner.describe_video(
         video_path=video,
-        frame_interval=frame_interval,
-        temperature=temperature,
-        top_p=top_p,
-        top_k=top_k,
-        max_new_tokens=max_new_tokens
+        frame_interval=frame_interval
     )
 
 if __name__ == "__main__":

llm_inference_video.py

@@ -1,175 +1,210 @@
 import os
-import random
+import time
+import requests
+from typing import Optional, Dict, Any, List
+import json
+import tempfile
+from PIL import Image
 from groq import Groq
 from openai import OpenAI
 from vlm_captions import VLMCaptioning
 
 class VideoLLMInferenceNode:
-    def __init__(self):
-        self.groq_api_key = os.getenv("GROQ_API_KEY")
-        self.sambanova_api_key = os.getenv("SAMBANOVA_API_KEY")
+    def __init__(self, vlm_captioner=None):
+        """
+        Initialize the VideoLLMInferenceNode
+        
+        Args:
+            vlm_captioner: The already initialized VLMCaptioning instance to use
+        """
+        self.vlm = vlm_captioner
+        self.sambanova_api_key = os.environ.get("SAMBANOVA_API_KEY", "")
+        self.groq_api_key = os.environ.get("GROQ_API_KEY", "")
         
         self.groq_client = Groq(api_key=self.groq_api_key)
         self.sambanova_client = OpenAI(
             api_key=self.sambanova_api_key,
             base_url="https://api.sambanova.ai/v1",
         )
-        
-        # Initialize VLM captioning
-        self.vlm = VLMCaptioning()
-
-    def analyze_image(self, image_path, question=None):
-        """Analyze image using MiniCPM-O"""
-        return self.vlm.describe_image(image_path, question)
 
-    def analyze_video(self, video_path):
-        """Analyze video using MiniCPM-O"""
-        return self.vlm.describe_video(video_path)
-
-    def generate_video_prompt(
-        self,
-        input_concept,
-        style,
-        camera_style,
-        camera_direction,
-        pacing,
-        special_effects,
-        custom_elements,
-        provider="SambaNova",
-        model=None,
-        prompt_length="Medium"
-    ):
+    def analyze_image(self, image_path: str, question: Optional[str] = None) -> str:
+        """
+        Analyze an image using the VLM model
+        
+        Args:
+            image_path: Path to the image file
+            question: Optional question to ask about the image
+            
+        Returns:
+            str: Analysis result
+        """
+        if not image_path:
+            return "Please upload an image."
+            
+        if not question or question.strip() == "":
+            question = "Describe this image in detail."
+            
         try:
-            # Helper function to format optional elements
-            def format_element(element, element_type):
-                if element == "None" or not element:
-                    return ""
-                
-                element_prefixes = {
-                    "camera": "utilizing",
-                    "direction": "with",
-                    "pacing": "with",
-                    "effects": "incorporating"
-                }
-                
-                return f" {element_prefixes.get(element_type, '')} {element}"
-
-            # Format camera movement combination
-            camera_movement = ""
-            if camera_style != "None" and camera_direction != "None":
-                camera_movement = f"{camera_style} {camera_direction}"
-            elif camera_style != "None":
-                camera_movement = camera_style
-            elif camera_direction != "None":
-                camera_movement = camera_direction
-
-            # Video prompt templates
-            default_style = "simple"  # Changed from "cinematic" to "simple" as default
+            # Use the passed vlm_captioner instance
+            return self.vlm.describe_image(image_path, question)
+        except Exception as e:
+            return f"Error analyzing image: {str(e)}"
+    
+    def analyze_video(self, video_path: str) -> str:
+        """
+        Analyze a video using the VLM model
+        
+        Args:
+            video_path: Path to the video file
             
-            prompt_templates = {
-                "minimalist": f"""Create an elegantly sparse video description focusing on {input_concept}.
-                    {format_element(camera_movement, 'camera')}
-                    {format_element(pacing, 'pacing')}
-                    {format_element(special_effects, 'effects')}
-                    {' with ' + custom_elements if custom_elements else ''}.""",
-
-                "dynamic": f"""Craft an energetic, fast-paced paragraph showcasing {input_concept} in constant motion. Utilize bold {camera_style} movements and {pacing} rhythm to create momentum. Layer {special_effects} effects and {custom_elements if custom_elements else 'powerful visual elements'} to maintain high energy throughout.""",
-
-                "simple": f"""Create a straightforward, easy-to-understand paragraph describing a video about {input_concept}. Use {camera_style} camera work and {pacing} pacing. Keep the visuals clear and uncomplicated, incorporating {special_effects} effects and {custom_elements if custom_elements else 'basic visual elements'} in an accessible way.""",
-
-                "detailed": f"""Construct a meticulous, technically precise paragraph outlining a video about {input_concept}. Incorporate specific details about {camera_style} cinematography, {pacing} timing, and {special_effects} effects. Include {custom_elements if custom_elements else 'precise technical elements'} while maintaining clarity and depth.""",
-
-                "descriptive": f"""Write a richly descriptive paragraph for a video exploring {input_concept}. Paint a vivid picture using sensory details, incorporating {camera_style} movement, {pacing} flow, and {special_effects} effects. Emphasize texture, color, and atmosphere, enhanced by {custom_elements if custom_elements else 'evocative visual elements'}.""",
-
-                "cinematic": f"""Create a single, detailed paragraph describing a cinematic video that captures {input_concept}. Focus on creating a cohesive narrative that incorporates {style} visual aesthetics, {camera_style} camera work, {pacing} pacing, and {special_effects} effects. Include atmospheric elements like {custom_elements if custom_elements else 'mood lighting and environmental details'} to enhance the storytelling. Describe the visual journey without technical timestamps or shot lists.""",
-
-                "documentary": f"""Write a comprehensive paragraph for a  documentary-style video exploring {input_concept}. Blend observational footage with {camera_style} cinematography, incorporating {pacing} editorial rhythm and {special_effects} visual treatments. Focus on creating an immersive narrative that educates and engages, enhanced by {custom_elements if custom_elements else 'authentic moments and natural lighting'}.""",
-
-                "animation": f"""Compose a vivid paragraph describing a {style} animated video showcasing {input_concept}. Detail the unique visual style, character movements, and world-building elements, incorporating {camera_style} perspectives and {pacing} story flow. Include {special_effects} animation effects and {custom_elements if custom_elements else 'signature artistic elements'} to create a memorable visual experience.""",
-
-                "action": f"""Craft an energetic paragraph describing an action sequence centered on {input_concept}. Emphasize the dynamic flow of action using {camera_style} cinematography, {pacing} rhythm, and {special_effects} visual effects. Incorporate {style} stylistic choices and {custom_elements if custom_elements else 'impactful moments'} to create an adrenaline-pumping experience.""",
-
-                "experimental": f"""Create an avant-garde paragraph describing an experimental video exploring {input_concept}. Embrace unconventional storytelling through {style} aesthetics, {camera_style} techniques, and {pacing} temporal flow. Incorporate {special_effects} digital manipulations and {custom_elements if custom_elements else 'abstract visual metaphors'} to challenge traditional narrative structures."""
-            }
-
-            # Get the template with a more neutral default
-            selected_style = style.lower()
-            if selected_style not in prompt_templates:
-                print(f"Warning: Style '{style}' not found, using '{default_style}' template")
-                selected_style = default_style
-                
-            base_prompt = prompt_templates[selected_style]
-
-            # Configure length requirements
-            length_config = {
-                "Short": {
-                    "guidance": "Create exactly very short, ONE impactful sentence that captures the essence of the video. Be concise but descriptive.",
-                    "structure": "Combine all elements into a single, powerful sentence."
-                },
-                "Medium": {
-                    "guidance": "Create 2-3 flowing sentences that paint a picture of the video.",
-                    "structure": "First sentence should set the scene, followed by 1-2 sentences developing the concept."
-                },
-                "Long": {
-                    "guidance": "Create 4-5 detailed sentences that thoroughly describe the video.",
-                    "structure": "Begin with the setting, develop the action/movement, and conclude with impact."
-                }
-            }
+        Returns:
+            str: Analysis result
+        """
+        if not video_path:
+            return "Please upload a video."
+            
+        try:
+            # Use the passed vlm_captioner instance
+            return self.vlm.describe_video(video_path)
+        except Exception as e:
+            return f"Error analyzing video: {str(e)}"
+    
+    def generate_video_prompt(
+        self,
+        concept: str,
+        style: str = "Simple",
+        camera_style: str = "None",
+        camera_direction: str = "None",
+        pacing: str = "None",
+        special_effects: str = "None",
+        custom_elements: str = "",
+        provider: str = "SambaNova",
+        model: str = "Meta-Llama-3.1-70B-Instruct",
+        prompt_length: str = "Medium"
+    ) -> str:
+        """
+        Generate a video prompt using the specified LLM provider
+        
+        Args:
+            concept: Core concept for the video
+            style: Video style
+            camera_style: Camera style
+            camera_direction: Camera direction
+            pacing: Pacing rhythm
+            special_effects: Special effects approach
+            custom_elements: Custom technical elements
+            provider: LLM provider (SambaNova or Groq)
+            model: Model name
+            prompt_length: Desired prompt length
             
-            config = length_config[prompt_length]
+        Returns:
+            str: Generated video prompt
+        """
+        if not concept:
+            return "Please enter a concept for the video."
             
-            system_message = f"""You are a visionary video director and creative storyteller. {config['guidance']}
-
-Structure: {config['structure']}
-
-Focus on these elements while maintaining the specified sentence count:
-1. Visual atmosphere and mood
-2. Camera movement and cinematography
-3. Narrative flow
-4. Style and aesthetic choices
-5. Key moments
-6. Emotional impact
-
-IMPORTANT REQUIREMENTS:
-- Deliver exactly the specified number of sentences
-- Short: ONE sentence
-- Medium: TWO to THREE sentences
-- Long: FOUR to FIVE sentences
-- If camera movements are specified, you MUST incorporate them into the description
-- Keep everything in a single paragraph format
-- Avoid technical specifications or shot lists
-- Avoid talking about 'video' or 'videos'. Do not start with 'The video opens with...' or 'The video starts with...' and do not include 'in this video' or 'focus of this video'. kind of terms"""
-
-            # Format the user prompt with style guidance and camera movement
-            user_prompt = f"""Style Guide: {selected_style.capitalize()} Style
-{prompt_templates[selected_style]}
-
-Camera Movement: {camera_movement if camera_movement else 'No specific camera movement'}
-Core Concept: {input_concept}
+        # Build the prompt
+        system_message = """You are a professional video prompt generator. Your task is to create detailed, technical, and creative video prompts based on user inputs. 
+The prompts should be suitable for text-to-video AI models and include specific technical details that match the requested style, camera movement, pacing, and effects.
+Focus on creating high-quality, cohesive prompts that could be used to generate impressive AI videos."""
+
+        # Set prompt length guidelines
+        length_guide = {
+            "Short": "Create a concise prompt of 2-3 sentences.",
+            "Medium": "Create a detailed prompt of 4-6 sentences.",
+            "Long": "Create an extensive prompt with 7-10 sentences covering all details."
+        }
+        
+        # Put together options for the prompt
+        options = []
+        if style and style != "None": 
+            options.append(f"Style: {style}")
+        if camera_style and camera_style != "None":
+            options.append(f"Camera Movement Style: {camera_style}")
+        if camera_direction and camera_direction != "None":
+            options.append(f"Camera Direction: {camera_direction}")
+        if pacing and pacing != "None":
+            options.append(f"Pacing Rhythm: {pacing}")
+        if special_effects and special_effects != "None":
+            options.append(f"Special Effects: {special_effects}")
+        if custom_elements:
+            options.append(f"Custom Elements: {custom_elements}")
+        
+        options_text = "\n".join(options)
+        
+        user_message = f"""Create a video prompt based on the following concept and specifications:
 
-Please create a {prompt_length.lower()}-length description incorporating these elements into a cohesive narrative.
-Avoid talking about 'video' or 'videos'. Do not start with 'The video opens with...' or 'The video starts with...' and do not include 'in this video' or 'focus of this video'. kind of terms"""
+CONCEPT: {concept}
 
-            # Select provider
-            if provider == "Groq":
-                client = self.groq_client
-                model = model or "llama-3.3-70b-versatile"
-            else:  # SambaNova as default
-                client = self.sambanova_client
-                model = model or "Meta-Llama-3.1-70B-Instruct"
+SPECIFICATIONS:
+{options_text}
 
-            response = client.chat.completions.create(
-                model=model,
-                messages=[
-                    {"role": "system", "content": system_message},
-                    {"role": "user", "content": user_prompt}
-                ],
-                temperature=1.2,
-                top_p=0.95,
-                seed=random.randint(0, 10000)
-            )
+{length_guide.get(prompt_length, length_guide["Medium"])}
 
-            return response.choices[0].message.content.strip()
+The prompt should be detailed and technical, specifically mentioning camera angles, movements, lighting, transitions, and other visual elements that would create an impressive AI-generated video.
+"""
 
+        # Call the appropriate API based on provider
+        try:
+            if provider == "SambaNova":
+                return self._call_sambanova_api(system_message, user_message, model)
+            elif provider == "Groq":
+                return self._call_groq_api(system_message, user_message, model)
+            else:
+                return "Unsupported provider. Please select SambaNova or Groq."
         except Exception as e:
-            return f"Error generating video prompt: {str(e)}"
+            return f"Error generating prompt: {str(e)}"
+    
+    def _call_sambanova_api(self, system_message: str, user_message: str, model: str) -> str:
+        """Call the SambaNova API for prompt generation"""
+        if not self.sambanova_api_key:
+            return "SambaNova API key not configured. Please set the SAMBANOVA_API_KEY environment variable."
+        
+        api_url = "https://api.sambanova.ai/api/v1/chat/completions"
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.sambanova_api_key}"
+        }
+        
+        payload = {
+            "model": model,
+            "messages": [
+                {"role": "system", "content": system_message},
+                {"role": "user", "content": user_message}
+            ]
+        }
+        
+        response = requests.post(api_url, headers=headers, json=payload)
+        
+        if response.status_code == 200:
+            result = response.json()
+            return result.get("choices", [{}])[0].get("message", {}).get("content", "No content returned")
+        else:
+            return f"Error from SambaNova API: {response.status_code} - {response.text}"
+    
+    def _call_groq_api(self, system_message: str, user_message: str, model: str) -> str:
+        """Call the Groq API for prompt generation"""
+        if not self.groq_api_key:
+            return "Groq API key not configured. Please set the GROQ_API_KEY environment variable."
+        
+        api_url = "https://api.groq.com/openai/v1/chat/completions"
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.groq_api_key}"
+        }
+        
+        payload = {
+            "model": model,
+            "messages": [
+                {"role": "system", "content": system_message},
+                {"role": "user", "content": user_message}
+            ]
+        }
+        
+        response = requests.post(api_url, headers=headers, json=payload)
+        
+        if response.status_code == 200:
+            result = response.json()
+            return result.get("choices", [{}])[0].get("message", {}).get("content", "No content returned")
+        else:
+            return f"Error from Groq API: {response.status_code} - {response.text}"