sliders added

app.py

@@ -3,6 +3,7 @@ import torch
 from diffusers import AnimateDiffPipeline, DDIMScheduler, MotionAdapter
 from diffusers.utils import export_to_gif
 from diffusers.utils import export_to_video
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 import uuid
 import spaces
 
@@ -22,21 +23,52 @@ device = "cuda"
 adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-2", torch_dtype=torch.float16)
 model_id = "SG161222/Realistic_Vision_V5.1_noVAE"
 
+model_llm = AutoModelForCausalLM.from_pretrained(
+    "microsoft/Phi-3-mini-128k-instruct",
+    device_map="cuda",
+    torch_dtype="auto",
+    trust_remote_code=True,
+    device="cuda"
+)
+tokenizer_llm = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct",device="cuda")
+
+pipe = AnimateDiffPipeline.from_pretrained(model_id, motion_adapter=adapter, torch_dtype=torch.float16).to(device)
+scheduler = DDIMScheduler.from_pretrained(
+    model_id,
+    subfolder="scheduler",
+    clip_sample=False,
+    timestep_spacing="linspace",
+    beta_schedule="linear",
+    steps_offset=1,
+)
+pipe.scheduler = scheduler
+
 @spaces.GPU
 def generate_video(prompt,negative_prompt, guidance_scale, num_inference_steps, adapter_choices):
-    pipe = AnimateDiffPipeline.from_pretrained(model_id, motion_adapter=adapter, torch_dtype=torch.float16).to(device)
-    scheduler = DDIMScheduler.from_pretrained(
-        model_id,
-        subfolder="scheduler",
-        clip_sample=False,
-        timestep_spacing="linspace",
-        beta_schedule="linear",
-        steps_offset=1,
-    )
-    pipe.scheduler = scheduler
 
     pipe.to(device)
 
+    messages = [
+        {"role": "user", "content": "You have to complete my given prompt into a complete description. The description should be heavily detailed. Feel free to add your own fillers if need. The purpose of this description is to descibe a video generation. My Prompt: " + prompt},
+    ]
+
+    pipe_llm = pipeline(
+        "text-generation",
+        model=model_llm,
+        tokenizer=tokenizer_llm,
+        device="cuda"
+    )
+
+    generation_args = {
+        "max_new_tokens": 500,
+        "return_full_text": False,
+        "temperature": 1,
+        "do_sample": False,
+    }
+
+    output = pipe_llm(messages, **generation_args)
+    print(output[0]['generated_text'])
+
     # Set adapters based on user selection
     if adapter_choices:
         for i in range(len(adapter_choices)):