adding files for app

app.py

@@ -0,0 +1,19 @@
+
+import gradio as gr
+from inference import Inference
+
+
+encoder_model_name='google/vit-large-patch32-224-in21k'
+decoder_model_name='gpt2-large'
+inference = Inference(
+  decoder_model_name=decoder_model_name,
+ 
+)
+
+def generate_text(video):
+  generated_text = inference.generate_text(video, encoder_model_name)
+
+  return generated_text
+
+app = gr.Interface(fn=generate_text, inputs='video', outputs='text')
+app.launch(share=True)

inference.py

@@ -0,0 +1,29 @@
+import torch
+from transformers import AutoTokenizer, VisionEncoderDecoderModel
+
+import utils
+
+class Inference:
+  def __init__(self, decoder_model_name, max_length=32):
+    self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    self.tokenizer = AutoTokenizer.from_pretrained(decoder_model_name)
+    self.encoder_decoder_model = VisionEncoderDecoderModel.from_pretrained('armgabrielyan/video-summarization')
+    self.encoder_decoder_model.to(self.device)
+
+    self.max_length = max_length
+
+  def generate_text(self, video, encoder_model_name):
+    if isinstance(video, str):
+      pixel_values = utils.video2image_from_path(video, encoder_model_name)
+    else:
+      pixel_values = video
+
+    if not self.tokenizer.pad_token:
+      self.tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+      self.encoder_decoder_model.decoder.resize_token_embeddings(len(self.tokenizer))
+
+    generated_ids = self.encoder_decoder_model.generate(pixel_values.unsqueeze(0).to(self.device), max_length=self.max_length)
+    generated_text = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+
+    return generated_text

utils.py

@@ -0,0 +1,42 @@
+from transformers import ViTFeatureExtractor
+import torchvision
+import torchvision.transforms.functional as fn
+import torch as th
+
+
+def video2image_from_path(video_path, feature_extractor_name):
+    video = torchvision.io.read_video(video_path)
+
+    return video2image(video[0], feature_extractor_name)
+
+
+def video2image(video, feature_extractor_name):
+    feature_extractor = ViTFeatureExtractor.from_pretrained(
+        feature_extractor_name
+    )
+
+    vid = th.permute(video, (3, 0, 1, 2))
+    samp = th.linspace(0, vid.shape[1]-1, 49, dtype=th.long)
+    vid = vid[:, samp, :, :]
+
+    im_l = list()
+    for i in range(vid.shape[1]):
+        im_l.append(vid[:, i, :, :])
+
+    inputs = feature_extractor(im_l, return_tensors="pt")
+
+    inputs = inputs['pixel_values']
+
+    im_h = list()
+    for i in range(7):
+        im_v = th.cat((inputs[0+i*7, :, :, :],
+                       inputs[1+i*7, :, :, :],
+                       inputs[2+i*7, :, :, :],
+                       inputs[3+i*7, :, :, :],
+                       inputs[4+i*7, :, :, :],
+                       inputs[5+i*7, :, :, :],
+                       inputs[6+i*7, :, :, :]), 2)
+        im_h.append(im_v)
+    resize = fn.resize(th.cat(im_h, 1), size=[224])
+
+    return resize