Duplicate from Amrrs/image-caption-with-vit-gpt2

Co-authored-by: amrrs <[email protected]>

.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,47 @@
+---
+title: Image Caption With Vit Gpt2
+emoji: 👀
+colorFrom: pink
+colorTo: pink
+sdk: gradio
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: Amrrs/image-caption-with-vit-gpt2
+---
+
+# Configuration
+
+`title`: _string_
+Display title for the Space
+
+`emoji`: _string_
+Space emoji (emoji-only character allowed)
+
+`colorFrom`: _string_
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`colorTo`: _string_
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`sdk`: _string_
+Can be either `gradio`, `streamlit`, or `static`
+
+`sdk_version` : _string_
+Only applicable for `streamlit` SDK.  
+See [doc](https://hf.co/docs/hub/spaces) for more info on supported versions.
+
+`app_file`: _string_
+Path to your main application file (which contains either `gradio` or `streamlit` Python code, or `static` html code).
+Path is relative to the root of the repository.
+
+`models`: _List[string]_
+HF model IDs (like "gpt2" or "deepset/roberta-base-squad2") used in the Space.
+Will be parsed automatically from your code if not specified here.
+
+`datasets`: _List[string]_
+HF dataset IDs (like "common_voice" or "oscar-corpus/OSCAR-2109") used in the Space.
+Will be parsed automatically from your code if not specified here.
+
+`pinned`: _boolean_
+Whether the Space stays on top of your list.

app.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""Image Captioning with ViT+GPT2
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1P3O0gO5AUqSmM8rE9dxy2tXJ-9jkhxHz
+"""
+
+#! pip install transformers -q
+
+#! pip install gradio -q
+
+from PIL import Image
+from transformers import VisionEncoderDecoderModel, ViTFeatureExtractor, PreTrainedTokenizerFast
+import requests
+
+model = VisionEncoderDecoderModel.from_pretrained("sachin/vit2distilgpt2")
+
+vit_feature_extractor = ViTFeatureExtractor.from_pretrained("google/vit-base-patch16-224-in21k")
+
+tokenizer = PreTrainedTokenizerFast.from_pretrained("distilgpt2")
+
+# url = 'https://d2gp644kobdlm6.cloudfront.net/wp-content/uploads/2016/06/bigstock-Shocked-and-surprised-boy-on-t-113798588-300x212.jpg'
+
+# with Image.open(requests.get(url, stream=True).raw) as img:
+#     pixel_values = vit_feature_extractor(images=img, return_tensors="pt").pixel_values
+
+#encoder_outputs = model.generate(pixel_values.to('cpu'),num_beams=5)
+
+#generated_sentences = tokenizer.batch_decode(encoder_outputs, skip_special_tokens=True)
+
+#generated_sentences
+
+#naive text processing 
+#generated_sentences[0].split('.')[0]
+
+# inference function
+
+def vit2distilgpt2(img):
+  pixel_values = vit_feature_extractor(images=img, return_tensors="pt").pixel_values
+  encoder_outputs = generated_ids = model.generate(pixel_values.to('cpu'),num_beams=5)
+  generated_sentences = tokenizer.batch_decode(encoder_outputs, skip_special_tokens=True)
+
+  return(generated_sentences[0].split('.')[0])
+
+#!wget https://media.glamour.com/photos/5f171c4fd35176eaedb36823/master/w_2560%2Cc_limit/bike.jpg
+
+import gradio as gr
+
+inputs = [
+    gr.inputs.Image(type="pil", label="Original Image")
+]
+
+outputs = [
+    gr.outputs.Textbox(label = 'Caption')
+]
+
+title = "Image Captioning using ViT + GPT2"
+description = "ViT and GPT2 are used to generate Image Caption for the uploaded image. COCO Dataset was used for training. This image captioning model might have some biases that we couldn't figure during our stress testing, so if you find any bias (gender, race and so on) please use `Flag` button to flag the image with bias"
+article = " <a href='https://huggingface.co/sachin/vit2distilgpt2'>Model Repo on Hugging Face Model Hub</a>"
+examples = [
+    ["people-walking-street-pedestrian-crossing-traffic-light-city.jpeg"],
+    ["elonmusk.jpeg"]
+
+]
+
+gr.Interface(
+    vit2distilgpt2,
+    inputs,
+    outputs,
+    title=title,
+    description=description,
+    article=article,
+    examples=examples,
+    theme="huggingface",
+).launch(debug=True, enable_queue=True)
+

requirements.txt

@@ -0,0 +1,3 @@
+torch
+transformers
+Pillow