Upload folder using huggingface_hub

README.md

@@ -1,3 +1,118 @@
----
-license: apache-2.0
----
+<h3>VARGPT: Unified Understanding and Generation in a Visual Autoregressive Multimodal Large Language Model</h3>
+
+VARGPT (7B+2B) modeling understanding and generation as two distinct paradigms within a unified model: **predicting the next token for visual understanding and predicting the next scale for visual generation**.
+
+We provide the simple generation process for using our model. For more details, you could refer to Github.
+
+### Multimodal Understanding
+Inference demo for **Multimodal Understanding**. You can execute the following code:
+```python
+# Or execute the following code
+import requests
+from PIL import Image
+
+import torch
+from transformers import AutoProcessor, AutoTokenizer
+from vargpt_llava.modeling_vargpt_llava import VARGPTLlavaForConditionalGeneration
+from vargpt_llava.prepare_vargpt_llava import prepare_vargpt_llava 
+from vargpt_llava.processing_vargpt_llava import VARGPTLlavaProcessor
+from patching_utils.patching import patching
+
+model_id = "VARGPT_LLaVA-v1"
+prepare_vargpt_llava(model_id)
+
+model = VARGPTLlavaForConditionalGeneration.from_pretrained(
+    model_id, 
+    torch_dtype=torch.float32, 
+    low_cpu_mem_usage=True, 
+).to(0)
+patching(model)
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+processor = VARGPTLlavaProcessor.from_pretrained(model_id)
+
+# Define a chat history and use `apply_chat_template` to get correctly formatted prompt
+# Each value in "content" has to be a list of dicts with types ("text", "image") 
+conversation = [
+    {
+      "role": "user",
+      "content": [
+          {"type": "text", "text": "Please explain the meme in detail."},
+          {"type": "image"},
+        ],
+    },
+]
+prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
+image_file = "./assets/llava_bench_demo.png"
+print(prompt)
+
+raw_image = Image.open(image_file)
+inputs = processor(images=raw_image, text=prompt, return_tensors='pt').to(0, torch.float32)
+
+output = model.generate(
+    **inputs, 
+    max_new_tokens=2048, 
+    do_sample=False)
+
+print(processor.decode(output[0], skip_special_tokens=True))
+
+
+```
+### Multimodal Generation
+
+
+Inference demo for **Text-to-Image Generation**. You can execute the following code:
+```python
+import requests
+from PIL import Image
+
+import torch
+from transformers import AutoProcessor, AutoTokenizer
+from vargpt_llava.modeling_vargpt_llava import VARGPTLlavaForConditionalGeneration
+from vargpt_llava.prepare_vargpt_llava import prepare_vargpt_llava 
+from vargpt_llava.processing_vargpt_llava import VARGPTLlavaProcessor
+from patching_utils.patching import patching
+model_id = "VARGPT_LLaVA-v1"
+
+prepare_vargpt_llava(model_id)
+
+model = VARGPTLlavaForConditionalGeneration.from_pretrained(
+    model_id, 
+    torch_dtype=torch.float32, 
+    low_cpu_mem_usage=True, 
+).to(0)
+
+patching(model)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+processor = VARGPTLlavaProcessor.from_pretrained(model_id)
+
+# some instruction examples:
+# Please design a drawing of a butterfly on a flower.
+# Please create a painting of a black weasel is standing in the grass.
+# Can you generate a rendered photo of a rabbit sitting in the grass.
+# I need a designed photo of a lighthouse is seen in the distance.
+# Please create a rendered drawing of an old photo of an aircraft carrier in the water.
+# Please produce a designed photo of a squirrel is standing in the snow.
+
+
+conversation = [
+    {
+      "role": "user",
+      "content": [
+          {"type": "text", "text": "Please design a drawing of a butterfly on a flower."},
+        ],
+    },
+]
+prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
+print(prompt)
+
+inputs = processor(text=prompt, return_tensors='pt').to(0, torch.float32)
+model._IMAGE_GEN_PATH = "output.png"
+output = model.generate(
+    **inputs, 
+    max_new_tokens=2048, 
+    do_sample=False)
+
+print(processor.decode(output[0], skip_special_tokens=True))
+
+```

added_tokens.json

@@ -0,0 +1,7 @@
+{
+  "<image>": 32000,
+  "<pad>": 32001,
+  "<|image_gen_end|>": 32003,
+  "<|image_gen_pad|>": 32004,
+  "<|image_gen_start|>": 32002
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 11.995625227852717,
+    "total_flos": 1.7779223938308506e+17,
+    "train_loss": 6.699234984606801,
+    "train_runtime": 169455.6516,
+    "train_samples_per_second": 99.441,
+    "train_steps_per_second": 0.097
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>\n' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}"
+}

config.json

@@ -0,0 +1,192 @@
+{
+  "_name_or_path": "/mnt/bn/yufan-lf/pretrained_models/VARGPT_LLaVA-7B-stage3/VARGPT_LLaVA-7B-v1",
+  "architectures": [
+    "VARGPTLlavaForConditionalGeneration"
+  ],
+  "hidden_size": 4096,
+  "ignore_index": -100,
+  "image_seq_length": 576,
+  "image_token_index": 32000,
+  "model_type": "vargpt_llava",
+  "pad_token_id": 32001,
+  "padding_side": "left",
+  "projector_hidden_act": "gelu",
+  "special_tokens": {
+    "image_gen_end": "<|image_gen_end|>",
+    "image_gen_end_token_id": 32003,
+    "image_gen_pad": "<|image_gen_pad|>",
+    "image_gen_pad_token_id": 32004,
+    "image_gen_start": "<|image_gen_start|>",
+    "image_gen_start_token_id": 32002
+  },
+  "text_config": {
+    "_attn_implementation_autoset": false,
+    "_name_or_path": "lmsys/vicuna-7b-v1.5",
+    "add_cross_attention": false,
+    "architectures": [
+      "LlamaForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 11008,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 4096,
+    "min_length": 0,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 32,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 32,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "pretraining_tp": 1,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": null,
+    "rope_theta": 10000.0,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "float16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "vocab_size": 32064
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "train_from_scratch": false,
+  "transformers_version": "4.46.1",
+  "use_cache": false,
+  "vision_config": {
+    "_attn_implementation_autoset": false,
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "quick_gelu",
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 336,
+    "initializer_factor": 1.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "clip_vision_model",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "projection_dim": 768,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "vocab_size": 32000
+  },
+  "vision_feature_layer": -2,
+  "vision_feature_select_strategy": "default",
+  "vocab_size": 32064
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 32001,
+  "transformers_version": "4.46.1"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:788c7bfdae26ef9c00297dddd4b47737b43a99202da9e27e7a5c61d2d986d546
+size 4992930688

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5851ee22d392303c11c957a5babb95de98de3d4b8f853c238c761c337137acc
+size 4957878552

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bff7b86adfc3f7602c564ce6ce6e99dea4d626bde98c56b8e5b6ca23e87d9c0
+size 4978083414

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0520b548f43ce52ac29bd4103562328851d442fabe58682b1eede96db52f983
+size 3628686856

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "crop_size": {
+    "height": 336,
+    "width": 336
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "processor_class": "VARGPTLlavaProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 336
+  }
+}

processor_config.json

@@ -0,0 +1,6 @@
+{
+  "image_token": "<image>",
+  "patch_size": 14,
+  "processor_class": "VARGPTLlavaProcessor",
+  "vision_feature_select_strategy": "default"
+}

special_tokens_map.json

@@ -0,0 +1,35 @@
+{
+  "additional_special_tokens": [
+    "<|image_gen_start|>",
+    "<|image_gen_end|>",
+    "<|image_gen_pad|>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,95 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|image_gen_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|image_gen_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|image_gen_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|image_gen_start|>",
+    "<|image_gen_end|>",
+    "<|image_gen_pad|>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{% set system_message = 'A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user\\'s questions.' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'USER: ' + content + ' ASSISTANT:' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {
+    "image_token": "<image>"
+  },
+  "image_token": "<image>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "processor_class": "VARGPTLlavaProcessor",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "trust_remote_code": false,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}