<jupyter_start><jupyter_text>Setup<jupyter_code>!pip install -q git+https://github.com/divyashreepathihalli/keras-cv.git@CLIP_refactor
!pip install -q keras-nlp
!pip install -q tf-keras
!pip install -q tensorflow-text
!pip install keras==3.0.2<jupyter_output>Installing build dependencies ... [?25l[?25hdone
  Getting requirements to build wheel ... [?25l[?25hdone
  Preparing metadata (pyproject.toml) ... [?25l[?25hdone
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m950.8/950.8 kB[0m [31m5.9 MB/s[0m eta [36m0:00:00[0m
[?25h  Building wheel for keras-cv (pyproject.toml) ... [?25l[?25hdone
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m415.4/415.4 kB[0m [31m5.4 MB/s[0m eta [36m0:00:00[0m
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m5.2/5.2 MB[0m [31m17.0 MB/s[0m eta [36m0:00:00[0m
[?25hCollecting keras==3.0.2
  Downloading keras-3.0.2-py3-none-any.whl (1.0 MB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m1.0/1.0 MB[0m [31m8.4 MB/s[0m eta [36m0:00:00[0m
[?25hRequirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from keras==3.0.2) (1.4.0)
Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packa[...]<jupyter_text>Import<jupyter_code>from keras_cv.models.feature_extractor.clip import CLIPProcessor
import keras
from keras_cv.models import CLIP
!wget https://i.imgur.com/8H7XCH0.jpg -O cat.jpg
!wget http://images.cocodataset.org/val2017/000000039769.jpg -O test.jpg
# @title Select which model weights you would like to convert
MODEL_CONFIGS = {
    "CLIP_B32": {
        "embed_dim": 512,
        "context_length": 77,
        "vocab_size": 49408,
        "transformer_width": 512,
        "transformer_heads": 8,
        "transformer_layers": 12,
        "vision_layers": 12,
        "vision_width": 768,
        "image_resolution": 224,
        "vision_patch_size": 32,
    },
    "CLIP_B16": {
        "embed_dim": 512,
        "context_length": 77,
        "vocab_size": 49408,
        "transformer_width": 512,
        "transformer_heads": 8,
        "transformer_layers": 12,
        "vision_layers": 12,
        "vision_width": 768,
        "image_resolution": 224,
        "vision_patch_size": 16,
    },
    "CLIP_L14": {
        "embed_dim": 768,
        "context_length": 77,
        "vocab_size": 49408,
        "transformer_width": 768,
        "transformer_heads": 12,
        "transformer_layers": 12,
        "vision_layers": 24,
        "vision_width": 1024,
        "image_resolution": 224,
        "vision_patch_size": 14,
    },
    "CLIP_L14_336": {
        "embed_dim": 768,
        "context_length": 77,
        "vocab_size": 49408,
        "transformer_width": 768,
        "transformer_heads": 12,
        "transformer_layers": 12,
        "vision_layers": 24,
        "vision_width": 1024,
        "image_resolution": 336,
        "vision_patch_size": 14,
    },
}
model_map_hf = {
    "CLIP_B16": "openai/clip-vit-base-patch32",
    "CLIP_B32": "openai/clip-vit-base-patch16",
    "CLIP_L14": "openai/clip-vit-large-patch14",
    "CLIP_L14_336": "openai/clip-vit-large-patch14-336",
}
config_name = "CLIP_L14_336"  # @param ["CLIP_B16", "CLIP_B32", "CLIP_L14", "CLIP_L14_336"]
config_name_hf = model_map_hf[config_name]<jupyter_output><empty_output><jupyter_text>Keras 3 CLIP<jupyter_code>embed_dim = MODEL_CONFIGS[config_name]["embed_dim"]
context_length = MODEL_CONFIGS[config_name]["context_length"]
vocab_size = MODEL_CONFIGS[config_name]["vocab_size"]
transformer_width = MODEL_CONFIGS[config_name]["transformer_width"]
transformer_heads = MODEL_CONFIGS[config_name]["transformer_heads"]
transformer_layers = MODEL_CONFIGS[config_name]["transformer_layers"]
vision_layers = MODEL_CONFIGS[config_name]["vision_layers"]
vision_width = MODEL_CONFIGS[config_name]["vision_width"]
vision_patch_size = MODEL_CONFIGS[config_name]["vision_patch_size"]
image_resolution = MODEL_CONFIGS[config_name]["image_resolution"]
model = CLIP(
    embed_dim,
    image_resolution,
    vision_layers,
    vision_width,
    vision_patch_size,
    context_length,
    vocab_size,
    transformer_width,
    transformer_heads,
    transformer_layers,
)
model.summary()
processor = CLIPProcessor(224, "vocab.json", "merges.txt")
image = processor.process_images(["cat.jpg"])
text_input = [
    "photo of a cat on a tortoise",
    "tortoise on a dog",
    "a photo of a tortoise",
]
text = processor.process_texts(text_input)
image_logits, text_logits = model(image, text)
output = keras.layers.Softmax()(image_logits)
print(image_logits)
print(text_input[keras.ops.argmax(output)])
model.summary()<jupyter_output><empty_output><jupyter_text>HF CLIP<jupyter_code>from PIL import Image
import requests

from transformers import CLIPProcessor as CP
from transformers import CLIPModel as CM
model_hf = CM.from_pretrained(config_name_hf)
processor = CP.from_pretrained(config_name_hf)
url = "https://i.imgur.com/8H7XCH0.jpg"
image_hf = Image.open(requests.get(url, stream=True).raw)
text_inputs = [
    "photo of a cat on a tortoise",
    "tortoise on a dog",
    "a photo of a tortoise",
]
inputs = processor(
    text=text_inputs, images=image_hf, return_tensors="pt", padding=True
)

outputs = model_hf(**inputs)
logits_per_image = (
    outputs.logits_per_image
)  # this is the image-text similarity score
probs = logits_per_image.softmax(
    dim=1
)  # we can take the softmax to get the label probabilitiesprobs
probs
# hugging face weights
hf_wts = model_hf.state_dict()<jupyter_output><empty_output><jupyter_text>Copy weights vision encoder<jupyter_code>model.logit_scale.assign(hf_wts.pop("logit_scale").numpy())
model.get_layer("image_encoder").get_layer(
    "clip_patching_and_embedding"
).class_embedding.assign(
    hf_wts.pop("vision_model.embeddings.class_embedding").numpy()
)
model.get_layer("image_encoder").get_layer(
    "clip_patching_and_embedding"
).positional_embedding.assign(
    hf_wts.pop("vision_model.embeddings.position_embedding.weight").numpy()
)
model.get_layer("image_encoder").get_layer(
    "clip_patching_and_embedding"
).conv1.weights[0].assign(
    hf_wts.pop("vision_model.embeddings.patch_embedding.weight")
    .permute(3, 2, 1, 0)
    .numpy()
)
model.get_layer("image_encoder").get_layer("ln_1").weights[0].assign(
    hf_wts.pop("vision_model.pre_layrnorm.weight").numpy()
)
model.get_layer("image_encoder").get_layer("ln_1").weights[1].assign(
    hf_wts.pop("vision_model.pre_layrnorm.bias").numpy()
)
model.get_layer("image_encoder").get_layer("ln_2").weights[0].assign(
    hf_wts.pop("vision_model.post_layernorm.weight").numpy()
)
model.get_layer("image_encoder").get_layer("ln_2").weights[1].assign(
    hf_wts.pop("vision_model.post_layernorm.bias").numpy()
)
model.get_layer("image_encoder").get_layer("vision_projector").weights[
    0
].assign(hf_wts.pop("visual_projection.weight").transpose(1, 0).numpy())
for i in range(0, MODEL_CONFIGS[config_name]["vision_layers"]):
    if i == 0:
        residual_attention = f"residual_attention"
    else:
        residual_attention = f"residual_attention_{i}"
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).attn.q_proj.weights[0].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.self_attn.q_proj.weight")
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).attn.q_proj.weights[1].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.self_attn.q_proj.bias")
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).attn.k_proj.weights[0].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.self_attn.k_proj.weight")
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).attn.k_proj.weights[1].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.self_attn.k_proj.bias")
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).attn.v_proj.weights[0].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.self_attn.v_proj.weight")
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).attn.v_proj.weights[1].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.self_attn.v_proj.bias")
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).attn.out_proj.weights[1].assign(
        hf_wts.pop(
            f"vision_model.encoder.layers.{i}.self_attn.out_proj.bias"
        ).numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).attn.out_proj.weights[0].assign(
        hf_wts.pop(
            f"vision_model.encoder.layers.{i}.self_attn.out_proj.weight"
        ).numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).ln_1.weights[0].assign(
        hf_wts.pop(
            f"vision_model.encoder.layers.{i}.layer_norm1.weight"
        ).numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).ln_1.weights[1].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.layer_norm1.bias").numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).ln_2.weights[0].assign(
        hf_wts.pop(
            f"vision_model.encoder.layers.{i}.layer_norm2.weight"
        ).numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).ln_2.weights[1].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.layer_norm2.bias").numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).mlp.get_layer("c_fc").weights[
        0
    ].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.mlp.fc1.weight")
        .transpose(1, 0)
        .numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).mlp.get_layer("c_fc").weights[
        1
    ].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.mlp.fc1.bias").numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).mlp.get_layer("c_proj").weights[
        0
    ].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.mlp.fc2.weight")
        .transpose(1, 0)
        .numpy()
    )
    model.get_layer("image_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(residual_attention).mlp.get_layer("c_proj").weights[
        1
    ].assign(
        hf_wts.pop(f"vision_model.encoder.layers.{i}.mlp.fc2.bias").numpy()
    )<jupyter_output><empty_output><jupyter_text>Text encoder<jupyter_code>num_transformer_layers = MODEL_CONFIGS[config_name]["vision_layers"]
model.get_layer("text_encoder").get_layer("text_projector").weights[0].assign(
    hf_wts.pop("text_projection.weight").numpy()
)
model.get_layer("text_encoder").get_layer("token_embedding").weights[0].assign(
    hf_wts.pop("text_model.embeddings.token_embedding.weight").numpy()
)
model.get_layer("text_encoder").positional_embedding.assign(
    hf_wts.pop("text_model.embeddings.position_embedding.weight").numpy()
)
model.get_layer("text_encoder").get_layer("ln_final").weights[0].assign(
    hf_wts.pop("text_model.final_layer_norm.weight")
)
model.get_layer("text_encoder").get_layer("ln_final").weights[1].assign(
    hf_wts.pop("text_model.final_layer_norm.bias")
)
for i in range(MODEL_CONFIGS[config_name]["transformer_layers"]):
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).attn.k_proj.weights[
        0
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.self_attn.k_proj.weight")
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).attn.k_proj.weights[
        1
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.self_attn.k_proj.bias")
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).attn.q_proj.weights[
        0
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.self_attn.q_proj.weight")
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).attn.q_proj.weights[
        1
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.self_attn.q_proj.bias")
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).attn.v_proj.weights[
        0
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.self_attn.v_proj.weight")
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).attn.v_proj.weights[
        1
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.self_attn.v_proj.bias")
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).attn.out_proj.weights[
        0
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.self_attn.out_proj.weight")
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).attn.out_proj.weights[
        1
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.self_attn.out_proj.bias")
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).ln_1.weights[
        0
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.layer_norm1.weight").numpy()
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).ln_1.weights[
        1
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.layer_norm1.bias").numpy()
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).ln_2.weights[
        0
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.layer_norm2.weight").numpy()
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).ln_2.weights[
        1
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.layer_norm2.bias").numpy()
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).mlp.get_layer(
        "c_fc"
    ).weights[
        0
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.mlp.fc1.weight")
        .transpose(1, 0)
        .numpy()
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).mlp.get_layer(
        "c_fc"
    ).weights[
        1
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.mlp.fc1.bias").numpy()
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).mlp.get_layer(
        "c_proj"
    ).weights[
        0
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.mlp.fc2.weight")
        .transpose(1, 0)
        .numpy()
    )
    model.get_layer("text_encoder").get_layer(
        "clip_encoder"
    ).resblocks.get_layer(
        f"residual_attention_{num_transformer_layers+i}"
    ).mlp.get_layer(
        "c_proj"
    ).weights[
        1
    ].assign(
        hf_wts.pop(f"text_model.encoder.layers.{i}.mlp.fc2.bias").numpy()
    )
# verify that we copied all weights
hf_wts.keys()<jupyter_output><empty_output><jupyter_text>save weights<jupyter_code>model.save_weights("clip-vit-base-patch32.weights.h5")<jupyter_output><empty_output>