Spaces:
Sleeping
Sleeping
| import torch | |
| import torch.nn as nn | |
| from functools import partial | |
| import clip | |
| import open_clip | |
| from einops import rearrange, repeat | |
| from transformers import CLIPTokenizer, CLIPTextModel | |
| # import kornia | |
| from transformers import BertTokenizerFast # TODO: add to reuquirements | |
| import os | |
| os.environ["TOKENIZERS_PARALLELISM"] = "false" | |
| from ldm.modules.x_transformer import Encoder, TransformerWrapper # TODO: can we directly rely on lucidrains code and simply add this as a reuirement? --> test | |
| class AbstractEncoder(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| def encode(self, *args, **kwargs): | |
| raise NotImplementedError | |
| class ClassEmbedder(nn.Module): | |
| def __init__(self, embed_dim, n_classes=1000, key='class'): | |
| super().__init__() | |
| self.key = key | |
| self.embedding = nn.Embedding(n_classes, embed_dim) | |
| def forward(self, batch, key=None): | |
| if key is None: | |
| key = self.key | |
| # this is for use in crossattn | |
| c = batch[key][:, None] | |
| c = self.embedding(c) | |
| return c | |
| class HeirClassEmbedder(nn.Module): | |
| def __init__(self, embed_dim, n_classes=[3, 6, 9, 38], key='class', device='cuda'): | |
| super().__init__() | |
| assert embed_dim % len(n_classes) == 0 | |
| self.key = key | |
| self.device = device | |
| self.embed_heir_dim = embed_dim//len(n_classes) | |
| self.embedding_layers = [] | |
| self.embedding_level0 = nn.Embedding(n_classes[0], self.embed_heir_dim) | |
| self.embedding_level1 = nn.Embedding(n_classes[1], self.embed_heir_dim) | |
| self.embedding_level2 = nn.Embedding(n_classes[2], self.embed_heir_dim) | |
| self.embedding_level3 = nn.Embedding(n_classes[3], self.embed_heir_dim) | |
| # for i in list(n_classes): | |
| # embedding = nn.Embedding(i, self.embed_heir_dim) | |
| # self.embedding_layers.append(embedding) | |
| def forward(self, batch, key=None): | |
| if key is None: | |
| key = self.key | |
| # this is for use in crossattn | |
| batch_size = len(batch[key][0]) | |
| heir_classes = batch[key] | |
| # heir_classes_list = [] | |
| # for s in heir_classes: | |
| # numbers = s.split(', ') | |
| # heir_classes_list.extend(int(num) for num in numbers) | |
| heir_classes = [[int(num) for num in item.split(', ')] for item in heir_classes[0]] | |
| transformed_list = [list(pair) for pair in zip(*heir_classes)] | |
| tensor_list = [torch.tensor(sublist).to(self.device) for sublist in transformed_list] | |
| tensor_reshaped = [torch.reshape(sublist, (batch_size, 1)) for sublist in tensor_list] | |
| embedding_list = [self.embedding_level0(tensor_reshaped[0]), self.embedding_level1(tensor_reshaped[1]), | |
| self.embedding_level2(tensor_reshaped[2]), self.embedding_level3(tensor_reshaped[3])] | |
| # embedding = [] | |
| # for i, classes in enumerate(heir_classes): | |
| # embedding.append(self.embedding_layers[i](classes)) | |
| embedding = torch.cat(embedding_list, dim=-1) | |
| return embedding | |
| class HeirClassEmbedderMultiLevel(nn.Module): | |
| def __init__(self, embed_dim, n_classes=[3, 6, 9, 38], key='class', device='cuda'): | |
| super().__init__() | |
| assert embed_dim % len(n_classes) == 0 | |
| self.key = key | |
| self.device = device | |
| self.n_classes = n_classes | |
| self.embed_heir_dim = embed_dim//len(n_classes) | |
| self.embedding_layers = [] | |
| self.embedding_level0 = nn.Embedding(n_classes[0], self.embed_heir_dim) | |
| self.embedding_level1 = nn.Embedding(n_classes[1], self.embed_heir_dim) | |
| self.embedding_level2 = nn.Embedding(n_classes[2], self.embed_heir_dim) | |
| self.embedding_level3 = nn.Embedding(n_classes[3], self.embed_heir_dim) | |
| # self.embedding_level4 = nn.Embedding(n_classes[4], self.embed_heir_dim) | |
| # self.embedding_layers = [] | |
| self.embedding_layers = nn.ModuleList() | |
| for i in list(n_classes): | |
| embedding = nn.Embedding(i, self.embed_heir_dim) | |
| self.embedding_layers.append(embedding.to(self.device)) | |
| # self.to(self.device) | |
| def forward(self, batch, key=None): | |
| if key is None: | |
| key = self.key | |
| # this is for use in crossattn | |
| batch_size = len(batch[key][0]) | |
| hier_classes = batch[key] | |
| # heir_classes_list = [] | |
| # for s in heir_classes: | |
| # numbers = s.split(', ') | |
| # heir_classes_list.extend(int(num) for num in numbers) | |
| hier_classes = [[int(num) for num in item.split(', ')] for item in hier_classes[0]] | |
| transformed_list = [list(pair) for pair in zip(*hier_classes)] | |
| tensor_list = [torch.tensor(sublist).to(self.device) for sublist in transformed_list] | |
| tensor_reshaped = [torch.reshape(sublist, (batch_size, 1)) for sublist in tensor_list] | |
| embedding_list = [] | |
| for i in range(len(self.n_classes)): | |
| embedding_list.append(self.embedding_layers[i](tensor_reshaped[i])) | |
| # embedding_list_org = [self.embedding_level0(tensor_reshaped[0]), self.embedding_level1(tensor_reshaped[1]), | |
| # self.embedding_level2(tensor_reshaped[2]), self.embedding_level3(tensor_reshaped[3]), | |
| # self.embedding_level3(tensor_reshaped[4])] | |
| # embedding_list_org = [self.embedding_level0(tensor_reshaped[0]), self.embedding_level1(tensor_reshaped[1]), | |
| # self.embedding_level2(tensor_reshaped[2]), self.embedding_level3(tensor_reshaped[3])] | |
| # embedding_org = torch.cat(embedding_list_org, dim=-1) | |
| embedding = torch.cat(embedding_list, dim=-1) | |
| return embedding | |
| class TransformerEmbedder(AbstractEncoder): | |
| """Some transformer encoder layers""" | |
| def __init__(self, n_embed, n_layer, vocab_size, max_seq_len=77, device="cuda"): | |
| super().__init__() | |
| self.device = device | |
| self.transformer = TransformerWrapper(num_tokens=vocab_size, max_seq_len=max_seq_len, | |
| attn_layers=Encoder(dim=n_embed, depth=n_layer)) | |
| def forward(self, tokens): | |
| tokens = tokens.to(self.device) # meh | |
| z = self.transformer(tokens, return_embeddings=True) | |
| return z | |
| def encode(self, x): | |
| return self(x) | |
| class BERTTokenizer(AbstractEncoder): | |
| """ Uses a pretrained BERT tokenizer by huggingface. Vocab size: 30522 (?)""" | |
| def __init__(self, device="cuda", vq_interface=True, max_length=77): | |
| super().__init__() | |
| self.tokenizer = BertTokenizerFast.from_pretrained("bert-base-uncased") | |
| self.device = device | |
| self.vq_interface = vq_interface | |
| self.max_length = max_length | |
| def forward(self, text): | |
| batch_encoding = self.tokenizer(text, truncation=True, max_length=self.max_length, return_length=True, | |
| return_overflowing_tokens=False, padding="max_length", return_tensors="pt") | |
| tokens = batch_encoding["input_ids"].to(self.device) | |
| return tokens | |
| def encode(self, text): | |
| tokens = self(text) | |
| if not self.vq_interface: | |
| return tokens | |
| return None, None, [None, None, tokens] | |
| def decode(self, text): | |
| return text | |
| class BERTEmbedderExtra(AbstractEncoder): | |
| """Uses the BERT tokenizr model and add some transformer encoder layers""" | |
| def __init__(self, n_embed, n_layer, vocab_size=30522, max_seq_len=77, | |
| device="cuda",use_tokenizer=True, embedding_dropout=0.0): | |
| super().__init__() | |
| self.use_tknz_fn = use_tokenizer | |
| if self.use_tknz_fn: | |
| self.tknz_fn = BERTTokenizer(vq_interface=False, max_length=max_seq_len) | |
| self.device = device | |
| special_tokens_dict = {'additional_special_tokens': ['<N>','<E>']} | |
| num_added_toks = self.tknz_fn.tokenizer.add_special_tokens(special_tokens_dict) | |
| vocab_size = len(self.tknz_fn.tokenizer) | |
| self.transformer = TransformerWrapper(num_tokens=vocab_size, max_seq_len=max_seq_len, | |
| attn_layers=Encoder(dim=n_embed, depth=n_layer), | |
| emb_dropout=embedding_dropout) | |
| def forward(self, text): | |
| if self.use_tknz_fn: | |
| tokens = self.tknz_fn(text)#.to(self.device) | |
| else: | |
| tokens = text | |
| z = self.transformer(tokens, return_embeddings=True) | |
| return z | |
| def encode(self, text): | |
| # output of length 77 | |
| return self(text) | |
| class BERTEmbedder(AbstractEncoder): | |
| """Uses the BERT tokenizr model and add some transformer encoder layers""" | |
| def __init__(self, n_embed, n_layer, vocab_size=30522, max_seq_len=77, | |
| device="cuda",use_tokenizer=True, embedding_dropout=0.0): | |
| super().__init__() | |
| self.use_tknz_fn = use_tokenizer | |
| if self.use_tknz_fn: | |
| self.tknz_fn = BERTTokenizer(vq_interface=False, max_length=max_seq_len) | |
| self.device = device | |
| self.transformer = TransformerWrapper(num_tokens=vocab_size, max_seq_len=max_seq_len, | |
| attn_layers=Encoder(dim=n_embed, depth=n_layer), | |
| emb_dropout=embedding_dropout) | |
| def forward(self, text): | |
| if self.use_tknz_fn: | |
| tokens = self.tknz_fn(text)#.to(self.device) | |
| else: | |
| tokens = text | |
| z = self.transformer(tokens, return_embeddings=True) | |
| return z | |
| def encode(self, text): | |
| # output of length 77 | |
| return self(text) | |
| class SpatialRescaler(nn.Module): | |
| def __init__(self, | |
| n_stages=1, | |
| method='bilinear', | |
| multiplier=0.5, | |
| in_channels=3, | |
| out_channels=None, | |
| bias=False): | |
| super().__init__() | |
| self.n_stages = n_stages | |
| assert self.n_stages >= 0 | |
| assert method in ['nearest','linear','bilinear','trilinear','bicubic','area'] | |
| self.multiplier = multiplier | |
| self.interpolator = partial(torch.nn.functional.interpolate, mode=method) | |
| self.remap_output = out_channels is not None | |
| if self.remap_output: | |
| print(f'Spatial Rescaler mapping from {in_channels} to {out_channels} channels after resizing.') | |
| self.channel_mapper = nn.Conv2d(in_channels,out_channels,1,bias=bias) | |
| def forward(self,x): | |
| for stage in range(self.n_stages): | |
| x = self.interpolator(x, scale_factor=self.multiplier) | |
| if self.remap_output: | |
| x = self.channel_mapper(x) | |
| return x | |
| def encode(self, x): | |
| return self(x) | |
| ### not using - hugging face implementation | |
| class FrozenCLIPEmbedder(AbstractEncoder): | |
| """Uses the CLIP transformer encoder for text (from Hugging Face)""" | |
| def __init__(self, version="openai/clip-vit-large-patch14", device="cuda", max_length=77): | |
| super().__init__() | |
| self.tokenizer = CLIPTokenizer.from_pretrained(version) | |
| self.transformer = CLIPTextModel.from_pretrained(version) | |
| self.transformer.projection_dim = 512 | |
| self.device = device | |
| self.max_length = max_length | |
| self.freeze() | |
| def freeze(self): | |
| self.transformer = self.transformer.eval() | |
| for param in self.parameters(): | |
| param.requires_grad = False | |
| def forward(self, text): | |
| batch_encoding = self.tokenizer(text, truncation=True, max_length=self.max_length, return_length=True, | |
| return_overflowing_tokens=False, padding="max_length", return_tensors="pt") | |
| tokens = batch_encoding["input_ids"].to(self.device) | |
| outputs = self.transformer(input_ids=tokens) | |
| z = outputs.last_hidden_state | |
| # pooled_output = outputs.pooler_output | |
| # return pooled_output | |
| return z | |
| def encode(self, text): | |
| return self(text) | |
| class FrozenCLIPTextEmbedder(nn.Module): | |
| """ | |
| Uses the CLIP transformer encoder for text. | |
| """ | |
| def __init__(self, version='ViT-L/14', device="cuda", max_length=77, n_repeat=1, normalize=True): | |
| super().__init__() | |
| self.model, _ = clip.load(version, jit=False, device="cpu") | |
| self.device = device | |
| self.max_length = max_length | |
| self.n_repeat = n_repeat | |
| self.normalize = normalize | |
| def freeze(self): | |
| self.model = self.model.eval() | |
| for param in self.parameters(): | |
| param.requires_grad = False | |
| def forward(self, text): | |
| tokens = clip.tokenize(text).to(self.device) | |
| z = self.model.encode_text(tokens) | |
| if self.normalize: | |
| z = z / torch.linalg.norm(z, dim=1, keepdim=True) | |
| return z | |
| def encode(self, text): | |
| z = self(text) | |
| if z.ndim==2: | |
| z = z[:, None, :] | |
| z = repeat(z, 'b 1 d -> b k d', k=self.n_repeat) | |
| return z | |
| class FrozenBioClipTextEmbedder(nn.Module): | |
| """ | |
| Uses the BioClip transformer encoder for text. | |
| """ | |
| def __init__(self, version='hf-hub:imageomics/bioclip', device="cuda", max_length=77, n_repeat=1, normalize=True): | |
| super().__init__() | |
| # self.model, _ = open_clip.create_model_and_transforms(version, jit=False, device="cpu") | |
| self.model, _, _ = open_clip.create_model_and_transforms(version) | |
| self.model = self.model.eval() | |
| self.model = self.model.to(device) | |
| self.tokenizer = open_clip.get_tokenizer(version) | |
| self.device = device | |
| self.max_length = max_length | |
| self.n_repeat = n_repeat | |
| self.normalize = normalize | |
| # model = model.eval() | |
| # model = model.to(device) | |
| def freeze(self): | |
| self.model = self.model.eval() | |
| for param in self.parameters(): | |
| param.requires_grad = False | |
| def forward(self, text): | |
| tokens = self.tokenizer(text).to(self.device) | |
| z = self.model.encode_text(tokens) | |
| if self.normalize: | |
| z = z / torch.linalg.norm(z, dim=1, keepdim=True) | |
| return z | |
| def encode(self, text): | |
| z = self(text) | |
| if z.ndim==2: | |
| z = z[:, None, :] | |
| z = repeat(z, 'b 1 d -> b k d', k=self.n_repeat) | |
| return z | |
| # class FrozenClipImageEmbedder(nn.Module): | |
| # """ | |
| # Uses the CLIP image encoder. | |
| # """ | |
| # def __init__( | |
| # self, | |
| # model, | |
| # jit=False, | |
| # device='cuda' if torch.cuda.is_available() else 'cpu', | |
| # antialias=False, | |
| # ): | |
| # super().__init__() | |
| # self.model, _ = clip.load(name=model, device=device, jit=jit) | |
| # self.antialias = antialias | |
| # self.register_buffer('mean', torch.Tensor([0.48145466, 0.4578275, 0.40821073]), persistent=False) | |
| # self.register_buffer('std', torch.Tensor([0.26862954, 0.26130258, 0.27577711]), persistent=False) | |
| # def preprocess(self, x): | |
| # # normalize to [0,1] | |
| # x = kornia.geometry.resize(x, (224, 224), | |
| # interpolation='bicubic',align_corners=True, | |
| # antialias=self.antialias) | |
| # x = (x + 1.) / 2. | |
| # # renormalize according to clip | |
| # x = kornia.enhance.normalize(x, self.mean, self.std) | |
| # return x | |
| # def forward(self, x): | |
| # # x is assumed to be in range [-1,1] | |
| # return self.model.encode_image(self.preprocess(x)) | |