cleanup clip tokenizer
MODIFY
examples/stable_diffusion.py
examples/stable_diffusion.py
@@ -490,7 +490,7 @@ class CLIPTextTransformer:
# Clip tokenizer, taken from https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py (MIT license)
@lru_cache()
def default_bpe():
- return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+ return os.path.join(os.path.dirname(os.path.abspath(__file__)), "../weights/bpe_simple_vocab_16e6.txt.gz")
def get_pairs(word):
"""Return set of symbol pairs in a word.
@@ -566,7 +566,7 @@ class ClipTokenizer(object):
j = word.index(first, i)
new_word.extend(word[i:j])
i = j
- except:
+ except Exception:
new_word.extend(word[i:])
break
@@ -650,6 +650,9 @@ if __name__ == "__main__":
tokenizer = ClipTokenizer()
phrase = tokenizer.encode("a horse sized cat eating a bagel")
+ _phrase = [49406, 320, 4558, 9832, 2368, 4371, 320, 28777, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407]
+ assert tuple(phrase) == tuple(_phrase)
+
# phrase = tokenizer.encode("penguin with fire extinguisher")
context = model.cond_stage_model.transformer.text_model(phrase).realize()

RENAME
examples/bpe_simple_vocab_16e6.txt.gz
weights/bpe_simple_vocab_16e6.txt.gz
cleanup clip tokenizer