import streamlit as st from tokenizers.tools import EncodingVisualizer from transformers import AutoTokenizer st.set_page_config(page_title="BigCode Tokenizer", page_icon='👩‍💻', layout="wide") models = ["bigcode/tokenizer", "facebook/incoder-6B", "Salesforce/codegen-16B-mono"] @st.cache() def load_tokenizer(): if selected_model == "bigcode/tokenizer": tokenizer = AutoTokenizer.from_pretrained("bigcode/tokenizer", subfolder="digit-custom_punctuation-bytelevel-bpe-py-js-java-50k") else: tokenizer = AutoTokenizer.from_pretrained(selected_model) return tokenizer col1, col2 = st.columns([1, 2]) with col1: selected_model = st.selectbox("Select a tokenizer", models, key=1) text = st.text_area(label="", placeholder="Text to tokenize") button_clicked = st.button("Tokenize") tokenizer = load_tokenizer() visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False) if text or button_clicked: st.write(f"The input was split into {len(tokenizer.tokenize(text))} tokens.") st.components.v1.html(visualizer(text), height=1500)