Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8" /> | |
| <meta name="viewport" content="width=device-width, initial-scale=1" /> | |
| <title>TokenVisualizer — Minimal</title> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet"> | |
| <style> | |
| :root{ | |
| --bg:#0b0f14; --text:#ffffff; --muted:#9aa4b2; --accent:#38bdf8; --border:#1f2a3a; | |
| --card1:#0c1624; --card2:#0a1220; --chip:#111827; --chip-border:#263246; --chip-hover:#1a2434; | |
| --mono:'JetBrains Mono',ui-monospace,Menlo,Consolas,monospace; --sans:Inter,system-ui,-apple-system,"Segoe UI",Roboto,Ubuntu,"Helvetica Neue",Arial; | |
| } | |
| *{box-sizing:border-box} body{margin:0;background:radial-gradient(900px 500px at 10% -10%, #07314a, transparent),var(--bg);color:var(--text);font-family:var(--sans)} | |
| .container{max-width:1100px;margin:0 auto;padding:1.25rem} | |
| header{padding-top:1.5rem} h1{margin:.2rem 0 .4rem;font-size:1.9rem} | |
| .sub{color:var(--muted);margin:.25rem 0 1rem} | |
| .card{background:linear-gradient(180deg,var(--card1),var(--card2));border:1px solid var(--border);border-radius:14px;padding:1rem;box-shadow:0 10px 40px rgba(0,0,0,.35)} | |
| label span{color:var(--muted);font-size:.9rem} | |
| select,textarea{width:100%;border-radius:10px;border:1px solid var(--border);background:#0a1220;color:var(--text);padding:.7rem .85rem;outline:none} | |
| select:focus,textarea:focus{border-color:var(--accent)} | |
| .controls{display:grid;gap:.8rem;margin-bottom:1rem} | |
| .row{display:flex;gap:.75rem;align-items:center} | |
| .status{color:var(--muted)} | |
| .grid{display:grid;gap:1rem;grid-template-columns:1fr} | |
| @media (min-width:900px){.grid{grid-template-columns:1fr 1fr}} | |
| .head{display:block;align-items:center;justify-content:space-between;margin-bottom:0rem} | |
| .tokens{display:flex;flex-wrap:wrap;gap:.5rem;max-height:360px;overflow:auto;padding:.25rem} | |
| .chip{border:1px solid var(--chip-border);background:var(--chip);padding:.35rem .5rem;border-radius:10px;font-family:var(--mono);font-size:.9rem;transition:background .12s,border-color .12s} | |
| .chip:hover{background:var(--chip-hover);border-color:var(--accent)} | |
| .chip.active{outline:2px solid var(--accent)} | |
| .chip.special {border-color: #38bdf8;background: #0b2235;} | |
| pre.ids{font-family:var(--mono);background:#0a1220;border:1px solid var(--border);border-radius:10px;padding:.75rem;max-height:360px;overflow:auto;white-space:pre-wrap} | |
| .caption{color:var(--muted);font-size:.9rem;margin-top:0rem;margin-bottom:.75rem;} | |
| footer{color:var(--muted);text-align:center;padding:1.25rem 0 2rem} | |
| a{color:var(--accent)} | |
| </style> | |
| </head> | |
| <body> | |
| <header class="container"> | |
| <h1>Token Visualizer</h1> | |
| <p class="sub">Enter any text and see how AI turns it into tokens and IDs, the building blocks of its thinking.</p> | |
| </header> | |
| <main class="container"> | |
| <section class="card controls"> | |
| <label> | |
| <span>Model</span> | |
| <select id="model"> | |
| <!-- Tip: keep this first so the demo works instantly once you upload /assets/gpt2/* --> | |
| <option value="local:gpt2">GPT-2 (local, fast)</option> | |
| <option value="Xenova/llama2-tokenizer">Llama-2 (Hub)</option> | |
| <option value="Xenova/mistral-tokenizer">Mistral (Hub)</option> | |
| <option value="Xenova/gemma-tokenizer">Gemma (Hub)</option> | |
| <option value="Xenova/bert-base-uncased">BERT Base Uncased (Hub)</option> | |
| </select> | |
| </label> | |
| <label> | |
| <span>Text</span> | |
| <textarea id="input" rows="3">Curiosity propelled the cat to unfathomable heights.</textarea> | |
| </label> | |
| <div class="row"> | |
| <span id="status" class="status">Loading tokenizer…</span> | |
| </div> | |
| </section> | |
| <section class="grid"> | |
| <article class="card"> | |
| <div class="head"><h3>Tokens</h3></div> | |
| <p class="caption">The smallest language units the model works with.</p> | |
| <div id="tokens" class="tokens"></div> | |
| </article> | |
| <article class="card"> | |
| <div class="head"><h3>Token IDs</h3></div> | |
| <p class="caption">Their numeric form inside the model’s computations.</p> | |
| <pre id="ids" class="ids"></pre> | |
| </article> | |
| </section> | |
| </main> | |
| <footer class="container"> | |
| <small>Built by Peter Adams • Powered in your browser by <a href="https://github.com/xenova/transformers.js" target="_blank" rel="noreferrer">Transformers.js</a></small> | |
| </footer> | |
| <!-- Minimal, robust script (no copy/export) --> | |
| <script type="module"> | |
| // Prefer keeping all requests on huggingface.co to avoid CORS/VPN issues. | |
| // Option 1 (simple): CDN import (works on many networks) | |
| const tf = await import('https://cdn.jsdelivr.net/npm/@xenova/[email protected]'); | |
| // Option 2 (bulletproof): self-host the file in your Space and use: | |
| // const tf = await import('./assets/vendor/transformers.min.js'); | |
| tf.env.useBrowserCache = true; | |
| tf.env.allowLocalModels = true; // <-- REQUIRED for local folder paths | |
| tf.env.localModelPath = '/'; | |
| const $ = s => document.querySelector(s); | |
| const modelSel = $('#model'); | |
| const inputEl = $('#input'); | |
| const statusEl = $('#status'); | |
| const tokensEl = $('#tokens'); | |
| const idsEl = $('#ids'); | |
| // Single state object; never reassign | |
| const state = { tokens: [], ids: [] }; | |
| let tokenizer = null; | |
| let runId = 0; | |
| const status = (msg) => { statusEl.textContent = msg; }; | |
| const debounce = (fn, ms=200) => { let t; return (...a)=>{ clearTimeout(t); t=setTimeout(()=>fn(...a), ms); }; }; | |
| async function loadTokenizer(modelId){ | |
| status('Loading tokenizer…'); | |
| try { | |
| if (modelId === 'local:gpt2') { | |
| // Note: no double slashes, no /resolve/main – just your folder. | |
| tokenizer = await tf.AutoTokenizer.from_pretrained('assets/gpt2'); | |
| } else { | |
| tokenizer = await tf.AutoTokenizer.from_pretrained(modelId); | |
| } | |
| status('Tokenizer ready.'); | |
| } catch (e) { | |
| console.error('Tokenizer load failed:', e); | |
| tokenizer = null; | |
| status('Failed to load tokenizer (network blocked or slow). Try GPT-2 or a different VPN route.'); | |
| } | |
| } | |
| async function tokenize(){ | |
| const myRun = ++runId; | |
| if (!tokenizer) { | |
| await loadTokenizer(modelSel.value); | |
| if (!tokenizer) { render(); return; } | |
| } | |
| // Make sure we always pass a string to encode() | |
| const text = String(inputEl.value ?? '').trim(); | |
| if (!text) { | |
| state.tokens = []; | |
| state.ids = []; | |
| render(); | |
| status('Type to tokenize…'); | |
| return; | |
| } | |
| status('Tokenizing…'); | |
| try { | |
| const enc = await tokenizer.encode(text); // include specials (default) | |
| // Handle both array/object return shapes | |
| const ids = Array.isArray(enc) | |
| ? enc | |
| : (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || []; | |
| // Map special IDs -> special token strings (if available) | |
| const specialIds = Array.from(tokenizer.all_special_ids || []); | |
| const specialTokens = Array.from(tokenizer.all_special_tokens || []); | |
| const idToSpecial = new Map(specialIds.map((id, i) => [id, specialTokens[i]])); | |
| // Build token strings for every ID (specials included) | |
| let tokens = []; | |
| if (typeof tokenizer.convert_ids_to_tokens === 'function') { | |
| tokens = tokenizer.convert_ids_to_tokens(ids); | |
| } else if (typeof tokenizer.id_to_token === 'function') { | |
| tokens = ids.map(id => tokenizer.id_to_token(id)); | |
| } else if (!Array.isArray(enc) && Array.isArray(enc.tokens)) { | |
| tokens = enc.tokens; | |
| } else { | |
| // Fallback: decode each ID as a single-piece token | |
| tokens = ids.map(id => | |
| tokenizer.decode([id], { | |
| // we WANT specials in the stream; decode may return "" for them | |
| skip_special_tokens: false, | |
| clean_up_tokenization_spaces: false, | |
| }) | |
| ); | |
| } | |
| // Ensure specials are visible: if a special token decodes to empty, | |
| // replace it with its canonical name or a generic tag. | |
| tokens = tokens.map((tok, i) => { | |
| const id = ids[i]; | |
| if (tok && tok.length) return tok; | |
| if (idToSpecial.has(id)) return idToSpecial.get(id); // e.g., <|endoftext|> for GPT-2 | |
| return `<special:${id}>`; | |
| }); | |
| if (myRun !== runId) return; | |
| state.tokens = tokens; | |
| state.ids = ids; // include specials in the count | |
| render(); | |
| status(`Done. ${state.tokens.length} tokens.`); | |
| } catch (e) { | |
| console.error('Tokenize failed:', e); | |
| render(); | |
| status('Error tokenizing. See console.'); | |
| } | |
| } | |
| function render(){ | |
| const tokens = Array.isArray(state.tokens) ? state.tokens : []; | |
| const ids = Array.isArray(state.ids) ? state.ids : []; | |
| const specialSet = new Set(tokenizer.all_special_ids || []); | |
| tokensEl.innerHTML = ''; | |
| tokens.forEach((tok, i) => { | |
| const chip = document.createElement('span'); | |
| chip.className = 'chip'; | |
| if (specialSet.has(ids[i])) chip.classList.add('special'); // <-- highlight specials | |
| chip.dataset.i = i; | |
| chip.textContent = tok; | |
| chip.addEventListener('mouseenter', ()=>highlight(i, true)); | |
| chip.addEventListener('mouseleave', ()=>highlight(i, false)); | |
| tokensEl.appendChild(chip); | |
| }); | |
| idsEl.textContent = ids.join(' '); | |
| if (tokens.length === 0) status('Type to tokenize…'); | |
| } | |
| function highlight(i, on){ | |
| const ids = Array.isArray(state.ids) ? state.ids : []; | |
| if (!ids.length) return; | |
| const parts = ids.map((id, idx) => (idx === i && on) ? `[${id}]` : String(id)); | |
| idsEl.textContent = parts.join(' '); | |
| const chip = tokensEl.querySelector(`[data-i="${i}"]`); | |
| if (chip) chip.classList.toggle('active', on); | |
| } | |
| const debounced = debounce(tokenize, 200); | |
| inputEl.addEventListener('input', debounced); | |
| modelSel.addEventListener('change', async ()=>{ | |
| tokenizer = null; // force reload | |
| await loadTokenizer(modelSel.value); | |
| tokenize(); | |
| }); | |
| // Initial load | |
| await loadTokenizer(modelSel.value); | |
| tokenize(); | |
| </script> | |
| </body> | |
| </html> | |