Update index.html

index.html

@@ -150,9 +150,12 @@
       try {
         // 1) Get IDs (no options arg!)
         const enc = await tokenizer.encode(text);
+
+        console.log('enc =', enc); // See what the tokenizer returns
         // robustly pluck ids out of whatever shape the lib returns
         let ids = (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || [];
-    
+        console.log('ids=', ids, 'specials=', tokenizer.all_special_ids);
+        
         // 2) Drop special tokens (e.g., BOS/EOS) for the demo
         const specials = new Set(tokenizer.all_special_ids || []);
         const idsNoSpecials = ids.filter(id => !specials.has(id));