fix tokenize

util.py

@@ -46,8 +46,10 @@ def tokenize(text, tokenizer_type, color_num=5):
             token_bytes = bytes(token_str, "utf-8")
             # json_dumps = json.dumps(token_str)
         else:
-            logger.error(f"typeError for token {token_id} with {type(token)} " + json.dumps({"text": text, "tokenizer_type": tokenizer_type}, ensure_ascii=False))
-            continue
+            logger.error(f"{idx}: wrong type for token {token_id} {type(token)} " + json.dumps({"text": text, "tokenizer_type": tokenizer_type}, ensure_ascii=False))
+            token_str = token
+            token_bytes = token
+            # continue
 
         # ⭐
         table.append(
@@ -61,7 +63,7 @@ def tokenize(text, tokenizer_type, color_num=5):
         )
 
     table_df = pd.DataFrame(table)
-    logger.info(f"Tokens={table[:2]}")
+    logger.info(f"tokenizer_type={tokenizer_type}, Tokens={table[:4]}")
     # print(table_df)
 
     return gr.update(value=pos_tokens, label=f"Tokens: {len(encoding)}"), table_df

vocab/gpt_35_turbo/__init__.py

@@ -31,10 +31,11 @@ def convert_ids_to_tokens(self, tokens, skip_special_tokens=False):
     """
     try:
         return self.decode_tokens_bytes(tokens)
-    except:
+    except Exception as e:
         # 什么要返回None？见zh_util.py
         # 16个空闲id, 100256 100261-100275
-        return [None for token in tokens]
+        logger.error(e)
+        return [None for _ in tokens]
 
 
 def get_vocab(self, token_type="str"):