Better handling of bad characters

app.py

@@ -34,19 +34,23 @@ def get_windows_batched(
 
 BAD_CHAR = chr(0xfffd)
 
-def ids_to_readable_tokens(tokenizer, ids, strip_whitespace=False):
+def ids_to_readable_tokens(tokenizer, ids, strip_whitespace=False, bad_token_replacement=BAD_CHAR):
     cur_ids = []
     result = []
+    bad_ids = [
+        _id for _id in tokenizer.convert_tokens_to_ids([BAD_CHAR, " " + BAD_CHAR])
+        if _id != tokenizer.unk_token_id
+    ]
     for idx in ids:
         cur_ids.append(idx)
         decoded = tokenizer.decode(cur_ids)
-        if BAD_CHAR not in decoded:
+        if BAD_CHAR not in decoded or any(_id in cur_ids for _id in bad_ids):
             if strip_whitespace:
                 decoded = decoded.strip()
             result.append(decoded)
             del cur_ids[:]
         else:
-            result.append("")
+            result.append(bad_token_replacement)
     return result
 
 def nll_score(logprobs, labels):