Fix invalid error messages.
@@ -144,9 +144,8 @@ class BaseTokenizer:
An Encoding
"""
if sequence is None:
- raise ValueError(
- "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
- )
+ raise ValueError("None input is not valid. Should be a string.")
+
return self._tokenizer.encode(sequence, pair, add_special_tokens)
def encode_batch(
@@ -168,7 +167,7 @@ class BaseTokenizer:
if sequences is None:
raise ValueError(
- "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
+ "None input is not valid. Should be a list of strings or a list of tuple of strings."
)
return self._tokenizer.encode_batch(sequences, add_special_tokens)
@@ -187,9 +186,8 @@ class BaseTokenizer:
The decoded string
"""
if ids is None:
- raise ValueError(
- "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
- )
+ raise ValueError("None input is not valid. Should be a list of integers.")
+
return self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)
def decode_batch(
@@ -208,9 +206,8 @@ class BaseTokenizer:
A list of decoded strings
"""
if sequences is None:
- raise ValueError(
- "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
- )
+ raise ValueError("None input is not valid. Should be list of list of integers.")
+
return self._tokenizer.decode_batch(sequences, skip_special_tokens=skip_special_tokens)
def token_to_id(self, token: str) -> Optional[int]:
