'''simple docstring'''




import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 	_lowercase(  a_ ,unittest.TestCase					):



   """simple docstring"""





   __lowerCamelCase     =	None
   __lowerCamelCase     =	BloomTokenizerFast
   __lowerCamelCase     =	BloomTokenizerFast
   __lowerCamelCase     =	True
   __lowerCamelCase     =	False
   __lowerCamelCase     =	'''tokenizer_file'''
   __lowerCamelCase     =	{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}





   def       snake_case						(				self:     Tuple						):
    super().setUp()
    __UpperCAmelCase								=       BloomTokenizerFast.from_pretrained('bigscience/tokenizer'						)
    tokenizer.save_pretrained(self.tmpdirname						)





   def       snake_case						(				self:     List[Any]      ,**a:     List[str]						):
    kwargs.update(self.special_tokens_map						)
    return BloomTokenizerFast.from_pretrained(self.tmpdirname      ,**UpperCamelCase_						)





   def       snake_case						(				self:     str						):
    __UpperCAmelCase								=       self.get_rust_tokenizer()

    __UpperCAmelCase								=       ['The quick brown fox</s>', 'jumps over the lazy dog</s>']
    __UpperCAmelCase								=       [[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

    __UpperCAmelCase								=       tokenizer.batch_encode_plus(UpperCamelCase_						)['input_ids']
    self.assertListEqual(UpperCamelCase_      ,UpperCamelCase_						)

    __UpperCAmelCase								=       tokenizer.batch_decode(UpperCamelCase_						)
    self.assertListEqual(UpperCamelCase_      ,UpperCamelCase_						)





   def       snake_case						(				self:     Tuple      ,a:     Tuple=6						):

    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
      __UpperCAmelCase								=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase_      ,**UpperCamelCase_						)
      # tokenizer_r.pad_token = None # Hotfixing padding = None
      # Simple input
      __UpperCAmelCase								=       'This is a simple input'
      __UpperCAmelCase								=       ['This is a simple input 1', 'This is a simple input 2']
      __UpperCAmelCase								=       ('This is a simple input', 'This is a pair')
      __UpperCAmelCase								=       [
          ('This is a simple input 1', 'This is a simple input 2'),
          ('This is a simple pair 1', 'This is a simple pair 2'),
      ]

      # Simple input tests
      try:
       tokenizer_r.encode(UpperCamelCase_      ,max_length=UpperCamelCase_						)
       tokenizer_r.encode_plus(UpperCamelCase_      ,max_length=UpperCamelCase_						)

       tokenizer_r.batch_encode_plus(UpperCamelCase_      ,max_length=UpperCamelCase_						)
       tokenizer_r.encode(UpperCamelCase_      ,max_length=UpperCamelCase_						)
       tokenizer_r.batch_encode_plus(UpperCamelCase_      ,max_length=UpperCamelCase_						)
      except ValueError:
       self.fail('Bloom Tokenizer should be able to deal with padding'						)

      __UpperCAmelCase								=       None  # Hotfixing padding = None
      self.assertRaises(UpperCamelCase_      ,tokenizer_r.encode      ,UpperCamelCase_      ,max_length=UpperCamelCase_      ,padding='max_length'						)

      # Simple input
      self.assertRaises(UpperCamelCase_      ,tokenizer_r.encode_plus      ,UpperCamelCase_      ,max_length=UpperCamelCase_      ,padding='max_length'						)

      # Simple input
      self.assertRaises(
          UpperCamelCase_      ,tokenizer_r.batch_encode_plus      ,UpperCamelCase_      ,max_length=UpperCamelCase_      ,padding='max_length'      ,)

      # Pair input
      self.assertRaises(UpperCamelCase_      ,tokenizer_r.encode      ,UpperCamelCase_      ,max_length=UpperCamelCase_      ,padding='max_length'						)

      # Pair input
      self.assertRaises(UpperCamelCase_      ,tokenizer_r.encode_plus      ,UpperCamelCase_      ,max_length=UpperCamelCase_      ,padding='max_length'						)

      # Pair input
      self.assertRaises(
          UpperCamelCase_      ,tokenizer_r.batch_encode_plus      ,UpperCamelCase_      ,max_length=UpperCamelCase_      ,padding='max_length'      ,)





   def       snake_case						(				self:     Dict						):
    __UpperCAmelCase								=       self.get_rust_tokenizer()
    __UpperCAmelCase								=       load_dataset('xnli'      ,'all_languages'      ,split='test'      ,streaming=UpperCamelCase_						)

    __UpperCAmelCase								=       next(iter(UpperCamelCase_						)						)['premise']  # pick up one data
    __UpperCAmelCase								=       list(sample_data.values()						)

    __UpperCAmelCase								=       list(map(tokenizer.encode      ,UpperCamelCase_						)						)
    __UpperCAmelCase								=       [tokenizer.decode(UpperCamelCase_      ,clean_up_tokenization_spaces=UpperCamelCase_						) for x in output_tokens]
    self.assertListEqual(UpperCamelCase_      ,UpperCamelCase_						)





   def       snake_case						(				self:     Tuple						):
    self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map						)      ,1						)
    self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()						)[0]						)      ,1						)











"""simple docstring"""
from ..utils import DummyObject, requires_backends



class        UpperCamelCase__ (				metaclass=a_):







    """simple docstring"""
    __UpperCAmelCase           = ["""flax""", """transformers"""]







    def __init__(		self :	List[Any]				,    *UpperCamelCase_ :	str				,    **UpperCamelCase_ :	Any ):





           '''simple docstring'''

           requires_backends(self				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	Optional[int]				,    *UpperCamelCase_ :	Tuple				,    **UpperCamelCase_ :	List[Any] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	str				,    *UpperCamelCase_ :	Optional[Any]				,    **UpperCamelCase_ :	Dict ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )



class        UpperCamelCase__ (				metaclass=a_):







    """simple docstring"""
    __UpperCAmelCase           = ["""flax""", """transformers"""]







    def __init__(		self :	List[Any]				,    *UpperCamelCase_ :	str				,    **UpperCamelCase_ :	str ):





           '''simple docstring'''

           requires_backends(self				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	Optional[int]				,    *UpperCamelCase_ :	Optional[int]				,    **UpperCamelCase_ :	List[str] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	Tuple				,    *UpperCamelCase_ :	List[str]				,    **UpperCamelCase_ :	Optional[Any] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )



class        UpperCamelCase__ (				metaclass=a_):







    """simple docstring"""
    __UpperCAmelCase           = ["""flax""", """transformers"""]







    def __init__(		self :	Optional[int]				,    *UpperCamelCase_ :	List[Any]				,    **UpperCamelCase_ :	List[str] ):





           '''simple docstring'''

           requires_backends(self				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	str				,    *UpperCamelCase_ :	List[str]				,    **UpperCamelCase_ :	Tuple ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	Tuple				,    *UpperCamelCase_ :	Any				,    **UpperCamelCase_ :	List[str] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )



class        UpperCamelCase__ (				metaclass=a_):







    """simple docstring"""
    __UpperCAmelCase           = ["""flax""", """transformers"""]







    def __init__(		self :	Optional[Any]				,    *UpperCamelCase_ :	Optional[Any]				,    **UpperCamelCase_ :	Dict ):





           '''simple docstring'''

           requires_backends(self				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	str				,    *UpperCamelCase_ :	Dict				,    **UpperCamelCase_ :	List[str] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	str				,    *UpperCamelCase_ :	Optional[Any]				,    **UpperCamelCase_ :	Union[str, Any] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )