





import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 							_UpperCAmelCase       (       _UpperCamelCase						,       unittest.TestCase       ):

  """simple docstring"""


  a_              =					None
  a_              =					BloomTokenizerFast
  a_              =					BloomTokenizerFast
  a_              =					True
  a_              =					False
  a_              =					"""tokenizer_file"""
  a_              =					{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}






  def 							lowercase						(							self					:						List[Any]	)				->		List[str]:
     super().setUp()
     __lowerCAmelCase				      =      BloomTokenizerFast.from_pretrained('bigscience/tokenizer'	)
     tokenizer.save_pretrained(self.tmpdirname	)






  def 							lowercase						(							self					:						Dict   ,    **lowerCAmelCase_					:						Optional[Any]	)				->		List[Any]:
     kwargs.update(self.special_tokens_map	)
     return BloomTokenizerFast.from_pretrained(self.tmpdirname   ,    **lowerCAmelCase_	)






  def 							lowercase						(							self					:						str	)				->		Dict:
     __lowerCAmelCase				      =      self.get_rust_tokenizer()

     __lowerCAmelCase				      =      ['The quick brown fox</s>', 'jumps over the lazy dog</s>']
     __lowerCAmelCase				      =      [[2_1_7_5, 2_3_7_1_4, 7_3_1_7_3, 1_4_4_2_5_2, 2], [7_7, 1_3_2_6_1_9, 3_4_7_8, 3_6_8, 1_0_9_5_8_6, 3_5_4_3_3, 2]]

     __lowerCAmelCase				      =      tokenizer.batch_encode_plus(lowerCAmelCase_	)['input_ids']
     self.assertListEqual(lowerCAmelCase_   ,    lowerCAmelCase_	)

     __lowerCAmelCase				      =      tokenizer.batch_decode(lowerCAmelCase_	)
     self.assertListEqual(lowerCAmelCase_   ,    lowerCAmelCase_	)






  def 							lowercase						(							self					:						Dict   ,    lowerCAmelCase_					:						List[Any]=6	)				->		int:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
           __lowerCAmelCase				      =      self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_   ,    **lowerCAmelCase_	)
           # tokenizer_r.pad_token = None # Hotfixing padding = None
           # Simple input
           __lowerCAmelCase				      =      'This is a simple input'
           __lowerCAmelCase				      =      ['This is a simple input 1', 'This is a simple input 2']
           __lowerCAmelCase				      =      ('This is a simple input', 'This is a pair')
           __lowerCAmelCase				      =      [
               ('This is a simple input 1', 'This is a simple input 2'),
               ('This is a simple pair 1', 'This is a simple pair 2'),
           ]

           # Simple input tests
           try:
              tokenizer_r.encode(lowerCAmelCase_   ,    max_length=lowerCAmelCase_	)
              tokenizer_r.encode_plus(lowerCAmelCase_   ,    max_length=lowerCAmelCase_	)

              tokenizer_r.batch_encode_plus(lowerCAmelCase_   ,    max_length=lowerCAmelCase_	)
              tokenizer_r.encode(lowerCAmelCase_   ,    max_length=lowerCAmelCase_	)
              tokenizer_r.batch_encode_plus(lowerCAmelCase_   ,    max_length=lowerCAmelCase_	)
           except ValueError:
              self.fail('Bloom Tokenizer should be able to deal with padding'	)

           __lowerCAmelCase				      =      None  # Hotfixing padding = None
           self.assertRaises(lowerCAmelCase_   ,    tokenizer_r.encode   ,    lowerCAmelCase_   ,    max_length=lowerCAmelCase_   ,    padding='max_length'	)

           # Simple input
           self.assertRaises(lowerCAmelCase_   ,    tokenizer_r.encode_plus   ,    lowerCAmelCase_   ,    max_length=lowerCAmelCase_   ,    padding='max_length'	)

           # Simple input
           self.assertRaises(
               lowerCAmelCase_   ,    tokenizer_r.batch_encode_plus   ,    lowerCAmelCase_   ,    max_length=lowerCAmelCase_   ,    padding='max_length'   ,    )

           # Pair input
           self.assertRaises(lowerCAmelCase_   ,    tokenizer_r.encode   ,    lowerCAmelCase_   ,    max_length=lowerCAmelCase_   ,    padding='max_length'	)

           # Pair input
           self.assertRaises(lowerCAmelCase_   ,    tokenizer_r.encode_plus   ,    lowerCAmelCase_   ,    max_length=lowerCAmelCase_   ,    padding='max_length'	)

           # Pair input
           self.assertRaises(
               lowerCAmelCase_   ,    tokenizer_r.batch_encode_plus   ,    lowerCAmelCase_   ,    max_length=lowerCAmelCase_   ,    padding='max_length'   ,    )






  def 							lowercase						(							self					:						List[str]	)				->		Optional[int]:
     __lowerCAmelCase				      =      self.get_rust_tokenizer()
     __lowerCAmelCase				      =      load_dataset('xnli'   ,    'all_languages'   ,    split='test'   ,    streaming=lowerCAmelCase_	)

     __lowerCAmelCase				      =      next(iter(lowerCAmelCase_	)	)['premise']  # pick up one data
     __lowerCAmelCase				      =      list(sample_data.values()	)

     __lowerCAmelCase				      =      list(map(tokenizer.encode   ,    lowerCAmelCase_	)	)
     __lowerCAmelCase				      =      [tokenizer.decode(lowerCAmelCase_   ,    clean_up_tokenization_spaces=lowerCAmelCase_	) for x in output_tokens]
     self.assertListEqual(lowerCAmelCase_   ,    lowerCAmelCase_	)






  def 							lowercase						(							self					:						List[Any]	)				->		Union[str, Any]:
     # The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
     # any sequence length constraints. This test of the parent class will fail since it relies on the
     # maximum sequence length of the positoonal embeddings.
     self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map	)   ,    1	)
     self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()	)[0]	)   ,    1	)








'''simple docstring'''






def   UpperCamelCase__ (						lowerCAmelCase    ):





      """simple docstring"""


      if not isinstance(lowerCAmelCase				,      lowerCAmelCase    ):
            raise ValueError("""Input must be an integer"""    )
      if input_num <= 0:
            raise ValueError("""Input must be positive"""    )
      return sum(
          divisor for divisor in range(1				,      input_num // 2 + 1    ) if input_num % divisor == 0    )


if __name__ == "__main__":
      import doctest

      doctest.testmod()






