


"""simple docstring"""
import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 	lowercase__				(					A_ ,unittest.TestCase				):
   __UpperCAmelCase							       =					None
   __UpperCAmelCase							       =					BloomTokenizerFast
   __UpperCAmelCase							       =					BloomTokenizerFast
   __UpperCAmelCase							       =					True
   __UpperCAmelCase							       =					False
   __UpperCAmelCase							       =					'''tokenizer_file'''
   __UpperCAmelCase							       =					{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}





   def 		UpperCamelCase_  (							self)  ->  int:
      super().setUp()
      _lowerCamelCase	:					Optional[int]   =		BloomTokenizerFast.from_pretrained("""bigscience/tokenizer""")
      tokenizer.save_pretrained(self.tmpdirname)





   def 		UpperCamelCase_  (							self      ,   **SCREAMING_SNAKE_CASE)  ->  str:
      kwargs.update(self.special_tokens_map)
      return BloomTokenizerFast.from_pretrained(self.tmpdirname      ,   **SCREAMING_SNAKE_CASE)





   def 		UpperCamelCase_  (							self)  ->  str:
      _lowerCamelCase	:					Union[str, Any]   =		self.get_rust_tokenizer()

      _lowerCamelCase	:					List[str]   =		["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
      _lowerCamelCase	:					Union[str, Any]   =		[[2175, 2_3714, 7_3173, 14_4252, 2], [77, 13_2619, 3478, 368, 10_9586, 3_5433, 2]]

      _lowerCamelCase	:					Any   =		tokenizer.batch_encode_plus(SCREAMING_SNAKE_CASE)["""input_ids"""]
      self.assertListEqual(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)

      _lowerCamelCase	:					List[str]   =		tokenizer.batch_decode(SCREAMING_SNAKE_CASE)
      self.assertListEqual(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE=6)  ->  str:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
            _lowerCamelCase	:					Optional[int]   =		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE      ,   **SCREAMING_SNAKE_CASE)
            # tokenizer_r.pad_token = None # Hotfixing padding = None
            # Simple input
            _lowerCamelCase	:					Optional[Any]   =		"""This is a simple input"""
            _lowerCamelCase	:					List[Any]   =		["""This is a simple input 1""", """This is a simple input 2"""]
            _lowerCamelCase	:					List[Any]   =		("""This is a simple input""", """This is a pair""")
            _lowerCamelCase	:					int   =		[
                ("""This is a simple input 1""", """This is a simple input 2"""),
                ("""This is a simple pair 1""", """This is a simple pair 2"""),
            ]

            # Simple input tests
            try:
               tokenizer_r.encode(SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE)
               tokenizer_r.encode_plus(SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE)

               tokenizer_r.batch_encode_plus(SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE)
               tokenizer_r.encode(SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE)
               tokenizer_r.batch_encode_plus(SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE)
            except ValueError:
               self.fail("""Bloom Tokenizer should be able to deal with padding""")

            _lowerCamelCase	:					Union[str, Any]   =		None  # Hotfixing padding = None
            self.assertRaises(SCREAMING_SNAKE_CASE      ,   tokenizer_r.encode      ,   SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE      ,   padding="""max_length""")

            # Simple input
            self.assertRaises(SCREAMING_SNAKE_CASE      ,   tokenizer_r.encode_plus      ,   SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE      ,   padding="""max_length""")

            # Simple input
            self.assertRaises(
                SCREAMING_SNAKE_CASE      ,   tokenizer_r.batch_encode_plus      ,   SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE      ,   padding="""max_length"""      ,   )

            # Pair input
            self.assertRaises(SCREAMING_SNAKE_CASE      ,   tokenizer_r.encode      ,   SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE      ,   padding="""max_length""")

            # Pair input
            self.assertRaises(SCREAMING_SNAKE_CASE      ,   tokenizer_r.encode_plus      ,   SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE      ,   padding="""max_length""")

            # Pair input
            self.assertRaises(
                SCREAMING_SNAKE_CASE      ,   tokenizer_r.batch_encode_plus      ,   SCREAMING_SNAKE_CASE      ,   max_length=SCREAMING_SNAKE_CASE      ,   padding="""max_length"""      ,   )





   def 		UpperCamelCase_  (							self)  ->  Union[str, Any]:
      _lowerCamelCase	:					Tuple   =		self.get_rust_tokenizer()
      _lowerCamelCase	:					int   =		load_dataset("""xnli"""      ,   """all_languages"""      ,   split="""test"""      ,   streaming=SCREAMING_SNAKE_CASE)

      _lowerCamelCase	:					List[Any]   =		next(iter(SCREAMING_SNAKE_CASE))["""premise"""]  # pick up one data
      _lowerCamelCase	:					Optional[int]   =		list(sample_data.values())

      _lowerCamelCase	:					str   =		list(map(tokenizer.encode      ,   SCREAMING_SNAKE_CASE))
      _lowerCamelCase	:					List[Any]   =		[tokenizer.decode(SCREAMING_SNAKE_CASE      ,   clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE) for x in output_tokens]
      self.assertListEqual(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)





   def 		UpperCamelCase_  (							self)  ->  Tuple:
      # The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
      # any sequence length constraints. This test of the parent class will fail since it relies on the
      # maximum sequence length of the positoonal embeddings.
      self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)      ,   1)
      self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])      ,   1)








'''simple docstring'''


from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_SCREAMING_SNAKE_CASE									=				{
    "configuration_mctct": ["MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MCTCTConfig"],
    "feature_extraction_mctct": ["MCTCTFeatureExtractor"],
    "processing_mctct": ["MCTCTProcessor"],
}


try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _SCREAMING_SNAKE_CASE									=				[
           "MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST",
           "MCTCTForCTC",
           "MCTCTModel",
           "MCTCTPreTrainedModel",
       ]


if TYPE_CHECKING:
       from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
       from .feature_extraction_mctct import MCTCTFeatureExtractor
       from .processing_mctct import MCTCTProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
       import sys

       _SCREAMING_SNAKE_CASE									=				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


