





import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							__lowercase			(     snake_case_   ,							unittest.TestCase       ):
      __UpperCAmelCase		      =						None
      __UpperCAmelCase		      =						BloomTokenizerFast
      __UpperCAmelCase		      =						BloomTokenizerFast
      __UpperCAmelCase		      =						True
      __UpperCAmelCase		      =						False
      __UpperCAmelCase		      =						'''tokenizer_file'''
      __UpperCAmelCase		      =						{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}






      def 			_a						(		self)     ->       Tuple:
       super().setUp()
       __snake_case         =			BloomTokenizerFast.from_pretrained('bigscience/tokenizer')
       tokenizer.save_pretrained(self.tmpdirname)






      def 			_a						(		self			,							**lowercase_)     ->       Tuple:
       kwargs.update(self.special_tokens_map)
       return BloomTokenizerFast.from_pretrained(self.tmpdirname			,							**__snake_case)






      def 			_a						(		self)     ->       Union[str, Any]:
       __snake_case         =			self.get_rust_tokenizer()

       __snake_case         =			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
       __snake_case         =			[[2_1_7_5, 2_3_7_1_4, 7_3_1_7_3, 1_4_4_2_5_2, 2], [7_7, 1_3_2_6_1_9, 3_4_7_8, 3_6_8, 1_0_9_5_8_6, 3_5_4_3_3, 2]]

       __snake_case         =			tokenizer.batch_encode_plus(__snake_case)['input_ids']
       self.assertListEqual(__snake_case			,							__snake_case)

       __snake_case         =			tokenizer.batch_decode(__snake_case)
       self.assertListEqual(__snake_case			,							__snake_case)






      def 			_a						(		self			,							lowercase_=6)     ->       int:
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
         __snake_case         =			self.rust_tokenizer_class.from_pretrained(__snake_case			,							**__snake_case)
         # tokenizer_r.pad_token = None # Hotfixing padding = None
         # Simple input
         __snake_case         =			'This is a simple input'
         __snake_case         =			['This is a simple input 1', 'This is a simple input 2']
         __snake_case         =			('This is a simple input', 'This is a pair')
         __snake_case         =			[
             ('This is a simple input 1', 'This is a simple input 2'),
             ('This is a simple pair 1', 'This is a simple pair 2'),
         ]

         # Simple input tests
         try:
          tokenizer_r.encode(__snake_case			,							max_length=__snake_case)
          tokenizer_r.encode_plus(__snake_case			,							max_length=__snake_case)

          tokenizer_r.batch_encode_plus(__snake_case			,							max_length=__snake_case)
          tokenizer_r.encode(__snake_case			,							max_length=__snake_case)
          tokenizer_r.batch_encode_plus(__snake_case			,							max_length=__snake_case)
         except ValueError:
          self.fail('Bloom Tokenizer should be able to deal with padding')

         __snake_case         =			None  # Hotfixing padding = None
         self.assertRaises(__snake_case			,							tokenizer_r.encode			,							__snake_case			,							max_length=__snake_case			,							padding='max_length')

         # Simple input
         self.assertRaises(__snake_case			,							tokenizer_r.encode_plus			,							__snake_case			,							max_length=__snake_case			,							padding='max_length')

         # Simple input
         self.assertRaises(
             __snake_case			,							tokenizer_r.batch_encode_plus			,							__snake_case			,							max_length=__snake_case			,							padding='max_length'			,							)

         # Pair input
         self.assertRaises(__snake_case			,							tokenizer_r.encode			,							__snake_case			,							max_length=__snake_case			,							padding='max_length')

         # Pair input
         self.assertRaises(__snake_case			,							tokenizer_r.encode_plus			,							__snake_case			,							max_length=__snake_case			,							padding='max_length')

         # Pair input
         self.assertRaises(
             __snake_case			,							tokenizer_r.batch_encode_plus			,							__snake_case			,							max_length=__snake_case			,							padding='max_length'			,							)






      def 			_a						(		self)     ->       Tuple:
       __snake_case         =			self.get_rust_tokenizer()
       __snake_case         =			load_dataset('xnli'			,							'all_languages'			,							split='test'			,							streaming=__snake_case)

       __snake_case         =			next(iter(__snake_case))['premise']  # pick up one data
       __snake_case         =			list(sample_data.values())

       __snake_case         =			list(map(tokenizer.encode			,							__snake_case))
       __snake_case         =			[tokenizer.decode(__snake_case			,							clean_up_tokenization_spaces=__snake_case) for x in output_tokens]
       self.assertListEqual(__snake_case			,							__snake_case)







      def 			_a						(		self)     ->       Any:
       # The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
       # any sequence length constraints. This test of the parent class will fail since it relies on the
       # maximum sequence length of the positoonal embeddings.
       self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)			,							1)
       self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])			,							1)








'''simple docstring'''






import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin




class 					UpperCAmelCase	(     snake_case_				,						unittest.TestCase  ):
      _lowercase:     Optional[int]			=  ProphetNetTokenizer
      _lowercase:     Union[str, Any]			=  False




      def 							lowercase__       (     self     :					Dict    )							-> Optional[int]:
            super().setUp()

            _lowerCAmelCase					=				[
                """[UNK]""",
                """[CLS]""",
                """[SEP]""",
                """[PAD]""",
                """[MASK]""",
                """want""",
                """##want""",
                """##ed""",
                """wa""",
                """un""",
                """runn""",
                """##ing""",
                """,""",
                """low""",
                """lowest""",
            ]
            _lowerCAmelCase					=				os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["""vocab_file"""]    )
            with open(self.vocab_file     ,     """w"""     ,     encoding="""utf-8"""    ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )




      def 							lowercase__       (     self     :					Tuple     ,     __snake_case     :					str    )							-> List[str]:
            _lowerCAmelCase					=				"""UNwant\u00E9d,running"""
            _lowerCAmelCase					=				"""unwanted, running"""
            return input_text, output_text




      def 							lowercase__       (     self     :					Union[str, Any]    )							-> Optional[int]:
            _lowerCAmelCase					=				self.tokenizer_class(self.vocab_file    )

            _lowerCAmelCase					=				tokenizer.tokenize("""UNwant\u00E9d,running"""    )
            self.assertListEqual(__snake_case     ,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]    )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case    )     ,     [9, 6, 7, 12, 10, 11]    )




      def 							lowercase__       (     self     :					List[Any]    )							-> List[str]:
            _lowerCAmelCase					=				BasicTokenizer()

            self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )     ,     ["""ah""", """\u535A""", """\u63A8""", """zz"""]    )




      def 							lowercase__       (     self     :					Optional[Any]    )							-> List[Any]:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )     ,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )     ,     ["""hello"""]    )




      def 							lowercase__       (     self     :					Dict    )							-> int:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     strip_accents=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )     ,     ["""h\u00E9llo"""]    )




      def 							lowercase__       (     self     :					Dict    )							-> int:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     strip_accents=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )     ,     ["""hello"""]    )




      def 							lowercase__       (     self     :					Dict    )							-> Optional[Any]:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )     ,     ["""hello"""]    )




      def 							lowercase__       (     self     :					str    )							-> List[str]:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )     ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )




      def 							lowercase__       (     self     :					Union[str, Any]    )							-> str:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     strip_accents=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )




      def 							lowercase__       (     self     :					Tuple    )							-> str:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     strip_accents=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )




      def 							lowercase__       (     self     :					Dict    )							-> Optional[int]:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     never_split=["""[UNK]"""]    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )     ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )




      def 							lowercase__       (     self     :					Any    )							-> Dict:
            _lowerCAmelCase					=				["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

            _lowerCAmelCase					=				{}
            for i, token in enumerate(__snake_case    ):
                  _lowerCAmelCase					=				i
            _lowerCAmelCase					=				WordpieceTokenizer(vocab=__snake_case     ,     unk_token="""[UNK]"""    )

            self.assertListEqual(tokenizer.tokenize(""""""    )     ,     []    )

            self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )     ,     ["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

            self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )     ,     ["""[UNK]""", """runn""", """##ing"""]    )




      @require_torch
      def 							lowercase__       (     self     :					Optional[int]    )							-> Optional[Any]:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""    )

            _lowerCAmelCase					=				["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
            _lowerCAmelCase					=				[10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
            _lowerCAmelCase					=				tokenizer(__snake_case     ,     padding=__snake_case     ,     return_tensors="""pt"""    )
            self.assertIsInstance(__snake_case     ,     __snake_case    )
            _lowerCAmelCase					=				list(batch.input_ids.numpy()[0]    )
            self.assertListEqual(__snake_case     ,     __snake_case    )

            self.assertEqual((2, 9)     ,     batch.input_ids.shape    )
            self.assertEqual((2, 9)     ,     batch.attention_mask.shape    )




      def 							lowercase__       (     self     :					List[Any]    )							-> Any:
            self.assertTrue(_is_whitespace(""" """    )    )
            self.assertTrue(_is_whitespace("""\t"""    )    )
            self.assertTrue(_is_whitespace("""\r"""    )    )
            self.assertTrue(_is_whitespace("""\n"""    )    )
            self.assertTrue(_is_whitespace("""\u00A0"""    )    )

            self.assertFalse(_is_whitespace("""A"""    )    )
            self.assertFalse(_is_whitespace("""-"""    )    )




      def 							lowercase__       (     self     :					str    )							-> Union[str, Any]:
            self.assertTrue(_is_control("""\u0005"""    )    )

            self.assertFalse(_is_control("""A"""    )    )
            self.assertFalse(_is_control(""" """    )    )
            self.assertFalse(_is_control("""\t"""    )    )
            self.assertFalse(_is_control("""\r"""    )    )




      def 							lowercase__       (     self     :					int    )							-> int:
            self.assertTrue(_is_punctuation("""-"""    )    )
            self.assertTrue(_is_punctuation("""$"""    )    )
            self.assertTrue(_is_punctuation("""`"""    )    )
            self.assertTrue(_is_punctuation("""."""    )    )

            self.assertFalse(_is_punctuation("""A"""    )    )
            self.assertFalse(_is_punctuation(""" """    )    )




      @slow
      def 							lowercase__       (     self     :					int    )							-> Optional[int]:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""    )

            _lowerCAmelCase					=				tokenizer.encode("""sequence builders"""     ,     add_special_tokens=__snake_case    )
            _lowerCAmelCase					=				tokenizer.encode("""multi-sequence build"""     ,     add_special_tokens=__snake_case    )

            _lowerCAmelCase					=				tokenizer.build_inputs_with_special_tokens(__snake_case    )
            _lowerCAmelCase					=				tokenizer.build_inputs_with_special_tokens(__snake_case     ,     __snake_case    )

            assert encoded_sentence == text + [1_02]
            assert encoded_pair == text + [1_02] + text_a + [1_02]






