





import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin







class       __A							(   lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   CanineTokenizer
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             super().setUp()
             _lowerCamelCase				:  Union[str, Any]						=		CanineTokenizer()
             tokenizer.save_pretrained(self.tmpdirname)


      @cached_property
      def 					__snake_case      (		self):



             """simple docstring"""





             return CanineTokenizer.from_pretrained('''google/canine-s''')


      def 					__snake_case      (		self   ,				**a__):



             """simple docstring"""





             _lowerCamelCase				:  Dict						=		self.tokenizer_class.from_pretrained(self.tmpdirname   ,				**a__)
             _lowerCamelCase				:  Optional[Any]						=		1024
             return tokenizer


      @require_torch
      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Tuple						=		self.canine_tokenizer
             _lowerCamelCase				:  Optional[int]						=		['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
             # fmt: off
             _lowerCamelCase				:  List[Any]						=		[5_7344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 5_7345, 0, 0, 0, 0]
             # fmt: on
             _lowerCamelCase				:  List[str]						=		tokenizer(a__   ,				padding=a__   ,				return_tensors='''pt''')
             self.assertIsInstance(a__   ,				a__)

             _lowerCamelCase				:  str						=		list(batch.input_ids.numpy()[0])

             self.assertListEqual(a__   ,				a__)

             self.assertEqual((2, 39)   ,				batch.input_ids.shape)
             self.assertEqual((2, 39)   ,				batch.attention_mask.shape)


      @require_torch
      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[Any]						=		self.canine_tokenizer
             _lowerCamelCase				:  List[Any]						=		['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
             _lowerCamelCase				:  str						=		tokenizer(a__   ,				padding=a__   ,				return_tensors='''pt''')
             # check if input_ids, attention_mask and token_type_ids are returned
             self.assertIn('''input_ids'''   ,				a__)
             self.assertIn('''attention_mask'''   ,				a__)
             self.assertIn('''token_type_ids'''   ,				a__)


      @require_torch
      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[int]						=		self.canine_tokenizer
             _lowerCamelCase				:  Tuple						=		[
                 '''What\'s the weater?''',
                 '''It\'s about 25 degrees.''',
             ]
             _lowerCamelCase				:  List[str]						=		tokenizer(
                 text_target=a__   ,				max_length=32   ,				padding='''max_length'''   ,				truncation=a__   ,				return_tensors='''pt''')
             self.assertEqual(32   ,				targets['''input_ids'''].shape[1])


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Dict						=		self.get_tokenizers()
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                           self.assertNotEqual(tokenizer.model_max_length   ,				42)

        # Now let's start the test
             _lowerCamelCase				:  Optional[Any]						=		self.get_tokenizers()
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                           # Isolate this from the other tests because we save additional tokens/etc
                           _lowerCamelCase				:  Union[str, Any]						=		tempfile.mkdtemp()

                           _lowerCamelCase				:  int						=		''' He is very happy, UNwant\u00E9d,running'''
                           _lowerCamelCase				:  Dict						=		tokenizer.encode(a__   ,				add_special_tokens=a__)
                           tokenizer.save_pretrained(a__)

                           _lowerCamelCase				:  Tuple						=		tokenizer.__class__.from_pretrained(a__)
                           _lowerCamelCase				:  int						=		after_tokenizer.encode(a__   ,				add_special_tokens=a__)
                           self.assertListEqual(a__   ,				a__)

                           shutil.rmtree(a__)

             _lowerCamelCase				:  str						=		self.get_tokenizers(model_max_length=42)
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                           # Isolate this from the other tests because we save additional tokens/etc
                           _lowerCamelCase				:  Any						=		tempfile.mkdtemp()

                           _lowerCamelCase				:  str						=		''' He is very happy, UNwant\u00E9d,running'''

                           _lowerCamelCase				:  Optional[Any]						=		tokenizer.additional_special_tokens

                           # We can add a new special token for Canine as follows:
                           _lowerCamelCase				:  Dict						=		chr(0Xe_0_0_7)
                           additional_special_tokens.append(a__)
                           tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens})
                           _lowerCamelCase				:  Tuple						=		tokenizer.encode(a__   ,				add_special_tokens=a__)
                           tokenizer.save_pretrained(a__)

                           _lowerCamelCase				:  Union[str, Any]						=		tokenizer.__class__.from_pretrained(a__)
                           _lowerCamelCase				:  Tuple						=		after_tokenizer.encode(a__   ,				add_special_tokens=a__)
                           self.assertListEqual(a__   ,				a__)
                           self.assertIn(a__   ,				after_tokenizer.additional_special_tokens)
                           self.assertEqual(after_tokenizer.model_max_length   ,				42)

                           _lowerCamelCase				:  Tuple						=		tokenizer.__class__.from_pretrained(a__   ,				model_max_length=43)
                           self.assertEqual(tokenizer.model_max_length   ,				43)

                           shutil.rmtree(a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Dict						=		self.get_tokenizers(do_lower_case=a__)
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                           _lowerCamelCase,	_lowerCamelCase				:  int						=		self.get_clean_sequence(a__)

                           # a special token for Canine can be defined as follows:
                           _lowerCamelCase				:  int						=		0Xe_0_0_5
                           _lowerCamelCase				:  int						=		chr(a__)

                           tokenizer.add_special_tokens({'''cls_token''': special_token})
                           _lowerCamelCase				:  str						=		tokenizer.encode(a__   ,				add_special_tokens=a__)
                           self.assertEqual(len(a__)   ,				1)

                           _lowerCamelCase				:  str						=		tokenizer.decode(ids + encoded_special_token   ,				clean_up_tokenization_spaces=a__)
                           _lowerCamelCase				:  Tuple						=		tokenizer.encode(a__   ,				add_special_tokens=a__)

                           _lowerCamelCase				:  Union[str, Any]						=		tokenizer.encode(a__   ,				add_special_tokens=a__)
                           _lowerCamelCase				:  Union[str, Any]						=		tokenizer.encode(a__   ,				add_special_tokens=a__)
                           self.assertEqual(a__   ,				input_encoded + special_token_id)

                           _lowerCamelCase				:  str						=		tokenizer.decode(a__   ,				skip_special_tokens=a__)
                           self.assertTrue(special_token not in decoded)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		self.get_tokenizers(do_lower_case=a__)
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                           _lowerCamelCase				:  Dict						=		chr(0Xe_0_0_5)
                           _lowerCamelCase				:  Optional[Any]						=		chr(0Xe_0_0_6)

                           # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
                           tokenizer.add_tokens([SPECIAL_TOKEN_1]   ,				special_tokens=a__)
                           # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
                           # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
                           tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]})

                           _lowerCamelCase				:  Any						=		tokenizer.tokenize(a__)
                           _lowerCamelCase				:  List[str]						=		tokenizer.tokenize(a__)

                           self.assertEqual(len(a__)   ,				1)
                           self.assertEqual(len(a__)   ,				1)
                           self.assertEqual(token_a[0]   ,				a__)
                           self.assertEqual(token_a[0]   ,				a__)


      @require_tokenizers
      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Any						=		self.get_tokenizers(do_lower_case=a__)
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                           # a special token for Canine can be defined as follows:
                           _lowerCamelCase				:  List[Any]						=		0Xe_0_0_6
                           _lowerCamelCase				:  List[str]						=		chr(a__)

                           _lowerCamelCase				:  Any						=		AddedToken(a__   ,				lstrip=a__)
                           tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]})

                           with tempfile.TemporaryDirectory() as tmp_dir_name:
                                  tokenizer.save_pretrained(a__)
                                  tokenizer.from_pretrained(a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[int]						=		[]
             if self.test_slow_tokenizer:
                    tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

             if self.test_rust_tokenizer:
                    tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

             for tokenizer_class, tokenizer_utils in tokenizer_list:
                    with tempfile.TemporaryDirectory() as tmp_dir:
                           tokenizer_utils.save_pretrained(a__)

                           with open(os.path.join(a__   ,				'''special_tokens_map.json''')   ,				encoding='''utf-8''') as json_file:
                                  _lowerCamelCase				:  Optional[int]						=		json.load(a__)

                           with open(os.path.join(a__   ,				'''tokenizer_config.json''')   ,				encoding='''utf-8''') as json_file:
                                  _lowerCamelCase				:  Dict						=		json.load(a__)

                           # a special token for Canine can be defined as follows:
                           _lowerCamelCase				:  Optional[Any]						=		0Xe_0_0_6
                           _lowerCamelCase				:  int						=		chr(a__)

                           _lowerCamelCase				:  Tuple						=		[new_token_a]
                           _lowerCamelCase				:  Any						=		[new_token_a]

                           with open(os.path.join(a__   ,				'''special_tokens_map.json''')   ,				'''w'''   ,				encoding='''utf-8''') as outfile:
                                  json.dump(a__   ,				a__)
                           with open(os.path.join(a__   ,				'''tokenizer_config.json''')   ,				'''w'''   ,				encoding='''utf-8''') as outfile:
                                  json.dump(a__   ,				a__)

                           # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                           # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                           # "special_tokens_map.json" files
                           _lowerCamelCase				:  Optional[int]						=		tokenizer_class.from_pretrained(a__   ,				extra_ids=0)
                           self.assertIn(a__   ,				tokenizer_without_change_in_init.additional_special_tokens)
                           # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                           self.assertEqual(
                               [new_token_a]   ,				tokenizer_without_change_in_init.convert_ids_to_tokens(
                                   tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]))   ,				)

                           _lowerCamelCase				:  List[Any]						=		0Xe_0_0_7
                           _lowerCamelCase				:  Dict						=		chr(a__)
                           # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                           _lowerCamelCase				:  Any						=		[AddedToken(a__   ,				lstrip=a__)]
                           _lowerCamelCase				:  Optional[int]						=		tokenizer_class.from_pretrained(
                               a__   ,				additional_special_tokens=a__   ,				extra_ids=0)

                           self.assertIn(a__   ,				tokenizer.additional_special_tokens)
                           # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
                           self.assertEqual(
                               [new_token_a]   ,				tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a])))


      @require_tokenizers
      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		self.get_tokenizers(do_lower_case=a__)
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                           _lowerCamelCase				:  List[str]						=		'''hello world'''
                           if self.space_between_special_tokens:
                                  _lowerCamelCase				:  Union[str, Any]						=		'''[CLS] hello world [SEP]'''
                           else:
                                  _lowerCamelCase				:  Tuple						=		input
                           _lowerCamelCase				:  Tuple						=		tokenizer.encode(a__   ,				add_special_tokens=a__)
                           _lowerCamelCase				:  Union[str, Any]						=		tokenizer.decode(a__   ,				spaces_between_special_tokens=self.space_between_special_tokens)
                           self.assertIn(a__   ,				[output, output.lower()])


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Union[str, Any]						=		self.get_tokenizers()
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                           _lowerCamelCase				:  Optional[int]						=		[
                               '''bos_token''',
                               '''eos_token''',
                               '''unk_token''',
                               '''sep_token''',
                               '''pad_token''',
                               '''cls_token''',
                               '''mask_token''',
                           ]

             _lowerCamelCase				:  str						=		'''a'''
             _lowerCamelCase				:  Optional[Any]						=		ord(a__)

             for attr in attributes_list:
                    setattr(a__   ,				attr + '''_id'''   ,				a__)
                    self.assertEqual(getattr(a__   ,				a__)   ,				a__)
                    self.assertEqual(getattr(a__   ,				attr + '''_id''')   ,				a__)

                    setattr(a__   ,				attr + '''_id'''   ,				a__)
                    self.assertEqual(getattr(a__   ,				a__)   ,				a__)
                    self.assertEqual(getattr(a__   ,				attr + '''_id''')   ,				a__)

             setattr(a__   ,				'''additional_special_tokens_ids'''   ,				[])
             self.assertListEqual(getattr(a__   ,				'''additional_special_tokens''')   ,				[])
             self.assertListEqual(getattr(a__   ,				'''additional_special_tokens_ids''')   ,				[])

             _lowerCamelCase				:  Any						=		0Xe_0_0_6
             _lowerCamelCase				:  Tuple						=		chr(a__)
             setattr(a__   ,				'''additional_special_tokens_ids'''   ,				[additional_special_token_id])
             self.assertListEqual(getattr(a__   ,				'''additional_special_tokens''')   ,				[additional_special_token])
             self.assertListEqual(getattr(a__   ,				'''additional_special_tokens_ids''')   ,				[additional_special_token_id])


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             pass



      def 					__snake_case      (		self):



             """simple docstring"""





             pass








import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __A							(   lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   OpenAIGPTTokenizer
      UpperCAmelCase__							=   OpenAIGPTTokenizerFast
      UpperCAmelCase__							=   True
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             _lowerCamelCase				:  Tuple						=		[
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''w</w>''',
                 '''r</w>''',
                 '''t</w>''',
                 '''lo''',
                 '''low''',
                 '''er</w>''',
                 '''low</w>''',
                 '''lowest</w>''',
                 '''newer</w>''',
                 '''wider</w>''',
                 '''<unk>''',
             ]
             _lowerCamelCase				:  Any						=		dict(zip(a__   ,				range(len(a__))))
             _lowerCamelCase				:  List[str]						=		['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

             _lowerCamelCase				:  str						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file'''])
             _lowerCamelCase				:  int						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file'''])
             with open(self.vocab_file   ,				'''w''') as fp:
                    fp.write(json.dumps(a__))
             with open(self.merges_file   ,				'''w''') as fp:
                    fp.write('''\n'''.join(a__))


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return "lower newer", "lower newer"


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		OpenAIGPTTokenizer(self.vocab_file   ,				self.merges_file)

             _lowerCamelCase				:  Dict						=		'''lower'''
             _lowerCamelCase				:  Optional[Any]						=		['''low''', '''er</w>''']
             _lowerCamelCase				:  List[Any]						=		tokenizer.tokenize(a__)
             self.assertListEqual(a__   ,				a__)

             _lowerCamelCase				:  Any						=		tokens + ['''<unk>''']
             _lowerCamelCase				:  Optional[int]						=		[14, 15, 20]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)   ,				a__)


      def 					__snake_case      (		self   ,				a__=15):



             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                           _lowerCamelCase				:  Dict						=		self.rust_tokenizer_class.from_pretrained(a__   ,				**a__)

                           # Simple input
                           _lowerCamelCase				:  Optional[int]						=		'''This is a simple input'''
                           _lowerCamelCase				:  Optional[Any]						=		['''This is a simple input 1''', '''This is a simple input 2''']
                           _lowerCamelCase				:  Optional[int]						=		('''This is a simple input''', '''This is a pair''')
                           _lowerCamelCase				:  List[Any]						=		[
                               ('''This is a simple input 1''', '''This is a simple input 2'''),
                               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                           ]

                           # Simple input tests
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)



      def 					__snake_case      (		self):



             """simple docstring"""





             pass







@require_ftfy
@require_spacy
@require_tokenizers
class       __A							(   lowerCamelCase__	):




      """simple docstring"""







      pass

