



import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin



class 			__lowerCAmelCase  (						lowerCAmelCase__	,						unittest.TestCase ):
 lowerCAmelCase__							       =	CanineTokenizer
 lowerCAmelCase__							       =	False




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    super().setUp()
    __lowerCamelCase    				=     CanineTokenizer()
    tokenizer.save_pretrained(self.tmpdirname					)




 @cached_property
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return CanineTokenizer.from_pretrained('''google/canine-s'''					)




 def 							lowerCamelCase   (    self						,					**__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.tokenizer_class.from_pretrained(self.tmpdirname						,					**UpperCAmelCase_					)
    __lowerCamelCase    				=     1024
    return tokenizer




 @require_torch
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.canine_tokenizer
    __lowerCamelCase    				=     ['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
    # fmt: off
    __lowerCamelCase    				=     [57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
    # fmt: on
    __lowerCamelCase    				=     tokenizer(UpperCAmelCase_						,					padding=UpperCAmelCase_						,					return_tensors='''pt'''					)
    self.assertIsInstance(UpperCAmelCase_						,					UpperCAmelCase_					)

    __lowerCamelCase    				=     list(batch.input_ids.numpy()[0]					)

    self.assertListEqual(UpperCAmelCase_						,					UpperCAmelCase_					)

    self.assertEqual((2, 39)						,					batch.input_ids.shape					)
    self.assertEqual((2, 39)						,					batch.attention_mask.shape					)




 @require_torch
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.canine_tokenizer
    __lowerCamelCase    				=     ['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
    __lowerCamelCase    				=     tokenizer(UpperCAmelCase_						,					padding=UpperCAmelCase_						,					return_tensors='''pt'''					)
    # check if input_ids, attention_mask and token_type_ids are returned
    self.assertIn('''input_ids'''						,					UpperCAmelCase_					)
    self.assertIn('''attention_mask'''						,					UpperCAmelCase_					)
    self.assertIn('''token_type_ids'''						,					UpperCAmelCase_					)




 @require_torch
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.canine_tokenizer
    __lowerCamelCase    				=     [
        '''What\'s the weater?''',
        '''It\'s about 25 degrees.''',
    ]
    __lowerCamelCase    				=     tokenizer(
        text_target=UpperCAmelCase_						,					max_length=32						,					padding='''max_length'''						,					truncation=UpperCAmelCase_						,					return_tensors='''pt'''					)
    self.assertEqual(32						,					targets['''input_ids'''].shape[1]					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    # safety check on max_len default value so we are sure the test works
    __lowerCamelCase    				=     self.get_tokenizers()
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          self.assertNotEqual(tokenizer.model_max_length						,					42					)

        # Now let's start the test
    __lowerCamelCase    				=     self.get_tokenizers()
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          # Isolate this from the other tests because we save additional tokens/etc
          __lowerCamelCase    				=     tempfile.mkdtemp()

          __lowerCamelCase    				=     ''' He is very happy, UNwant\u00E9d,running'''
          __lowerCamelCase    				=     tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)
          tokenizer.save_pretrained(UpperCAmelCase_					)

          __lowerCamelCase    				=     tokenizer.__class__.from_pretrained(UpperCAmelCase_					)
          __lowerCamelCase    				=     after_tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)
          self.assertListEqual(UpperCAmelCase_						,					UpperCAmelCase_					)

          shutil.rmtree(UpperCAmelCase_					)

    __lowerCamelCase    				=     self.get_tokenizers(model_max_length=42					)
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          # Isolate this from the other tests because we save additional tokens/etc
          __lowerCamelCase    				=     tempfile.mkdtemp()

          __lowerCamelCase    				=     ''' He is very happy, UNwant\u00E9d,running'''

          __lowerCamelCase    				=     tokenizer.additional_special_tokens

          # We can add a new special token for Canine as follows:
          __lowerCamelCase    				=     chr(0Xe_007					)
          additional_special_tokens.append(UpperCAmelCase_					)
          tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}					)
          __lowerCamelCase    				=     tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)
          tokenizer.save_pretrained(UpperCAmelCase_					)

          __lowerCamelCase    				=     tokenizer.__class__.from_pretrained(UpperCAmelCase_					)
          __lowerCamelCase    				=     after_tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)
          self.assertListEqual(UpperCAmelCase_						,					UpperCAmelCase_					)
          self.assertIn(UpperCAmelCase_						,					after_tokenizer.additional_special_tokens					)
          self.assertEqual(after_tokenizer.model_max_length						,					42					)

          __lowerCamelCase    				=     tokenizer.__class__.from_pretrained(UpperCAmelCase_						,					model_max_length=43					)
          self.assertEqual(tokenizer.model_max_length						,					43					)

          shutil.rmtree(UpperCAmelCase_					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.get_tokenizers(do_lower_case=UpperCAmelCase_					)
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          __lowerCamelCase				,__lowerCamelCase    				=     self.get_clean_sequence(UpperCAmelCase_					)

          # a special token for Canine can be defined as follows:
          __lowerCamelCase    				=     0Xe_005
          __lowerCamelCase    				=     chr(UpperCAmelCase_					)

          tokenizer.add_special_tokens({'''cls_token''': special_token}					)
          __lowerCamelCase    				=     tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)
          self.assertEqual(len(UpperCAmelCase_					)						,					1					)

          __lowerCamelCase    				=     tokenizer.decode(ids + encoded_special_token						,					clean_up_tokenization_spaces=UpperCAmelCase_					)
          __lowerCamelCase    				=     tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)

          __lowerCamelCase    				=     tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)
          __lowerCamelCase    				=     tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)
          self.assertEqual(UpperCAmelCase_						,					input_encoded + special_token_id					)

          __lowerCamelCase    				=     tokenizer.decode(UpperCAmelCase_						,					skip_special_tokens=UpperCAmelCase_					)
          self.assertTrue(special_token not in decoded					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.get_tokenizers(do_lower_case=UpperCAmelCase_					)
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          __lowerCamelCase    				=     chr(0Xe_005					)
          __lowerCamelCase    				=     chr(0Xe_006					)

          # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
          tokenizer.add_tokens([SPECIAL_TOKEN_1]						,					special_tokens=UpperCAmelCase_					)
          # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
          # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
          tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]}					)

          __lowerCamelCase    				=     tokenizer.tokenize(UpperCAmelCase_					)
          __lowerCamelCase    				=     tokenizer.tokenize(UpperCAmelCase_					)

          self.assertEqual(len(UpperCAmelCase_					)						,					1					)
          self.assertEqual(len(UpperCAmelCase_					)						,					1					)
          self.assertEqual(token_a[0]						,					UpperCAmelCase_					)
          self.assertEqual(token_a[0]						,					UpperCAmelCase_					)




 @require_tokenizers
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.get_tokenizers(do_lower_case=UpperCAmelCase_					)
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          # a special token for Canine can be defined as follows:
          __lowerCamelCase    				=     0Xe_006
          __lowerCamelCase    				=     chr(UpperCAmelCase_					)

          __lowerCamelCase    				=     AddedToken(UpperCAmelCase_						,					lstrip=UpperCAmelCase_					)
          tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]}					)

          with tempfile.TemporaryDirectory() as tmp_dir_name:
             tokenizer.save_pretrained(UpperCAmelCase_					)
             tokenizer.from_pretrained(UpperCAmelCase_					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     []
    if self.test_slow_tokenizer:
       tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

    if self.test_rust_tokenizer:
       tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

    for tokenizer_class, tokenizer_utils in tokenizer_list:
       with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer_utils.save_pretrained(UpperCAmelCase_					)

          with open(os.path.join(UpperCAmelCase_						,					'''special_tokens_map.json'''					)						,					encoding='''utf-8'''					) as json_file:
             __lowerCamelCase    				=     json.load(UpperCAmelCase_					)

          with open(os.path.join(UpperCAmelCase_						,					'''tokenizer_config.json'''					)						,					encoding='''utf-8'''					) as json_file:
             __lowerCamelCase    				=     json.load(UpperCAmelCase_					)

          # a special token for Canine can be defined as follows:
          __lowerCamelCase    				=     0Xe_006
          __lowerCamelCase    				=     chr(UpperCAmelCase_					)

          __lowerCamelCase    				=     [new_token_a]
          __lowerCamelCase    				=     [new_token_a]

          with open(os.path.join(UpperCAmelCase_						,					'''special_tokens_map.json'''					)						,					'''w'''						,					encoding='''utf-8'''					) as outfile:
             json.dump(UpperCAmelCase_						,					UpperCAmelCase_					)
          with open(os.path.join(UpperCAmelCase_						,					'''tokenizer_config.json'''					)						,					'''w'''						,					encoding='''utf-8'''					) as outfile:
             json.dump(UpperCAmelCase_						,					UpperCAmelCase_					)

          # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
          # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
          # "special_tokens_map.json" files
          __lowerCamelCase    				=     tokenizer_class.from_pretrained(UpperCAmelCase_						,					extra_ids=0					)
          self.assertIn(UpperCAmelCase_						,					tokenizer_without_change_in_init.additional_special_tokens					)
          # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
          self.assertEqual(
              [new_token_a]						,					tokenizer_without_change_in_init.convert_ids_to_tokens(
                  tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]					)					)						,					)

          __lowerCamelCase    				=     0Xe_007
          __lowerCamelCase    				=     chr(UpperCAmelCase_					)
          # Now we test that we can change the value of additional_special_tokens in the from_pretrained
          __lowerCamelCase    				=     [AddedToken(UpperCAmelCase_						,					lstrip=UpperCAmelCase_					)]
          __lowerCamelCase    				=     tokenizer_class.from_pretrained(
              UpperCAmelCase_						,					additional_special_tokens=UpperCAmelCase_						,					extra_ids=0					)

          self.assertIn(UpperCAmelCase_						,					tokenizer.additional_special_tokens					)
          # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
          self.assertEqual(
              [new_token_a]						,					tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]					)					)					)




 @require_tokenizers
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.get_tokenizers(do_lower_case=UpperCAmelCase_					)
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          __lowerCamelCase    				=     '''hello world'''
          if self.space_between_special_tokens:
             __lowerCamelCase    				=     '''[CLS] hello world [SEP]'''
          else:
             __lowerCamelCase    				=     input
          __lowerCamelCase    				=     tokenizer.encode(UpperCAmelCase_						,					add_special_tokens=UpperCAmelCase_					)
          __lowerCamelCase    				=     tokenizer.decode(UpperCAmelCase_						,					spaces_between_special_tokens=self.space_between_special_tokens					)
          self.assertIn(UpperCAmelCase_						,					[output, output.lower()]					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.get_tokenizers()
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          __lowerCamelCase    				=     [
              '''bos_token''',
              '''eos_token''',
              '''unk_token''',
              '''sep_token''',
              '''pad_token''',
              '''cls_token''',
              '''mask_token''',
          ]

    __lowerCamelCase    				=     '''a'''
    __lowerCamelCase    				=     ord(UpperCAmelCase_					)

    for attr in attributes_list:
       setattr(UpperCAmelCase_						,					attr + '''_id'''						,					UpperCAmelCase_					)
       self.assertEqual(getattr(UpperCAmelCase_						,					UpperCAmelCase_					)						,					UpperCAmelCase_					)
       self.assertEqual(getattr(UpperCAmelCase_						,					attr + '''_id'''					)						,					UpperCAmelCase_					)

       setattr(UpperCAmelCase_						,					attr + '''_id'''						,					UpperCAmelCase_					)
       self.assertEqual(getattr(UpperCAmelCase_						,					UpperCAmelCase_					)						,					UpperCAmelCase_					)
       self.assertEqual(getattr(UpperCAmelCase_						,					attr + '''_id'''					)						,					UpperCAmelCase_					)

    setattr(UpperCAmelCase_						,					'''additional_special_tokens_ids'''						,					[]					)
    self.assertListEqual(getattr(UpperCAmelCase_						,					'''additional_special_tokens'''					)						,					[]					)
    self.assertListEqual(getattr(UpperCAmelCase_						,					'''additional_special_tokens_ids'''					)						,					[]					)

    __lowerCamelCase    				=     0Xe_006
    __lowerCamelCase    				=     chr(UpperCAmelCase_					)
    setattr(UpperCAmelCase_						,					'''additional_special_tokens_ids'''						,					[additional_special_token_id]					)
    self.assertListEqual(getattr(UpperCAmelCase_						,					'''additional_special_tokens'''					)						,					[additional_special_token]					)
    self.assertListEqual(getattr(UpperCAmelCase_						,					'''additional_special_tokens_ids'''					)						,					[additional_special_token_id]					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass











'''simple docstring'''





from __future__ import annotations




def 							lowerCamelCase	(					_snake_case	:    int  ,_snake_case	:    int							):





    '''simple docstring'''


    if b == 0:
        return (1, 0)
    ((lowercase__)							,							(lowercase__))									=    extended_euclid(_snake_case  ,a % b							)
    lowercase__									=    a // b
    return (y, x - k * y)




def 							lowerCamelCase	(					_snake_case	:    int  ,_snake_case	:    int  ,_snake_case	:    int  ,_snake_case	:    int							):





    '''simple docstring'''


    ((lowercase__)							,							(lowercase__))									=    extended_euclid(_snake_case  ,_snake_case							)
    lowercase__									=    na * na
    lowercase__									=    ra * x * na + ra * y * na
    return (n % m + m) % m




def 							lowerCamelCase	(					_snake_case	:    int  ,_snake_case	:    int							):





    '''simple docstring'''


    ((lowercase__)							,							(lowercase__))									=    extended_euclid(_snake_case  ,_snake_case							)
    if b < 0:
        lowercase__									=    (b % n + n) % n
    return b




def 							lowerCamelCase	(					_snake_case	:    int  ,_snake_case	:    int  ,_snake_case	:    int  ,_snake_case	:    int							):





    '''simple docstring'''


    lowercase__							,							lowercase__									=    invert_modulo(_snake_case  ,_snake_case							), invert_modulo(_snake_case  ,_snake_case							)
    lowercase__									=    na * na
    lowercase__									=    ra * x * na + ra * y * na
    return (n % m + m) % m


if __name__ == "__main__":
 from doctest import testmod

 testmod(name="chinese_remainder_theorem", verbose=True)
 testmod(name="chinese_remainder_theorem2", verbose=True)
 testmod(name="invert_modulo", verbose=True)
 testmod(name="extended_euclid", verbose=True)




