


'''simple docstring'''




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
       _a   :   Dict												=   "pt"
elif is_tf_available():
       _a   :   Any												=   "tf"
else:
       _a   :   Any												=   "jax"




class 			_lowercase (	__lowercase						,					unittest.TestCase							):
   _SCREAMING_SNAKE_CASE  :  str          =	ByTaTokenizer
   _SCREAMING_SNAKE_CASE  :  List[Any]          =	False
   def a  (						self       :      int      ) ->							int:
        super().setUp()
        __snake_case           =						ByTaTokenizer()
        tokenizer.save_pretrained(self.tmpdirname      )
   @cached_property
   def a  (						self       :      Any      ) ->							Optional[int]:
        return ByTaTokenizer.from_pretrained('google/byt5-small'      )
   def a  (						self       :      Union[str, Any]       ,   **SCREAMING_SNAKE_CASE_       :      Tuple      ) ->							ByTaTokenizer:
        return self.tokenizer_class.from_pretrained(self.tmpdirname       ,   **SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      Dict       ,   SCREAMING_SNAKE_CASE_       :      Optional[Any]       ,   SCREAMING_SNAKE_CASE_       :      str=False       ,   SCREAMING_SNAKE_CASE_       :      Dict=20       ,   SCREAMING_SNAKE_CASE_       :      List[str]=5      ) ->							Tuple[str, list]:
        # XXX The default common tokenizer tests assume that every ID is decodable on its own.
        # This assumption is invalid for ByT5 because single bytes might not be
        # valid utf-8 (byte 128 for instance).
        # Here we're overriding the smallest possible method to provide
        # a clean sequence without making the same assumption.

        __snake_case           =						[]
        for i in range(len(SCREAMING_SNAKE_CASE_      )      ):
             try:
                  __snake_case           =						tokenizer.decode([i]       ,   clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_      )
             except UnicodeDecodeError:
                  pass
             toks.append((i, tok)      )

        __snake_case           =						list(filter(lambda SCREAMING_SNAKE_CASE_      : re.match(r'^[ a-zA-Z]+$'       ,   t[1]      )       ,   SCREAMING_SNAKE_CASE_      )      )
        __snake_case           =						list(filter(lambda SCREAMING_SNAKE_CASE_      : [t[0]] == tokenizer.encode(t[1]       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )       ,   SCREAMING_SNAKE_CASE_      )      )
        if max_length is not None and len(SCREAMING_SNAKE_CASE_      ) > max_length:
             __snake_case           =						toks[:max_length]
        if min_length is not None and len(SCREAMING_SNAKE_CASE_      ) < min_length and len(SCREAMING_SNAKE_CASE_      ) > 0:
             while len(SCREAMING_SNAKE_CASE_      ) < min_length:
                  __snake_case           =						toks + toks
        # toks_str = [t[1] for t in toks]
        __snake_case           =						[t[0] for t in toks]

        # Ensure consistency
        __snake_case           =						tokenizer.decode(SCREAMING_SNAKE_CASE_       ,   clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_      )
        if " " not in output_txt and len(SCREAMING_SNAKE_CASE_      ) > 1:
             __snake_case           =						(
                 tokenizer.decode([toks_ids[0]]       ,   clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_      )
                 + ' '
                 + tokenizer.decode(toks_ids[1:]       ,   clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_      )
             )
        if with_prefix_space:
             __snake_case           =						' ' + output_txt
        __snake_case           =						tokenizer.encode(SCREAMING_SNAKE_CASE_       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )
        return output_txt, output_ids
   def a  (						self       :      Optional[int]      ) ->							Optional[int]:
        __snake_case           =						self.ta_base_tokenizer
        __snake_case           =						tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']      )
        __snake_case           =						tokenizer(['hi', 'I went to the gym', '']      )
        self.assertListEqual(batch_with_eos_added['input_ids']       ,   batch_without_eos_added['input_ids']      )
   def a  (						self       :      Any      ) ->							List[Any]:
        __snake_case           =						self.ta_base_tokenizer
        __snake_case           =						'Unicode €.'
        __snake_case           =						tokenizer(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
        self.assertEqual(encoded['input_ids']       ,   SCREAMING_SNAKE_CASE_      )

        # decoding
        __snake_case           =						tokenizer.decode(SCREAMING_SNAKE_CASE_      )
        self.assertEqual(SCREAMING_SNAKE_CASE_       ,   'Unicode €.</s>'      )

        __snake_case           =						tokenizer('e è é ê ë'      )
        __snake_case           =						[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
        self.assertEqual(encoded['input_ids']       ,   SCREAMING_SNAKE_CASE_      )
        # decoding
        __snake_case           =						tokenizer.decode(SCREAMING_SNAKE_CASE_      )
        self.assertEqual(SCREAMING_SNAKE_CASE_       ,   'e è é ê ë</s>'      )

        # encode/decode, but with `encode` instead of `__call__`
        self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'      )      )       ,   'e è é ê ë</s>'      )
   def a  (						self       :      int      ) ->							Union[str, Any]:
        __snake_case           =						self.ta_base_tokenizer
        __snake_case           =						['A long paragraph for summarization.', 'Another paragraph for summarization.']
        # fmt: off
        __snake_case           =						[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
        # fmt: on
        __snake_case           =						tokenizer(SCREAMING_SNAKE_CASE_       ,   padding=SCREAMING_SNAKE_CASE_       ,   return_tensors=SCREAMING_SNAKE_CASE_      )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

        if FRAMEWORK != "jax":
             __snake_case           =						list(batch.input_ids.numpy()[0]      )
        else:
             __snake_case           =						list(batch.input_ids.tolist()[0]      )

        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

        self.assertEqual((2, 37)       ,   batch.input_ids.shape      )
        self.assertEqual((2, 37)       ,   batch.attention_mask.shape      )
   def a  (						self       :      Optional[Any]      ) ->							Union[str, Any]:
        __snake_case           =						self.ta_base_tokenizer
        __snake_case           =						['A long paragraph for summarization.', 'Another paragraph for summarization.']
        __snake_case           =						tokenizer(SCREAMING_SNAKE_CASE_       ,   padding=SCREAMING_SNAKE_CASE_       ,   return_tensors=SCREAMING_SNAKE_CASE_      )
        # check if input_ids are returned and no decoder_input_ids
        self.assertIn('input_ids'       ,   SCREAMING_SNAKE_CASE_      )
        self.assertIn('attention_mask'       ,   SCREAMING_SNAKE_CASE_      )
        self.assertNotIn('decoder_input_ids'       ,   SCREAMING_SNAKE_CASE_      )
        self.assertNotIn('decoder_attention_mask'       ,   SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      int      ) ->							Optional[Any]:
        __snake_case           =						self.ta_base_tokenizer
        __snake_case           =						[
            'Summary of the text.',
            'Another summary.',
        ]
        __snake_case           =						tokenizer(
            text_target=SCREAMING_SNAKE_CASE_       ,   max_length=32       ,   padding='max_length'       ,   truncation=SCREAMING_SNAKE_CASE_       ,   return_tensors=SCREAMING_SNAKE_CASE_      )
        self.assertEqual(32       ,   targets['input_ids'].shape[1]      )
   def a  (						self       :      Dict      ) ->							Union[str, Any]:
        __snake_case           =						self.ta_base_tokenizer
        __snake_case           =						['A long paragraph for summarization. </s>']
        __snake_case           =						['Summary of the text. </s>']
        # fmt: off
        __snake_case           =						[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
        __snake_case           =						[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
        # fmt: on

        __snake_case           =						tokenizer(SCREAMING_SNAKE_CASE_       ,   text_target=SCREAMING_SNAKE_CASE_      )

        self.assertEqual(SCREAMING_SNAKE_CASE_       ,   batch['input_ids'][0]      )
        self.assertEqual(SCREAMING_SNAKE_CASE_       ,   batch['labels'][0]      )
   def a  (						self       :      Tuple      ) ->							Optional[int]:
        # safety check on max_len default value so we are sure the test works
        __snake_case           =						self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'      ):
                  self.assertNotEqual(tokenizer.model_max_length       ,   42      )

        # Now let's start the test
        __snake_case           =						self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'      ):
                  # Isolate this from the other tests because we save additional tokens/etc
                  __snake_case           =						tempfile.mkdtemp()

                  __snake_case           =						' He is very happy, UNwant\u00E9d,running'
                  __snake_case           =						tokenizer.encode(SCREAMING_SNAKE_CASE_       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )
                  tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_      )

                  __snake_case           =						tokenizer.__class__.from_pretrained(SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						after_tokenizer.encode(SCREAMING_SNAKE_CASE_       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )
                  self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

                  shutil.rmtree(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						self.get_tokenizers(model_max_length=42      )
        for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'      ):
                  # Isolate this from the other tests because we save additional tokens/etc
                  __snake_case           =						tempfile.mkdtemp()

                  __snake_case           =						' He is very happy, UNwant\u00E9d,running'
                  tokenizer.add_tokens(['bim', 'bambam']      )
                  __snake_case           =						tokenizer.additional_special_tokens
                  additional_special_tokens.append('new_additional_special_token'      )
                  tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}      )
                  __snake_case           =						tokenizer.encode(SCREAMING_SNAKE_CASE_       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )
                  tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_      )

                  __snake_case           =						tokenizer.__class__.from_pretrained(SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						after_tokenizer.encode(SCREAMING_SNAKE_CASE_       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )
                  self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
                  self.assertIn('new_additional_special_token'       ,   after_tokenizer.additional_special_tokens      )
                  self.assertEqual(after_tokenizer.model_max_length       ,   42      )

                  __snake_case           =						tokenizer.__class__.from_pretrained(SCREAMING_SNAKE_CASE_       ,   model_max_length=43      )
                  self.assertEqual(tokenizer.model_max_length       ,   43      )

                  shutil.rmtree(SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      int      ) ->							Dict:
        __snake_case           =						[]
        if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

        if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

        for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                  tokenizer_utils.save_pretrained(SCREAMING_SNAKE_CASE_      )

                  with open(os.path.join(SCREAMING_SNAKE_CASE_       ,   'special_tokens_map.json'      )       ,   encoding='utf-8'      ) as json_file:
                       __snake_case           =						json.load(SCREAMING_SNAKE_CASE_      )

                  with open(os.path.join(SCREAMING_SNAKE_CASE_       ,   'tokenizer_config.json'      )       ,   encoding='utf-8'      ) as json_file:
                       __snake_case           =						json.load(SCREAMING_SNAKE_CASE_      )

                  __snake_case           =						[f'<extra_id_{i}>' for i in range(125      )]

                  __snake_case           =						added_tokens_extra_ids + [
                      'an_additional_special_token'
                  ]
                  __snake_case           =						added_tokens_extra_ids + [
                      'an_additional_special_token'
                  ]

                  with open(os.path.join(SCREAMING_SNAKE_CASE_       ,   'special_tokens_map.json'      )       ,   'w'       ,   encoding='utf-8'      ) as outfile:
                       json.dump(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
                  with open(os.path.join(SCREAMING_SNAKE_CASE_       ,   'tokenizer_config.json'      )       ,   'w'       ,   encoding='utf-8'      ) as outfile:
                       json.dump(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

                  # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                  # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                  # "special_tokens_map.json" files
                  __snake_case           =						tokenizer_class.from_pretrained(
                      SCREAMING_SNAKE_CASE_       ,   )
                  self.assertIn(
                      'an_additional_special_token'       ,   tokenizer_without_change_in_init.additional_special_tokens      )
                  # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                  self.assertEqual(
                      ['an_additional_special_token']       ,   tokenizer_without_change_in_init.convert_ids_to_tokens(
                          tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']      )      )       ,   )

                  # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                  __snake_case           =						added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'       ,   lstrip=SCREAMING_SNAKE_CASE_      )]
                  __snake_case           =						tokenizer_class.from_pretrained(
                      SCREAMING_SNAKE_CASE_       ,   additional_special_tokens=SCREAMING_SNAKE_CASE_       ,   )

                  self.assertIn('a_new_additional_special_token'       ,   tokenizer.additional_special_tokens      )
                  self.assertEqual(
                      ['a_new_additional_special_token']       ,   tokenizer.convert_ids_to_tokens(
                          tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']      )      )       ,   )
   def a  (						self       :      Tuple      ) ->							List[Any]:
        __snake_case           =						[]
        if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

        if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

        for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                  tokenizer_utils.save_pretrained(SCREAMING_SNAKE_CASE_      )

                  __snake_case           =						tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_      )

                  self.assertTrue(tokenizer.decode([255]      ) == ''      )
   def a  (						self       :      Optional[int]      ) ->							Optional[int]:
        pass
   def a  (						self       :      Optional[Any]      ) ->							Any:
        pass
   def a  (						self       :      Any      ) ->							Union[str, Any]:
        pass
   def a  (						self       :      Dict      ) ->							Union[str, Any]:
        pass
   def a  (						self       :      List[str]      ) ->							Any:
        # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
        # and special added tokens as tokens
        __snake_case           =						self.get_tokenizers(fast=SCREAMING_SNAKE_CASE_       ,   do_lower_case=SCREAMING_SNAKE_CASE_      )
        for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'      ):
                  __snake_case           =						['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                  __snake_case           =						tokenizer.convert_tokens_to_string(SCREAMING_SNAKE_CASE_      )

                  self.assertIsInstance(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )





   def a  (						self       :      List[str]      ) ->							Union[str, Any]:
        __snake_case           =						self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'      ):
                  __snake_case           =						[
                      'bos_token',
                      'eos_token',
                      'unk_token',
                      'sep_token',
                      'pad_token',
                      'cls_token',
                      'mask_token',
                  ]

                  __snake_case           =						0
                  __snake_case           =						tokenizer.convert_ids_to_tokens(
                      SCREAMING_SNAKE_CASE_       ,   skip_special_tokens=SCREAMING_SNAKE_CASE_      )

                  for attr in attributes_list:
                       setattr(SCREAMING_SNAKE_CASE_       ,   attr + '_id'       ,   SCREAMING_SNAKE_CASE_      )
                       self.assertEqual(getattr(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )       ,   SCREAMING_SNAKE_CASE_      )
                       self.assertEqual(getattr(SCREAMING_SNAKE_CASE_       ,   attr + '_id'      )       ,   SCREAMING_SNAKE_CASE_      )

                       setattr(SCREAMING_SNAKE_CASE_       ,   attr + '_id'       ,   SCREAMING_SNAKE_CASE_      )
                       self.assertEqual(getattr(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )       ,   SCREAMING_SNAKE_CASE_      )
                       self.assertEqual(getattr(SCREAMING_SNAKE_CASE_       ,   attr + '_id'      )       ,   SCREAMING_SNAKE_CASE_      )

                  setattr(SCREAMING_SNAKE_CASE_       ,   'additional_special_tokens_ids'       ,   []      )
                  self.assertListEqual(getattr(SCREAMING_SNAKE_CASE_       ,   'additional_special_tokens'      )       ,   []      )
                  self.assertListEqual(getattr(SCREAMING_SNAKE_CASE_       ,   'additional_special_tokens_ids'      )       ,   []      )

                  setattr(SCREAMING_SNAKE_CASE_       ,   'additional_special_tokens_ids'       ,   [token_id_to_test_setters]      )
                  self.assertListEqual(getattr(SCREAMING_SNAKE_CASE_       ,   'additional_special_tokens'      )       ,   [token_to_test_setters]      )
                  self.assertListEqual(getattr(SCREAMING_SNAKE_CASE_       ,   'additional_special_tokens_ids'      )       ,   [token_id_to_test_setters]      )








'''simple docstring'''






def 							UpperCamelCase_     (       A__   :  bytes   ):
  '''simple docstring'''





  return "".join([hex(A__   )[2:].zfill(2   ).upper() for byte in list(A__   )]   )







def 							UpperCamelCase_     (       A__   :  str   ):
  '''simple docstring'''





  if (len(A__   ) % 2) != 0:
    raise ValueError(
        """Base16 encoded data is invalid:
Data does not have an even number of hex digits."""   )
  # Check the character set - the standard base16 alphabet
  # is uppercase according to RFC3548 section 6
  if not set(A__   ) <= set("""0123456789ABCDEF"""   ):
    raise ValueError(
        """Base16 encoded data is invalid:
Data is not uppercase hex or it contains invalid characters."""   )
  # For every two hexadecimal digits (= a byte), turn it into an integer.
  # Then, string the result together into bytes, and return it.
  return bytes(int(data[i] + data[i + 1]				,       16   ) for i in range(0				,       len(A__   )				,       2   )   )


if __name__ == "__main__":
  import doctest

  doctest.testmod()



