


"""simple docstring"""




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
					__magic_name__						: List[str]    =				'pt'
elif is_tf_available():
					__magic_name__						: Dict    =				'tf'
else:
					__magic_name__						: Optional[int]    =				'jax'




class 			__snake_case   (lowerCamelCase			,		unittest.TestCase  ):
							__a					 =       ByTaTokenizer
							__a					 =       False




							def   __a   (     self:    int   ):
										super().setUp()
										__lowerCamelCase          =      ByTaTokenizer()
										tokenizer.save_pretrained(self.tmpdirname   )




							@cached_property
							def   __a   (     self:    Any   ):
										return ByTaTokenizer.from_pretrained("""google/byt5-small"""   )




							def   __a   (     self:    Tuple     ,							**A_:    List[Any]   ):
										return self.tokenizer_class.from_pretrained(self.tmpdirname     ,							**A_   )




							def   __a   (     self:    str     ,							A_:    Tuple     ,							A_:    int=False     ,							A_:    List[Any]=20     ,							A_:    str=5   ):
										# XXX The default common tokenizer tests assume that every ID is decodable on its own.
										# This assumption is invalid for ByT5 because single bytes might not be
										# valid utf-8 (byte 128 for instance).
										# Here we're overriding the smallest possible method to provide
										# a clean sequence without making the same assumption.

										__lowerCamelCase          =      []
										for i in range(len(A_   )   ):
													try:
																__lowerCamelCase          =      tokenizer.decode([i]     ,							clean_up_tokenization_spaces=A_   )
													except UnicodeDecodeError:
																pass
													toks.append((i, tok)   )

										__lowerCamelCase          =      list(filter(lambda A_   : re.match(r"""^[ a-zA-Z]+$"""     ,							t[1]   )     ,							A_   )   )
										__lowerCamelCase          =      list(filter(lambda A_   : [t[0]] == tokenizer.encode(t[1]     ,							add_special_tokens=A_   )     ,							A_   )   )
										if max_length is not None and len(A_   ) > max_length:
													__lowerCamelCase          =      toks[:max_length]
										if min_length is not None and len(A_   ) < min_length and len(A_   ) > 0:
													while len(A_   ) < min_length:
																__lowerCamelCase          =      toks + toks
        # toks_str = [t[1] for t in toks]
										__lowerCamelCase          =      [t[0] for t in toks]

										# Ensure consistency
										__lowerCamelCase          =      tokenizer.decode(A_     ,							clean_up_tokenization_spaces=A_   )
										if " " not in output_txt and len(A_   ) > 1:
													__lowerCamelCase          =      (
													    tokenizer.decode([toks_ids[0]]     ,							clean_up_tokenization_spaces=A_   )
													    + """ """
													    + tokenizer.decode(toks_ids[1:]     ,							clean_up_tokenization_spaces=A_   )
													)
										if with_prefix_space:
													__lowerCamelCase          =      """ """ + output_txt
										__lowerCamelCase          =      tokenizer.encode(A_     ,							add_special_tokens=A_   )
										return output_txt, output_ids




							def   __a   (     self:    int   ):
										__lowerCamelCase          =      self.ta_base_tokenizer
										__lowerCamelCase          =      tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]   )
										__lowerCamelCase          =      tokenizer(["""hi""", """I went to the gym""", """"""]   )
										self.assertListEqual(batch_with_eos_added["""input_ids"""]     ,							batch_without_eos_added["""input_ids"""]   )




							def   __a   (     self:    Any   ):
										__lowerCamelCase          =      self.ta_base_tokenizer
										__lowerCamelCase          =      """Unicode €."""
										__lowerCamelCase          =      tokenizer(A_   )
										__lowerCamelCase          =      [88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
										self.assertEqual(encoded["""input_ids"""]     ,							A_   )

										# decoding
										__lowerCamelCase          =      tokenizer.decode(A_   )
										self.assertEqual(A_     ,							"""Unicode €.</s>"""   )

										__lowerCamelCase          =      tokenizer("""e è é ê ë"""   )
										__lowerCamelCase          =      [1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
										self.assertEqual(encoded["""input_ids"""]     ,							A_   )
										# decoding
										__lowerCamelCase          =      tokenizer.decode(A_   )
										self.assertEqual(A_     ,							"""e è é ê ë</s>"""   )

										# encode/decode, but with `encode` instead of `__call__`
										self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""   )   )     ,							"""e è é ê ë</s>"""   )




							def   __a   (     self:    Optional[int]   ):
										__lowerCamelCase          =      self.ta_base_tokenizer
										__lowerCamelCase          =      ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
										# fmt: off
										__lowerCamelCase          =      [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
										# fmt: on
										__lowerCamelCase          =      tokenizer(A_     ,							padding=A_     ,							return_tensors=A_   )
										self.assertIsInstance(A_     ,							A_   )

										if FRAMEWORK != "jax":
													__lowerCamelCase          =      list(batch.input_ids.numpy()[0]   )
										else:
													__lowerCamelCase          =      list(batch.input_ids.tolist()[0]   )

										self.assertListEqual(A_     ,							A_   )

										self.assertEqual((2, 37)     ,							batch.input_ids.shape   )
										self.assertEqual((2, 37)     ,							batch.attention_mask.shape   )




							def   __a   (     self:    Tuple   ):
										__lowerCamelCase          =      self.ta_base_tokenizer
										__lowerCamelCase          =      ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
										__lowerCamelCase          =      tokenizer(A_     ,							padding=A_     ,							return_tensors=A_   )
										# check if input_ids are returned and no decoder_input_ids
										self.assertIn("""input_ids"""     ,							A_   )
										self.assertIn("""attention_mask"""     ,							A_   )
										self.assertNotIn("""decoder_input_ids"""     ,							A_   )
										self.assertNotIn("""decoder_attention_mask"""     ,							A_   )




							def   __a   (     self:    Optional[Any]   ):
										__lowerCamelCase          =      self.ta_base_tokenizer
										__lowerCamelCase          =      [
										    """Summary of the text.""",
										    """Another summary.""",
										]
										__lowerCamelCase          =      tokenizer(
										    text_target=A_     ,							max_length=32     ,							padding="""max_length"""     ,							truncation=A_     ,							return_tensors=A_   )
										self.assertEqual(32     ,							targets["""input_ids"""].shape[1]   )




							def   __a   (     self:    Union[str, Any]   ):
										__lowerCamelCase          =      self.ta_base_tokenizer
										__lowerCamelCase          =      ["""A long paragraph for summarization. </s>"""]
										__lowerCamelCase          =      ["""Summary of the text. </s>"""]
										# fmt: off
										__lowerCamelCase          =      [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
										__lowerCamelCase          =      [86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
										# fmt: on

										__lowerCamelCase          =      tokenizer(A_     ,							text_target=A_   )

										self.assertEqual(A_     ,							batch["""input_ids"""][0]   )
										self.assertEqual(A_     ,							batch["""labels"""][0]   )




							def   __a   (     self:    Dict   ):
										# safety check on max_len default value so we are sure the test works
										__lowerCamelCase          =      self.get_tokenizers()
										for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																self.assertNotEqual(tokenizer.model_max_length     ,							42   )

        # Now let's start the test
										__lowerCamelCase          =      self.get_tokenizers()
										for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																# Isolate this from the other tests because we save additional tokens/etc
																__lowerCamelCase          =      tempfile.mkdtemp()

																__lowerCamelCase          =      """ He is very happy, UNwant\u00E9d,running"""
																__lowerCamelCase          =      tokenizer.encode(A_     ,							add_special_tokens=A_   )
																tokenizer.save_pretrained(A_   )

																__lowerCamelCase          =      tokenizer.__class__.from_pretrained(A_   )
																__lowerCamelCase          =      after_tokenizer.encode(A_     ,							add_special_tokens=A_   )
																self.assertListEqual(A_     ,							A_   )

																shutil.rmtree(A_   )

										__lowerCamelCase          =      self.get_tokenizers(model_max_length=42   )
										for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																# Isolate this from the other tests because we save additional tokens/etc
																__lowerCamelCase          =      tempfile.mkdtemp()

																__lowerCamelCase          =      """ He is very happy, UNwant\u00E9d,running"""
																tokenizer.add_tokens(["""bim""", """bambam"""]   )
																__lowerCamelCase          =      tokenizer.additional_special_tokens
																additional_special_tokens.append("""new_additional_special_token"""   )
																tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}   )
																__lowerCamelCase          =      tokenizer.encode(A_     ,							add_special_tokens=A_   )
																tokenizer.save_pretrained(A_   )

																__lowerCamelCase          =      tokenizer.__class__.from_pretrained(A_   )
																__lowerCamelCase          =      after_tokenizer.encode(A_     ,							add_special_tokens=A_   )
																self.assertListEqual(A_     ,							A_   )
																self.assertIn("""new_additional_special_token"""     ,							after_tokenizer.additional_special_tokens   )
																self.assertEqual(after_tokenizer.model_max_length     ,							42   )

																__lowerCamelCase          =      tokenizer.__class__.from_pretrained(A_     ,							model_max_length=43   )
																self.assertEqual(tokenizer.model_max_length     ,							43   )

																shutil.rmtree(A_   )




							def   __a   (     self:    Union[str, Any]   ):
										__lowerCamelCase          =      []
										if self.test_slow_tokenizer:
													tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

										if self.test_rust_tokenizer:
													tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

										for tokenizer_class, tokenizer_utils in tokenizer_list:
													with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer_utils.save_pretrained(A_   )

																with open(os.path.join(A_     ,							"""special_tokens_map.json"""   )     ,							encoding="""utf-8"""   ) as json_file:
																			__lowerCamelCase          =      json.load(A_   )

																with open(os.path.join(A_     ,							"""tokenizer_config.json"""   )     ,							encoding="""utf-8"""   ) as json_file:
																			__lowerCamelCase          =      json.load(A_   )

																__lowerCamelCase          =      [f'<extra_id_{i}>' for i in range(1_25   )]

																__lowerCamelCase          =      added_tokens_extra_ids + [
																    """an_additional_special_token"""
																]
																__lowerCamelCase          =      added_tokens_extra_ids + [
																    """an_additional_special_token"""
																]

																with open(os.path.join(A_     ,							"""special_tokens_map.json"""   )     ,							"""w"""     ,							encoding="""utf-8"""   ) as outfile:
																			json.dump(A_     ,							A_   )
																with open(os.path.join(A_     ,							"""tokenizer_config.json"""   )     ,							"""w"""     ,							encoding="""utf-8"""   ) as outfile:
																			json.dump(A_     ,							A_   )

																# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																# "special_tokens_map.json" files
																__lowerCamelCase          =      tokenizer_class.from_pretrained(
																    A_     ,							)
																self.assertIn(
																    """an_additional_special_token"""     ,							tokenizer_without_change_in_init.additional_special_tokens   )
																# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																self.assertEqual(
																    ["""an_additional_special_token"""]     ,							tokenizer_without_change_in_init.convert_ids_to_tokens(
																        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]   )   )     ,							)

																# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																__lowerCamelCase          =      added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""     ,							lstrip=A_   )]
																__lowerCamelCase          =      tokenizer_class.from_pretrained(
																    A_     ,							additional_special_tokens=A_     ,							)

																self.assertIn("""a_new_additional_special_token"""     ,							tokenizer.additional_special_tokens   )
																self.assertEqual(
																    ["""a_new_additional_special_token"""]     ,							tokenizer.convert_ids_to_tokens(
																        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]   )   )     ,							)




							def   __a   (     self:    Union[str, Any]   ):
										__lowerCamelCase          =      []
										if self.test_slow_tokenizer:
													tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

										if self.test_rust_tokenizer:
													tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

										for tokenizer_class, tokenizer_utils in tokenizer_list:
													with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer_utils.save_pretrained(A_   )

																__lowerCamelCase          =      tokenizer_class.from_pretrained(A_   )

																self.assertTrue(tokenizer.decode([2_55]   ) == """"""   )




							def   __a   (     self:    Any   ):
										pass




							def   __a   (     self:    Any   ):
										pass




							def   __a   (     self:    Optional[Any]   ):
										pass




							def   __a   (     self:    Tuple   ):
										pass




							def   __a   (     self:    Dict   ):
										# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
										# and special added tokens as tokens
										__lowerCamelCase          =      self.get_tokenizers(fast=A_     ,							do_lower_case=A_   )
										for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																__lowerCamelCase          =      ["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
																__lowerCamelCase          =      tokenizer.convert_tokens_to_string(A_   )

																self.assertIsInstance(A_     ,							A_   )




							def   __a   (     self:    int   ):
										__lowerCamelCase          =      self.get_tokenizers()
										for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																__lowerCamelCase          =      [
																    """bos_token""",
																    """eos_token""",
																    """unk_token""",
																    """sep_token""",
																    """pad_token""",
																    """cls_token""",
																    """mask_token""",
																]

																__lowerCamelCase          =      0
																__lowerCamelCase          =      tokenizer.convert_ids_to_tokens(
																    A_     ,							skip_special_tokens=A_   )

																for attr in attributes_list:
																			setattr(A_     ,							attr + """_id"""     ,							A_   )
																			self.assertEqual(getattr(A_     ,							A_   )     ,							A_   )
																			self.assertEqual(getattr(A_     ,							attr + """_id"""   )     ,							A_   )

																			setattr(A_     ,							attr + """_id"""     ,							A_   )
																			self.assertEqual(getattr(A_     ,							A_   )     ,							A_   )
																			self.assertEqual(getattr(A_     ,							attr + """_id"""   )     ,							A_   )

																setattr(A_     ,							"""additional_special_tokens_ids"""     ,							[]   )
																self.assertListEqual(getattr(A_     ,							"""additional_special_tokens"""   )     ,							[]   )
																self.assertListEqual(getattr(A_     ,							"""additional_special_tokens_ids"""   )     ,							[]   )

																setattr(A_     ,							"""additional_special_tokens_ids"""     ,							[token_id_to_test_setters]   )
																self.assertListEqual(getattr(A_     ,							"""additional_special_tokens"""   )     ,							[token_to_test_setters]   )
																self.assertListEqual(getattr(A_     ,							"""additional_special_tokens_ids"""   )     ,							[token_id_to_test_setters]   )








"""simple docstring"""




import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


__magic_name__						: Tuple    =				'hf-internal-testing/tiny-random-bert'
__magic_name__						: Dict    =				os.path.join(TRANSFORMERS_CACHE, 'models--hf-internal-testing--tiny-random-bert')
__magic_name__						: Optional[Any]    =				'9b8c223d42b2188cb49d29af482996f9d0f3e5a6'




class 			__snake_case   (unittest.TestCase  ):




							def   __a   (     self:    List[Any]   ):
										__lowerCamelCase          =      cached_file(A_     ,							A_   )
										# Should have downloaded the file in here
										self.assertTrue(os.path.isdir(A_   )   )
										# Cache should contain at least those three subfolders:
										for subfolder in ["blobs", "refs", "snapshots"]:
													self.assertTrue(os.path.isdir(os.path.join(A_     ,							A_   )   )   )
										with open(os.path.join(A_     ,							"""refs"""     ,							"""main"""   )   ) as f:
													__lowerCamelCase          =      f.read()
										self.assertEqual(A_     ,							os.path.join(A_     ,							"""snapshots"""     ,							A_     ,							A_   )   )
										self.assertTrue(os.path.isfile(A_   )   )

										# File is cached at the same place the second time.
										__lowerCamelCase          =      cached_file(A_     ,							A_   )
										self.assertEqual(A_     ,							A_   )

										# Using a specific revision to test the full commit hash.
										__lowerCamelCase          =      cached_file(A_     ,							A_     ,							revision="""9b8c223"""   )
										self.assertEqual(A_     ,							os.path.join(A_     ,							"""snapshots"""     ,							A_     ,							A_   )   )




							def   __a   (     self:    int   ):
										with self.assertRaisesRegex(A_     ,							"""is not a valid model identifier"""   ):
													__lowerCamelCase          =      cached_file("""tiny-random-bert"""     ,							A_   )

										with self.assertRaisesRegex(A_     ,							"""is not a valid git identifier"""   ):
													__lowerCamelCase          =      cached_file(A_     ,							A_     ,							revision="""aaaa"""   )

										with self.assertRaisesRegex(A_     ,							"""does not appear to have a file named"""   ):
													__lowerCamelCase          =      cached_file(A_     ,							"""conf"""   )




							def   __a   (     self:    Optional[int]   ):
										with self.assertRaisesRegex(A_     ,							"""does not appear to have a file named"""   ):
													__lowerCamelCase          =      cached_file(A_     ,							"""conf"""   )

										with open(os.path.join(A_     ,							"""refs"""     ,							"""main"""   )   ) as f:
													__lowerCamelCase          =      f.read()
										self.assertTrue(os.path.isfile(os.path.join(A_     ,							""".no_exist"""     ,							A_     ,							"""conf"""   )   )   )

										__lowerCamelCase          =      cached_file(A_     ,							"""conf"""     ,							_raise_exceptions_for_missing_entries=A_   )
										self.assertIsNone(A_   )

										__lowerCamelCase          =      cached_file(A_     ,							"""conf"""     ,							local_files_only=A_     ,							_raise_exceptions_for_missing_entries=A_   )
										self.assertIsNone(A_   )

										__lowerCamelCase          =      mock.Mock()
										__lowerCamelCase          =      5_00
										__lowerCamelCase          =      {}
										__lowerCamelCase          =      HTTPError
										__lowerCamelCase          =      {}

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("""requests.Session.request"""     ,							return_value=A_   ) as mock_head:
													__lowerCamelCase          =      cached_file(A_     ,							"""conf"""     ,							_raise_exceptions_for_connection_errors=A_   )
													self.assertIsNone(A_   )
													# This check we did call the fake head request
													mock_head.assert_called()




							def   __a   (     self:    str   ):
										self.assertTrue(has_file("""hf-internal-testing/tiny-bert-pt-only"""     ,							A_   )   )
										self.assertFalse(has_file("""hf-internal-testing/tiny-bert-pt-only"""     ,							A_   )   )
										self.assertFalse(has_file("""hf-internal-testing/tiny-bert-pt-only"""     ,							A_   )   )




							def   __a   (     self:    str   ):
										# `get_file_from_repo` returns None if the file does not exist
										self.assertIsNone(get_file_from_repo("""bert-base-cased"""     ,							"""ahah.txt"""   )   )

										# The function raises if the repository does not exist.
										with self.assertRaisesRegex(A_     ,							"""is not a valid model identifier"""   ):
													get_file_from_repo("""bert-base-case"""     ,							A_   )

										# The function raises if the revision does not exist.
										with self.assertRaisesRegex(A_     ,							"""is not a valid git identifier"""   ):
													get_file_from_repo("""bert-base-cased"""     ,							A_     ,							revision="""ahaha"""   )

										__lowerCamelCase          =      get_file_from_repo("""bert-base-cased"""     ,							A_   )
										# The name is the cached name which is not very easy to test, so instead we load the content.
										__lowerCamelCase          =      json.loads(open(A_     ,							"""r"""   ).read()   )
										self.assertEqual(config["""hidden_size"""]     ,							7_68   )




							def   __a   (     self:    Union[str, Any]   ):
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowerCamelCase          =      Path(A_   ) / """a.txt"""
													filename.touch()
													self.assertEqual(get_file_from_repo(A_     ,							"""a.txt"""   )     ,							str(A_   )   )

													self.assertIsNone(get_file_from_repo(A_     ,							"""b.txt"""   )   )




