





'''simple docstring'''


import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
		lowerCAmelCase_   :   List[str]       =					'''pt'''
elif is_tf_available():
		lowerCAmelCase_   :   Tuple       =					'''tf'''
else:
		lowerCAmelCase_   :   Optional[int]       =					'''jax'''






class       UpperCamelCase__  (		__lowerCAmelCase	,unittest.TestCase     ):
	lowerCAmelCase__					:			Optional[int]								=						ByTaTokenizer
	lowerCAmelCase__					:			Any								=						False







	def 						__a   (				self			:   Tuple		):




				'''simple docstring'''
				super().setUp()
				a__     = ByTaTokenizer()
				tokenizer.save_pretrained(self.tmpdirname		)







	@cached_property
	def 						__a   (				self			:   int		):




				'''simple docstring'''
				return ByTaTokenizer.from_pretrained("google/byt5-small"		)







	def 						__a   (				self			:   int , **lowerCamelCase			:   List[Any]		):




				'''simple docstring'''
				return self.tokenizer_class.from_pretrained(self.tmpdirname , **UpperCamelCase_		)







	def 						__a   (				self			:   Optional[int] , lowerCamelCase			:   str , lowerCamelCase			:   Union[str, Any]=False , lowerCamelCase			:   Tuple=2_0 , lowerCamelCase			:   Optional[int]=5		):




				'''simple docstring'''
				a__     = []
				for i in range(len(UpperCamelCase_		)		):
							try:
										a__     = tokenizer.decode([i] , clean_up_tokenization_spaces=UpperCamelCase_		)
							except UnicodeDecodeError:
										pass
							toks.append((i, tok)		)

				a__     = list(filter(lambda lowerCamelCase		: re.match(r"^[ a-zA-Z]+$" , t[1]		) , UpperCamelCase_		)		)
				a__     = list(filter(lambda lowerCamelCase		: [t[0]] == tokenizer.encode(t[1] , add_special_tokens=UpperCamelCase_		) , UpperCamelCase_		)		)
				if max_length is not None and len(UpperCamelCase_		) > max_length:
							a__     = toks[:max_length]
				if min_length is not None and len(UpperCamelCase_		) < min_length and len(UpperCamelCase_		) > 0:
							while len(UpperCamelCase_		) < min_length:
										a__     = toks + toks
        # toks_str = [t[1] for t in toks]
				a__     = [t[0] for t in toks]

				# Ensure consistency
				a__     = tokenizer.decode(UpperCamelCase_ , clean_up_tokenization_spaces=UpperCamelCase_		)
				if " " not in output_txt and len(UpperCamelCase_		) > 1:
							a__     = (
							    tokenizer.decode([toks_ids[0]] , clean_up_tokenization_spaces=UpperCamelCase_		)
							    + ' '
							    + tokenizer.decode(toks_ids[1:] , clean_up_tokenization_spaces=UpperCamelCase_		)
							)
				if with_prefix_space:
							a__     = ' ' + output_txt
				a__     = tokenizer.encode(UpperCamelCase_ , add_special_tokens=UpperCamelCase_		)
				return output_txt, output_ids







	def 						__a   (				self			:   Optional[int]		):




				'''simple docstring'''
				a__     = self.ta_base_tokenizer
				a__     = tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]		)
				a__     = tokenizer(["hi", "I went to the gym", ""]		)
				self.assertListEqual(batch_with_eos_added["input_ids"] , batch_without_eos_added["input_ids"]		)







	def 						__a   (				self			:   Tuple		):




				'''simple docstring'''
				a__     = self.ta_base_tokenizer
				a__     = 'Unicode €.'
				a__     = tokenizer(UpperCamelCase_		)
				a__     = [8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
				self.assertEqual(encoded["input_ids"] , UpperCamelCase_		)

				# decoding
				a__     = tokenizer.decode(UpperCamelCase_		)
				self.assertEqual(UpperCamelCase_ , "Unicode €.</s>"		)

				a__     = tokenizer("e è é ê ë"		)
				a__     = [1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
				self.assertEqual(encoded["input_ids"] , UpperCamelCase_		)
				# decoding
				a__     = tokenizer.decode(UpperCamelCase_		)
				self.assertEqual(UpperCamelCase_ , "e è é ê ë</s>"		)

				# encode/decode, but with `encode` instead of `__call__`
				self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"		)		) , "e è é ê ë</s>"		)







	def 						__a   (				self			:   Tuple		):




				'''simple docstring'''
				a__     = self.ta_base_tokenizer
				a__     = ['A long paragraph for summarization.', 'Another paragraph for summarization.']
				# fmt: off
				a__     = [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
				# fmt: on
				a__     = tokenizer(UpperCamelCase_ , padding=UpperCamelCase_ , return_tensors=UpperCamelCase_		)
				self.assertIsInstance(UpperCamelCase_ , UpperCamelCase_		)

				if FRAMEWORK != "jax":
							a__     = list(batch.input_ids.numpy()[0]		)
				else:
							a__     = list(batch.input_ids.tolist()[0]		)

				self.assertListEqual(UpperCamelCase_ , UpperCamelCase_		)

				self.assertEqual((2, 3_7) , batch.input_ids.shape		)
				self.assertEqual((2, 3_7) , batch.attention_mask.shape		)







	def 						__a   (				self			:   Optional[int]		):




				'''simple docstring'''
				a__     = self.ta_base_tokenizer
				a__     = ['A long paragraph for summarization.', 'Another paragraph for summarization.']
				a__     = tokenizer(UpperCamelCase_ , padding=UpperCamelCase_ , return_tensors=UpperCamelCase_		)
				# check if input_ids are returned and no decoder_input_ids
				self.assertIn("input_ids" , UpperCamelCase_		)
				self.assertIn("attention_mask" , UpperCamelCase_		)
				self.assertNotIn("decoder_input_ids" , UpperCamelCase_		)
				self.assertNotIn("decoder_attention_mask" , UpperCamelCase_		)







	def 						__a   (				self			:   Any		):




				'''simple docstring'''
				a__     = self.ta_base_tokenizer
				a__     = [
				    'Summary of the text.',
				    'Another summary.',
				]
				a__     = tokenizer(
				    text_target=UpperCamelCase_ , max_length=3_2 , padding="max_length" , truncation=UpperCamelCase_ , return_tensors=UpperCamelCase_		)
				self.assertEqual(3_2 , targets["input_ids"].shape[1]		)







	def 						__a   (				self			:   Dict		):




				'''simple docstring'''
				a__     = self.ta_base_tokenizer
				a__     = ['A long paragraph for summarization. </s>']
				a__     = ['Summary of the text. </s>']
				# fmt: off
				a__     = [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
				a__     = [8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
				# fmt: on

				a__     = tokenizer(UpperCamelCase_ , text_target=UpperCamelCase_		)

				self.assertEqual(UpperCamelCase_ , batch["input_ids"][0]		)
				self.assertEqual(UpperCamelCase_ , batch["labels"][0]		)







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				a__     = self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''		):
										self.assertNotEqual(tokenizer.model_max_length , 4_2		)

        # Now let's start the test
				a__     = self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''		):
										# Isolate this from the other tests because we save additional tokens/etc
										a__     = tempfile.mkdtemp()

										a__     = ' He is very happy, UNwant\u00E9d,running'
										a__     = tokenizer.encode(UpperCamelCase_ , add_special_tokens=UpperCamelCase_		)
										tokenizer.save_pretrained(UpperCamelCase_		)

										a__     = tokenizer.__class__.from_pretrained(UpperCamelCase_		)
										a__     = after_tokenizer.encode(UpperCamelCase_ , add_special_tokens=UpperCamelCase_		)
										self.assertListEqual(UpperCamelCase_ , UpperCamelCase_		)

										shutil.rmtree(UpperCamelCase_		)

				a__     = self.get_tokenizers(model_max_length=4_2		)
				for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''		):
										# Isolate this from the other tests because we save additional tokens/etc
										a__     = tempfile.mkdtemp()

										a__     = ' He is very happy, UNwant\u00E9d,running'
										tokenizer.add_tokens(["bim", "bambam"]		)
										a__     = tokenizer.additional_special_tokens
										additional_special_tokens.append("new_additional_special_token"		)
										tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}		)
										a__     = tokenizer.encode(UpperCamelCase_ , add_special_tokens=UpperCamelCase_		)
										tokenizer.save_pretrained(UpperCamelCase_		)

										a__     = tokenizer.__class__.from_pretrained(UpperCamelCase_		)
										a__     = after_tokenizer.encode(UpperCamelCase_ , add_special_tokens=UpperCamelCase_		)
										self.assertListEqual(UpperCamelCase_ , UpperCamelCase_		)
										self.assertIn("new_additional_special_token" , after_tokenizer.additional_special_tokens		)
										self.assertEqual(after_tokenizer.model_max_length , 4_2		)

										a__     = tokenizer.__class__.from_pretrained(UpperCamelCase_ , model_max_length=4_3		)
										self.assertEqual(tokenizer.model_max_length , 4_3		)

										shutil.rmtree(UpperCamelCase_		)







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				a__     = []
				if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())		)

				if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())		)

				for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(UpperCamelCase_		)

										with open(os.path.join(UpperCamelCase_ , "special_tokens_map.json"		) , encoding="utf-8"		) as json_file:
													a__     = json.load(UpperCamelCase_		)

										with open(os.path.join(UpperCamelCase_ , "tokenizer_config.json"		) , encoding="utf-8"		) as json_file:
													a__     = json.load(UpperCamelCase_		)

										a__     = [F'''<extra_id_{i}>''' for i in range(1_2_5		)]

										a__     = added_tokens_extra_ids + [
										    'an_additional_special_token'
										]
										a__     = added_tokens_extra_ids + [
										    'an_additional_special_token'
										]

										with open(os.path.join(UpperCamelCase_ , "special_tokens_map.json"		) , "w" , encoding="utf-8"		) as outfile:
													json.dump(UpperCamelCase_ , UpperCamelCase_		)
										with open(os.path.join(UpperCamelCase_ , "tokenizer_config.json"		) , "w" , encoding="utf-8"		) as outfile:
													json.dump(UpperCamelCase_ , UpperCamelCase_		)

										# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
										# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
										# "special_tokens_map.json" files
										a__     = tokenizer_class.from_pretrained(
										    UpperCamelCase_ , )
										self.assertIn(
										    "an_additional_special_token" , tokenizer_without_change_in_init.additional_special_tokens		)
										# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    ["an_additional_special_token"] , tokenizer_without_change_in_init.convert_ids_to_tokens(
										        tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]		)		) , )

										# Now we test that we can change the value of additional_special_tokens in the from_pretrained
										a__     = added_tokens_extra_ids + [AddedToken("a_new_additional_special_token" , lstrip=UpperCamelCase_		)]
										a__     = tokenizer_class.from_pretrained(
										    UpperCamelCase_ , additional_special_tokens=UpperCamelCase_ , )

										self.assertIn("a_new_additional_special_token" , tokenizer.additional_special_tokens		)
										self.assertEqual(
										    ["a_new_additional_special_token"] , tokenizer.convert_ids_to_tokens(
										        tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]		)		) , )







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				a__     = []
				if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())		)

				if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())		)

				for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(UpperCamelCase_		)

										a__     = tokenizer_class.from_pretrained(UpperCamelCase_		)

										self.assertTrue(tokenizer.decode([2_5_5]		) == ""		)







	def 						__a   (				self			:   Optional[int]		):




				'''simple docstring'''
				pass







	def 						__a   (				self			:   str		):




				'''simple docstring'''
				pass







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				pass







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				pass







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				a__     = self.get_tokenizers(fast=UpperCamelCase_ , do_lower_case=UpperCamelCase_		)
				for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''		):
										a__     = ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
										a__     = tokenizer.convert_tokens_to_string(UpperCamelCase_		)

										self.assertIsInstance(UpperCamelCase_ , UpperCamelCase_		)







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				a__     = self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''		):
										a__     = [
										    'bos_token',
										    'eos_token',
										    'unk_token',
										    'sep_token',
										    'pad_token',
										    'cls_token',
										    'mask_token',
										]

										a__     = 0
										a__     = tokenizer.convert_ids_to_tokens(
										    UpperCamelCase_ , skip_special_tokens=UpperCamelCase_		)

										for attr in attributes_list:
													setattr(UpperCamelCase_ , attr + "_id" , UpperCamelCase_		)
													self.assertEqual(getattr(UpperCamelCase_ , UpperCamelCase_		) , UpperCamelCase_		)
													self.assertEqual(getattr(UpperCamelCase_ , attr + "_id"		) , UpperCamelCase_		)

													setattr(UpperCamelCase_ , attr + "_id" , UpperCamelCase_		)
													self.assertEqual(getattr(UpperCamelCase_ , UpperCamelCase_		) , UpperCamelCase_		)
													self.assertEqual(getattr(UpperCamelCase_ , attr + "_id"		) , UpperCamelCase_		)

										setattr(UpperCamelCase_ , "additional_special_tokens_ids" , []		)
										self.assertListEqual(getattr(UpperCamelCase_ , "additional_special_tokens"		) , []		)
										self.assertListEqual(getattr(UpperCamelCase_ , "additional_special_tokens_ids"		) , []		)

										setattr(UpperCamelCase_ , "additional_special_tokens_ids" , [token_id_to_test_setters]		)
										self.assertListEqual(getattr(UpperCamelCase_ , "additional_special_tokens"		) , [token_to_test_setters]		)
										self.assertListEqual(getattr(UpperCamelCase_ , "additional_special_tokens_ids"		) , [token_id_to_test_setters]		)










'''simple docstring'''


# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


lowerCAmelCase_   :   List[Any]       =					[
    "EAGER",
    "AOT_EAGER",
    "INDUCTOR",
    "NVFUSER",
    "AOT_NVFUSER",
    "AOT_CUDAGRAPHS",
    "OFI",
    "FX2TRT",
    "ONNXRT",
    "IPEX",
]







def    _lowerCamelCase  (__lowerCamelCase  :     Tuple					,  __lowerCamelCase  :     Tuple=None					,  __lowerCamelCase  :     Tuple=None					,  __lowerCamelCase  :     List[Any]=None     )   ->    Any:
			a__     = True
			while ask_again:
						a__     = input(__lowerCamelCase     )
						try:
									if default is not None and len(__lowerCamelCase     ) == 0:
												return default
									return convert_value(__lowerCamelCase     ) if convert_value is not None else result
						except Exception:
									if error_message is not None:
												print(__lowerCamelCase     )







def    _lowerCamelCase  (__lowerCamelCase  :     Dict					,  __lowerCamelCase  :     List[Any]=[]					,  __lowerCamelCase  :     str=None					,  __lowerCamelCase  :     List[Any]=0     )   ->    List[Any]:
			a__     = BulletMenu(__lowerCamelCase					,  __lowerCamelCase     )
			a__     = menu.run(default_choice=__lowerCamelCase     )
			return convert_value(__lowerCamelCase     ) if convert_value is not None else result







def    _lowerCamelCase  (__lowerCamelCase  :     Tuple     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return ComputeEnvironment(["LOCAL_MACHINE", "AMAZON_SAGEMAKER"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     List[str]     )   ->    Optional[int]:
			a__     = int(__lowerCamelCase     )
			return DistributedType(["NO", "MULTI_CPU", "MULTI_XPU", "MULTI_GPU", "MULTI_NPU", "TPU"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     List[Any]     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return DynamoBackend(DYNAMO_BACKENDS[value]     ).value







def    _lowerCamelCase  (__lowerCamelCase  :     str     )   ->    str:
			a__     = int(__lowerCamelCase     )
			return PrecisionType(["no", "fp16", "bf16", "fp8"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     int     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return SageMakerDistributedType(["NO", "DATA_PARALLEL", "MODEL_PARALLEL"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     Any     )   ->    List[Any]:
			return {"yes": True, "no": False}[value.lower()]






class       UpperCamelCase__  (		argparse.RawDescriptionHelpFormatter     ):







	def 						__a   (				self			:   Dict , lowerCamelCase			:   int , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Dict		):




				'''simple docstring'''
				a__     = super()._format_usage(lowerCamelCase , lowerCamelCase , lowerCamelCase , lowerCamelCase		)
				a__     = usage.replace("<command> [<args>] " , ""		)
				return usage



