
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
			_A						      =				"pt"
elif is_tf_available():
			_A						      =				"tf"
else:
			_A						      =				"jax"







class   lowerCamelCase	(						__A							,				unittest.TestCase						):
					UpperCAmelCase__       :							Union[str, Any]     =	ByTaTokenizer
					UpperCAmelCase__       :							str     =	False





					def 	UpperCAmelCase(self   :					str     )		->		List[str]:
									super().setUp()
									snake_case				  =      ByTaTokenizer()
									tokenizer.save_pretrained(self.tmpdirname     )





					@cached_property
					def 	UpperCAmelCase(self   :					Dict     )		->		Tuple:
									return ByTaTokenizer.from_pretrained("google/byt5-small"     )





					def 	UpperCAmelCase(self   :					Optional[Any]       ,					**_A   :					List[Any]     )		->		Optional[Any]:
									return self.tokenizer_class.from_pretrained(self.tmpdirname       ,					**UpperCamelCase__     )





					def 	UpperCAmelCase(self   :					int       ,					_A   :					int       ,					_A   :					Dict=False       ,					_A   :					Optional[int]=2_0       ,					_A   :					Dict=5     )		->		Optional[Any]:
									# XXX The default common tokenizer tests assume that every ID is decodable on its own.
									# This assumption is invalid for ByT5 because single bytes might not be
									# valid utf-8 (byte 128 for instance).
									# Here we're overriding the smallest possible method to provide
									# a clean sequence without making the same assumption.

									snake_case				  =      []
									for i in range(len(UpperCamelCase__     )     ):
													try:
																	snake_case				  =      tokenizer.decode([i]       ,					clean_up_tokenization_spaces=UpperCamelCase__     )
													except UnicodeDecodeError:
																	pass
													toks.append((i, tok)     )

									snake_case				  =      list(filter(lambda _A     : re.match(r"^[ a-zA-Z]+$"       ,					t[1]     )       ,					UpperCamelCase__     )     )
									snake_case				  =      list(filter(lambda _A     : [t[0]] == tokenizer.encode(t[1]       ,					add_special_tokens=UpperCamelCase__     )       ,					UpperCamelCase__     )     )
									if max_length is not None and len(UpperCamelCase__     ) > max_length:
													snake_case				  =      toks[:max_length]
									if min_length is not None and len(UpperCamelCase__     ) < min_length and len(UpperCamelCase__     ) > 0:
													while len(UpperCamelCase__     ) < min_length:
																	snake_case				  =      toks + toks
        # toks_str = [t[1] for t in toks]
									snake_case				  =      [t[0] for t in toks]

									# Ensure consistency
									snake_case				  =      tokenizer.decode(UpperCamelCase__       ,					clean_up_tokenization_spaces=UpperCamelCase__     )
									if " " not in output_txt and len(UpperCamelCase__     ) > 1:
													snake_case				  =      (
													    tokenizer.decode([toks_ids[0]]       ,					clean_up_tokenization_spaces=UpperCamelCase__     )
													    + ' '
													    + tokenizer.decode(toks_ids[1:]       ,					clean_up_tokenization_spaces=UpperCamelCase__     )
													)
									if with_prefix_space:
													snake_case				  =      ' ' + output_txt
									snake_case				  =      tokenizer.encode(UpperCamelCase__       ,					add_special_tokens=UpperCamelCase__     )
									return output_txt, output_ids





					def 	UpperCAmelCase(self   :					Tuple     )		->		Union[str, Any]:
									snake_case				  =      self.ta_base_tokenizer
									snake_case				  =      tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]     )
									snake_case				  =      tokenizer(["hi", "I went to the gym", ""]     )
									self.assertListEqual(batch_with_eos_added["input_ids"]       ,					batch_without_eos_added["input_ids"]     )





					def 	UpperCAmelCase(self   :					int     )		->		int:
									snake_case				  =      self.ta_base_tokenizer
									snake_case				  =      'Unicode €.'
									snake_case				  =      tokenizer(UpperCamelCase__     )
									snake_case				  =      [8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
									self.assertEqual(encoded["input_ids"]       ,					UpperCamelCase__     )

									# decoding
									snake_case				  =      tokenizer.decode(UpperCamelCase__     )
									self.assertEqual(UpperCamelCase__       ,					"Unicode €.</s>"     )

									snake_case				  =      tokenizer("e è é ê ë"     )
									snake_case				  =      [1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
									self.assertEqual(encoded["input_ids"]       ,					UpperCamelCase__     )
									# decoding
									snake_case				  =      tokenizer.decode(UpperCamelCase__     )
									self.assertEqual(UpperCamelCase__       ,					"e è é ê ë</s>"     )

									# encode/decode, but with `encode` instead of `__call__`
									self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"     )     )       ,					"e è é ê ë</s>"     )





					def 	UpperCAmelCase(self   :					List[str]     )		->		Optional[int]:
									snake_case				  =      self.ta_base_tokenizer
									snake_case				  =      ['A long paragraph for summarization.', 'Another paragraph for summarization.']
									# fmt: off
									snake_case				  =      [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
									# fmt: on
									snake_case				  =      tokenizer(UpperCamelCase__       ,					padding=UpperCamelCase__       ,					return_tensors=UpperCamelCase__     )
									self.assertIsInstance(UpperCamelCase__       ,					UpperCamelCase__     )

									if FRAMEWORK != "jax":
													snake_case				  =      list(batch.input_ids.numpy()[0]     )
									else:
													snake_case				  =      list(batch.input_ids.tolist()[0]     )

									self.assertListEqual(UpperCamelCase__       ,					UpperCamelCase__     )

									self.assertEqual((2, 3_7)       ,					batch.input_ids.shape     )
									self.assertEqual((2, 3_7)       ,					batch.attention_mask.shape     )





					def 	UpperCAmelCase(self   :					List[Any]     )		->		Dict:
									snake_case				  =      self.ta_base_tokenizer
									snake_case				  =      ['A long paragraph for summarization.', 'Another paragraph for summarization.']
									snake_case				  =      tokenizer(UpperCamelCase__       ,					padding=UpperCamelCase__       ,					return_tensors=UpperCamelCase__     )
									# check if input_ids are returned and no decoder_input_ids
									self.assertIn("input_ids"       ,					UpperCamelCase__     )
									self.assertIn("attention_mask"       ,					UpperCamelCase__     )
									self.assertNotIn("decoder_input_ids"       ,					UpperCamelCase__     )
									self.assertNotIn("decoder_attention_mask"       ,					UpperCamelCase__     )





					def 	UpperCAmelCase(self   :					Union[str, Any]     )		->		List[str]:
									snake_case				  =      self.ta_base_tokenizer
									snake_case				  =      [
									    'Summary of the text.',
									    'Another summary.',
									]
									snake_case				  =      tokenizer(
									    text_target=UpperCamelCase__       ,					max_length=3_2       ,					padding="max_length"       ,					truncation=UpperCamelCase__       ,					return_tensors=UpperCamelCase__     )
									self.assertEqual(3_2       ,					targets["input_ids"].shape[1]     )





					def 	UpperCAmelCase(self   :					List[str]     )		->		List[Any]:
									snake_case				  =      self.ta_base_tokenizer
									snake_case				  =      ['A long paragraph for summarization. </s>']
									snake_case				  =      ['Summary of the text. </s>']
									# fmt: off
									snake_case				  =      [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
									snake_case				  =      [8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
									# fmt: on

									snake_case				  =      tokenizer(UpperCamelCase__       ,					text_target=UpperCamelCase__     )

									self.assertEqual(UpperCamelCase__       ,					batch["input_ids"][0]     )
									self.assertEqual(UpperCamelCase__       ,					batch["labels"][0]     )





					def 	UpperCAmelCase(self   :					List[str]     )		->		Tuple:
									# safety check on max_len default value so we are sure the test works
									snake_case				  =      self.get_tokenizers()
									for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'     ):
																	self.assertNotEqual(tokenizer.model_max_length       ,					4_2     )

        # Now let's start the test
									snake_case				  =      self.get_tokenizers()
									for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'     ):
																	# Isolate this from the other tests because we save additional tokens/etc
																	snake_case				  =      tempfile.mkdtemp()

																	snake_case				  =      ' He is very happy, UNwant\u00E9d,running'
																	snake_case				  =      tokenizer.encode(UpperCamelCase__       ,					add_special_tokens=UpperCamelCase__     )
																	tokenizer.save_pretrained(UpperCamelCase__     )

																	snake_case				  =      tokenizer.__class__.from_pretrained(UpperCamelCase__     )
																	snake_case				  =      after_tokenizer.encode(UpperCamelCase__       ,					add_special_tokens=UpperCamelCase__     )
																	self.assertListEqual(UpperCamelCase__       ,					UpperCamelCase__     )

																	shutil.rmtree(UpperCamelCase__     )

									snake_case				  =      self.get_tokenizers(model_max_length=4_2     )
									for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'     ):
																	# Isolate this from the other tests because we save additional tokens/etc
																	snake_case				  =      tempfile.mkdtemp()

																	snake_case				  =      ' He is very happy, UNwant\u00E9d,running'
																	tokenizer.add_tokens(["bim", "bambam"]     )
																	snake_case				  =      tokenizer.additional_special_tokens
																	additional_special_tokens.append("new_additional_special_token"     )
																	tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}     )
																	snake_case				  =      tokenizer.encode(UpperCamelCase__       ,					add_special_tokens=UpperCamelCase__     )
																	tokenizer.save_pretrained(UpperCamelCase__     )

																	snake_case				  =      tokenizer.__class__.from_pretrained(UpperCamelCase__     )
																	snake_case				  =      after_tokenizer.encode(UpperCamelCase__       ,					add_special_tokens=UpperCamelCase__     )
																	self.assertListEqual(UpperCamelCase__       ,					UpperCamelCase__     )
																	self.assertIn("new_additional_special_token"       ,					after_tokenizer.additional_special_tokens     )
																	self.assertEqual(after_tokenizer.model_max_length       ,					4_2     )

																	snake_case				  =      tokenizer.__class__.from_pretrained(UpperCamelCase__       ,					model_max_length=4_3     )
																	self.assertEqual(tokenizer.model_max_length       ,					4_3     )

																	shutil.rmtree(UpperCamelCase__     )





					def 	UpperCAmelCase(self   :					int     )		->		int:
									snake_case				  =      []
									if self.test_slow_tokenizer:
													tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

									if self.test_rust_tokenizer:
													tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

									for tokenizer_class, tokenizer_utils in tokenizer_list:
													with tempfile.TemporaryDirectory() as tmp_dir:
																	tokenizer_utils.save_pretrained(UpperCamelCase__     )

																	with open(os.path.join(UpperCamelCase__       ,					"special_tokens_map.json"     )       ,					encoding="utf-8"     ) as json_file:
																					snake_case				  =      json.load(UpperCamelCase__     )

																	with open(os.path.join(UpperCamelCase__       ,					"tokenizer_config.json"     )       ,					encoding="utf-8"     ) as json_file:
																					snake_case				  =      json.load(UpperCamelCase__     )

																	snake_case				  =      [f'<extra_id_{i}>' for i in range(1_2_5     )]

																	snake_case				  =      added_tokens_extra_ids + [
																	    'an_additional_special_token'
																	]
																	snake_case				  =      added_tokens_extra_ids + [
																	    'an_additional_special_token'
																	]

																	with open(os.path.join(UpperCamelCase__       ,					"special_tokens_map.json"     )       ,					"w"       ,					encoding="utf-8"     ) as outfile:
																					json.dump(UpperCamelCase__       ,					UpperCamelCase__     )
																	with open(os.path.join(UpperCamelCase__       ,					"tokenizer_config.json"     )       ,					"w"       ,					encoding="utf-8"     ) as outfile:
																					json.dump(UpperCamelCase__       ,					UpperCamelCase__     )

																	# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																	# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																	# "special_tokens_map.json" files
																	snake_case				  =      tokenizer_class.from_pretrained(
																	    UpperCamelCase__       ,					)
																	self.assertIn(
																	    "an_additional_special_token"       ,					tokenizer_without_change_in_init.additional_special_tokens     )
																	# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																	self.assertEqual(
																	    ["an_additional_special_token"]       ,					tokenizer_without_change_in_init.convert_ids_to_tokens(
																	        tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]     )     )       ,					)

																	# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																	snake_case				  =      added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"       ,					lstrip=UpperCamelCase__     )]
																	snake_case				  =      tokenizer_class.from_pretrained(
																	    UpperCamelCase__       ,					additional_special_tokens=UpperCamelCase__       ,					)

																	self.assertIn("a_new_additional_special_token"       ,					tokenizer.additional_special_tokens     )
																	self.assertEqual(
																	    ["a_new_additional_special_token"]       ,					tokenizer.convert_ids_to_tokens(
																	        tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]     )     )       ,					)





					def 	UpperCAmelCase(self   :					List[Any]     )		->		List[str]:
									snake_case				  =      []
									if self.test_slow_tokenizer:
													tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

									if self.test_rust_tokenizer:
													tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

									for tokenizer_class, tokenizer_utils in tokenizer_list:
													with tempfile.TemporaryDirectory() as tmp_dir:
																	tokenizer_utils.save_pretrained(UpperCamelCase__     )

																	snake_case				  =      tokenizer_class.from_pretrained(UpperCamelCase__     )

																	self.assertTrue(tokenizer.decode([2_5_5]     ) == ""     )





					def 	UpperCAmelCase(self   :					int     )		->		Union[str, Any]:
									pass





					def 	UpperCAmelCase(self   :					Dict     )		->		Optional[Any]:
									pass





					def 	UpperCAmelCase(self   :					str     )		->		str:
									pass





					def 	UpperCAmelCase(self   :					List[str]     )		->		str:
									pass





					def 	UpperCAmelCase(self   :					Dict     )		->		Optional[int]:
									# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
									# and special added tokens as tokens
									snake_case				  =      self.get_tokenizers(fast=UpperCamelCase__       ,					do_lower_case=UpperCamelCase__     )
									for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'     ):
																	snake_case				  =      ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
																	snake_case				  =      tokenizer.convert_tokens_to_string(UpperCamelCase__     )

																	self.assertIsInstance(UpperCamelCase__       ,					UpperCamelCase__     )





					def 	UpperCAmelCase(self   :					Dict     )		->		Union[str, Any]:
									snake_case				  =      self.get_tokenizers()
									for tokenizer in tokenizers:
													with self.subTest(f'{tokenizer.__class__.__name__}'     ):
																	snake_case				  =      [
																	    'bos_token',
																	    'eos_token',
																	    'unk_token',
																	    'sep_token',
																	    'pad_token',
																	    'cls_token',
																	    'mask_token',
																	]

																	snake_case				  =      0
																	snake_case				  =      tokenizer.convert_ids_to_tokens(
																	    UpperCamelCase__       ,					skip_special_tokens=UpperCamelCase__     )

																	for attr in attributes_list:
																					setattr(UpperCamelCase__       ,					attr + "_id"       ,					UpperCamelCase__     )
																					self.assertEqual(getattr(UpperCamelCase__       ,					UpperCamelCase__     )       ,					UpperCamelCase__     )
																					self.assertEqual(getattr(UpperCamelCase__       ,					attr + "_id"     )       ,					UpperCamelCase__     )

																					setattr(UpperCamelCase__       ,					attr + "_id"       ,					UpperCamelCase__     )
																					self.assertEqual(getattr(UpperCamelCase__       ,					UpperCamelCase__     )       ,					UpperCamelCase__     )
																					self.assertEqual(getattr(UpperCamelCase__       ,					attr + "_id"     )       ,					UpperCamelCase__     )

																	setattr(UpperCamelCase__       ,					"additional_special_tokens_ids"       ,					[]     )
																	self.assertListEqual(getattr(UpperCamelCase__       ,					"additional_special_tokens"     )       ,					[]     )
																	self.assertListEqual(getattr(UpperCamelCase__       ,					"additional_special_tokens_ids"     )       ,					[]     )

																	setattr(UpperCamelCase__       ,					"additional_special_tokens_ids"       ,					[token_id_to_test_setters]     )
																	self.assertListEqual(getattr(UpperCamelCase__       ,					"additional_special_tokens"     )       ,					[token_to_test_setters]     )
																	self.assertListEqual(getattr(UpperCamelCase__       ,					"additional_special_tokens_ids"     )       ,					[token_id_to_test_setters]     )



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_A						      =				{
    "configuration_wav2vec2": ["WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP", "Wav2Vec2Config"],
    "feature_extraction_wav2vec2": ["Wav2Vec2FeatureExtractor"],
    "processing_wav2vec2": ["Wav2Vec2Processor"],
    "tokenization_wav2vec2": ["Wav2Vec2CTCTokenizer", "Wav2Vec2Tokenizer"],
}


try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A						      =				[
       "WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST",
       "Wav2Vec2ForAudioFrameClassification",
       "Wav2Vec2ForCTC",
       "Wav2Vec2ForMaskedLM",
       "Wav2Vec2ForPreTraining",
       "Wav2Vec2ForSequenceClassification",
       "Wav2Vec2ForXVector",
       "Wav2Vec2Model",
       "Wav2Vec2PreTrainedModel",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A						      =				[
       "TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST",
       "TFWav2Vec2ForCTC",
       "TFWav2Vec2Model",
       "TFWav2Vec2PreTrainedModel",
       "TFWav2Vec2ForSequenceClassification",
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A						      =				[
       "FlaxWav2Vec2ForCTC",
       "FlaxWav2Vec2ForPreTraining",
       "FlaxWav2Vec2Model",
       "FlaxWav2Vec2PreTrainedModel",
   ]


if TYPE_CHECKING:
   from .configuration_wavaveca import WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP, WavaVecaConfig
   from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
   from .processing_wavaveca import WavaVecaProcessor
   from .tokenization_wavaveca import WavaVecaCTCTokenizer, WavaVecaTokenizer

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_wavaveca import (
          WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
          WavaVecaForAudioFrameClassification,
          WavaVecaForCTC,
          WavaVecaForMaskedLM,
          WavaVecaForPreTraining,
          WavaVecaForSequenceClassification,
          WavaVecaForXVector,
          WavaVecaModel,
          WavaVecaPreTrainedModel,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_wavaveca import (
          TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFWavaVecaForCTC,
          TFWavaVecaForSequenceClassification,
          TFWavaVecaModel,
          TFWavaVecaPreTrainedModel,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_wavaveca import (
          FlaxWavaVecaForCTC,
          FlaxWavaVecaForPreTraining,
          FlaxWavaVecaModel,
          FlaxWavaVecaPreTrainedModel,
      )


else:
   import sys

   _A						      =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

