
import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin





class 			_lowerCAmelCase      (			lowercase__      ,   unittest.TestCase ):
				_lowercase    	=CanineTokenizer
				_lowercase    	=False




				def   __a		(      self  )       ->   Dict:
							super().setUp()
							lowerCAmelCase_      			=			CanineTokenizer()
							tokenizer.save_pretrained(self.tmpdirname  )




				@cached_property
				def   __a		(      self  )       ->   List[Any]:
							return CanineTokenizer.from_pretrained("google/canine-s"  )




				def   __a		(      self       ,       **_UpperCamelCase  )       ->   List[str]:
							lowerCAmelCase_      			=			self.tokenizer_class.from_pretrained(self.tmpdirname       ,       **__lowercase  )
							lowerCAmelCase_      			=			1_024
							return tokenizer




				@require_torch
				def   __a		(      self  )       ->   List[str]:
							lowerCAmelCase_      			=			self.canine_tokenizer
							lowerCAmelCase_      			=			["Life is like a box of chocolates.", "You never know what you're gonna get."]
							# fmt: off
							lowerCAmelCase_      			=			[57_344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57_345, 0, 0, 0, 0]
							# fmt: on
							lowerCAmelCase_      			=			tokenizer(__lowercase       ,       padding=__lowercase       ,       return_tensors="pt"  )
							self.assertIsInstance(__lowercase       ,       __lowercase  )

							lowerCAmelCase_      			=			list(batch.input_ids.numpy()[0]  )

							self.assertListEqual(__lowercase       ,       __lowercase  )

							self.assertEqual((2, 39)       ,       batch.input_ids.shape  )
							self.assertEqual((2, 39)       ,       batch.attention_mask.shape  )




				@require_torch
				def   __a		(      self  )       ->   Optional[Any]:
							lowerCAmelCase_      			=			self.canine_tokenizer
							lowerCAmelCase_      			=			["Once there was a man.", "He wrote a test in HuggingFace Tranformers."]
							lowerCAmelCase_      			=			tokenizer(__lowercase       ,       padding=__lowercase       ,       return_tensors="pt"  )
							# check if input_ids, attention_mask and token_type_ids are returned
							self.assertIn("input_ids"       ,       __lowercase  )
							self.assertIn("attention_mask"       ,       __lowercase  )
							self.assertIn("token_type_ids"       ,       __lowercase  )




				@require_torch
				def   __a		(      self  )       ->   Optional[int]:
							lowerCAmelCase_      			=			self.canine_tokenizer
							lowerCAmelCase_      			=			[
							    "What's the weater?",
							    "It's about 25 degrees.",
							]
							lowerCAmelCase_      			=			tokenizer(
							    text_target=__lowercase       ,       max_length=32       ,       padding="max_length"       ,       truncation=__lowercase       ,       return_tensors="pt"  )
							self.assertEqual(32       ,       targets["input_ids"].shape[1]  )




				def   __a		(      self  )       ->   Optional[Any]:
							# safety check on max_len default value so we are sure the test works
							lowerCAmelCase_      			=			self.get_tokenizers()
							for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
													self.assertNotEqual(tokenizer.model_max_length       ,       42  )

        # Now let's start the test
							lowerCAmelCase_      			=			self.get_tokenizers()
							for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
													# Isolate this from the other tests because we save additional tokens/etc
													lowerCAmelCase_      			=			tempfile.mkdtemp()

													lowerCAmelCase_      			=			" He is very happy, UNwant\u00E9d,running"
													lowerCAmelCase_      			=			tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )
													tokenizer.save_pretrained(__lowercase  )

													lowerCAmelCase_      			=			tokenizer.__class__.from_pretrained(__lowercase  )
													lowerCAmelCase_      			=			after_tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )
													self.assertListEqual(__lowercase       ,       __lowercase  )

													shutil.rmtree(__lowercase  )

							lowerCAmelCase_      			=			self.get_tokenizers(model_max_length=42  )
							for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
													# Isolate this from the other tests because we save additional tokens/etc
													lowerCAmelCase_      			=			tempfile.mkdtemp()

													lowerCAmelCase_      			=			" He is very happy, UNwant\u00E9d,running"

													lowerCAmelCase_      			=			tokenizer.additional_special_tokens

													# We can add a new special token for Canine as follows:
													lowerCAmelCase_      			=			chr(0xE_0_0_7  )
													additional_special_tokens.append(__lowercase  )
													tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}  )
													lowerCAmelCase_      			=			tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )
													tokenizer.save_pretrained(__lowercase  )

													lowerCAmelCase_      			=			tokenizer.__class__.from_pretrained(__lowercase  )
													lowerCAmelCase_      			=			after_tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )
													self.assertListEqual(__lowercase       ,       __lowercase  )
													self.assertIn(__lowercase       ,       after_tokenizer.additional_special_tokens  )
													self.assertEqual(after_tokenizer.model_max_length       ,       42  )

													lowerCAmelCase_      			=			tokenizer.__class__.from_pretrained(__lowercase       ,       model_max_length=43  )
													self.assertEqual(tokenizer.model_max_length       ,       43  )

													shutil.rmtree(__lowercase  )




				def   __a		(      self  )       ->   Optional[Any]:
							lowerCAmelCase_      			=			self.get_tokenizers(do_lower_case=__lowercase  )
							for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
													lowerCAmelCase_			,						lowerCAmelCase_      			=			self.get_clean_sequence(__lowercase  )

													# a special token for Canine can be defined as follows:
													lowerCAmelCase_      			=			0xE_0_0_5
													lowerCAmelCase_      			=			chr(__lowercase  )

													tokenizer.add_special_tokens({"cls_token": special_token}  )
													lowerCAmelCase_      			=			tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )
													self.assertEqual(len(__lowercase  )       ,       1  )

													lowerCAmelCase_      			=			tokenizer.decode(ids + encoded_special_token       ,       clean_up_tokenization_spaces=__lowercase  )
													lowerCAmelCase_      			=			tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )

													lowerCAmelCase_      			=			tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )
													lowerCAmelCase_      			=			tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )
													self.assertEqual(__lowercase       ,       input_encoded + special_token_id  )

													lowerCAmelCase_      			=			tokenizer.decode(__lowercase       ,       skip_special_tokens=__lowercase  )
													self.assertTrue(special_token not in decoded  )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			self.get_tokenizers(do_lower_case=__lowercase  )
							for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
													lowerCAmelCase_      			=			chr(0xE_0_0_5  )
													lowerCAmelCase_      			=			chr(0xE_0_0_6  )

													# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
													tokenizer.add_tokens([SPECIAL_TOKEN_1]       ,       special_tokens=__lowercase  )
													# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
													# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
													tokenizer.add_special_tokens({"additional_special_tokens": [SPECIAL_TOKEN_2]}  )

													lowerCAmelCase_      			=			tokenizer.tokenize(__lowercase  )
													lowerCAmelCase_      			=			tokenizer.tokenize(__lowercase  )

													self.assertEqual(len(__lowercase  )       ,       1  )
													self.assertEqual(len(__lowercase  )       ,       1  )
													self.assertEqual(token_a[0]       ,       __lowercase  )
													self.assertEqual(token_a[0]       ,       __lowercase  )




				@require_tokenizers
				def   __a		(      self  )       ->   List[Any]:
							lowerCAmelCase_      			=			self.get_tokenizers(do_lower_case=__lowercase  )
							for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
													# a special token for Canine can be defined as follows:
													lowerCAmelCase_      			=			0xE_0_0_6
													lowerCAmelCase_      			=			chr(__lowercase  )

													lowerCAmelCase_      			=			AddedToken(__lowercase       ,       lstrip=__lowercase  )
													tokenizer.add_special_tokens({"additional_special_tokens": [new_token]}  )

													with tempfile.TemporaryDirectory() as tmp_dir_name:
																tokenizer.save_pretrained(__lowercase  )
																tokenizer.from_pretrained(__lowercase  )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			[]
							if self.test_slow_tokenizer:
										tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

							if self.test_rust_tokenizer:
										tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

							for tokenizer_class, tokenizer_utils in tokenizer_list:
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer_utils.save_pretrained(__lowercase  )

													with open(os.path.join(__lowercase       ,       "special_tokens_map.json"  )       ,       encoding="utf-8"  ) as json_file:
																lowerCAmelCase_      			=			json.load(__lowercase  )

													with open(os.path.join(__lowercase       ,       "tokenizer_config.json"  )       ,       encoding="utf-8"  ) as json_file:
																lowerCAmelCase_      			=			json.load(__lowercase  )

													# a special token for Canine can be defined as follows:
													lowerCAmelCase_      			=			0xE_0_0_6
													lowerCAmelCase_      			=			chr(__lowercase  )

													lowerCAmelCase_      			=			[new_token_a]
													lowerCAmelCase_      			=			[new_token_a]

													with open(os.path.join(__lowercase       ,       "special_tokens_map.json"  )       ,       "w"       ,       encoding="utf-8"  ) as outfile:
																json.dump(__lowercase       ,       __lowercase  )
													with open(os.path.join(__lowercase       ,       "tokenizer_config.json"  )       ,       "w"       ,       encoding="utf-8"  ) as outfile:
																json.dump(__lowercase       ,       __lowercase  )

													# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
													# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
													# "special_tokens_map.json" files
													lowerCAmelCase_      			=			tokenizer_class.from_pretrained(__lowercase       ,       extra_ids=0  )
													self.assertIn(__lowercase       ,       tokenizer_without_change_in_init.additional_special_tokens  )
													# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
													self.assertEqual(
													    [new_token_a]       ,       tokenizer_without_change_in_init.convert_ids_to_tokens(
													        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]  )  )       ,       )

													lowerCAmelCase_      			=			0xE_0_0_7
													lowerCAmelCase_      			=			chr(__lowercase  )
													# Now we test that we can change the value of additional_special_tokens in the from_pretrained
													lowerCAmelCase_      			=			[AddedToken(__lowercase       ,       lstrip=__lowercase  )]
													lowerCAmelCase_      			=			tokenizer_class.from_pretrained(
													    __lowercase       ,       additional_special_tokens=__lowercase       ,       extra_ids=0  )

													self.assertIn(__lowercase       ,       tokenizer.additional_special_tokens  )
													# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
													self.assertEqual(
													    [new_token_a]       ,       tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]  )  )  )




				@require_tokenizers
				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			self.get_tokenizers(do_lower_case=__lowercase  )
							for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
													lowerCAmelCase_      			=			"hello world"
													if self.space_between_special_tokens:
																lowerCAmelCase_      			=			"[CLS] hello world [SEP]"
													else:
																lowerCAmelCase_      			=			input
													lowerCAmelCase_      			=			tokenizer.encode(__lowercase       ,       add_special_tokens=__lowercase  )
													lowerCAmelCase_      			=			tokenizer.decode(__lowercase       ,       spaces_between_special_tokens=self.space_between_special_tokens  )
													self.assertIn(__lowercase       ,       [output, output.lower()]  )




				def   __a		(      self  )       ->   List[str]:
							lowerCAmelCase_      			=			self.get_tokenizers()
							for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
													lowerCAmelCase_      			=			[
													    "bos_token",
													    "eos_token",
													    "unk_token",
													    "sep_token",
													    "pad_token",
													    "cls_token",
													    "mask_token",
													]

							lowerCAmelCase_      			=			"a"
							lowerCAmelCase_      			=			ord(__lowercase  )

							for attr in attributes_list:
										setattr(__lowercase       ,       attr + "_id"       ,       __lowercase  )
										self.assertEqual(getattr(__lowercase       ,       __lowercase  )       ,       __lowercase  )
										self.assertEqual(getattr(__lowercase       ,       attr + "_id"  )       ,       __lowercase  )

										setattr(__lowercase       ,       attr + "_id"       ,       __lowercase  )
										self.assertEqual(getattr(__lowercase       ,       __lowercase  )       ,       __lowercase  )
										self.assertEqual(getattr(__lowercase       ,       attr + "_id"  )       ,       __lowercase  )

							setattr(__lowercase       ,       "additional_special_tokens_ids"       ,       []  )
							self.assertListEqual(getattr(__lowercase       ,       "additional_special_tokens"  )       ,       []  )
							self.assertListEqual(getattr(__lowercase       ,       "additional_special_tokens_ids"  )       ,       []  )

							lowerCAmelCase_      			=			0xE_0_0_6
							lowerCAmelCase_      			=			chr(__lowercase  )
							setattr(__lowercase       ,       "additional_special_tokens_ids"       ,       [additional_special_token_id]  )
							self.assertListEqual(getattr(__lowercase       ,       "additional_special_tokens"  )       ,       [additional_special_token]  )
							self.assertListEqual(getattr(__lowercase       ,       "additional_special_tokens_ids"  )       ,       [additional_special_token_id]  )




				def   __a		(      self  )       ->   str:
							pass




				def   __a		(      self  )       ->   int:
							pass




				def   __a		(      self  )       ->   Any:
							pass




				def   __a		(      self  )       ->   List[Any]:
							pass




				def   __a		(      self  )       ->   Optional[int]:
							pass




				def   __a		(      self  )       ->   Optional[Any]:
							pass




				def   __a		(      self  )       ->   Optional[Any]:
							pass




				def   __a		(      self  )       ->   Optional[Any]:
							pass





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


_A     =  {}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A     =  ["NllbTokenizer"]

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A     =  ["NllbTokenizerFast"]


if TYPE_CHECKING:
			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_nllb import NllbTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_nllb_fast import NllbTokenizerFast

else:
			import sys

			_A     =  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



