


import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin







class 	__lowercase    (__SCREAMING_SNAKE_CASE	,       unittest.TestCase  ):





							"""simple docstring"""







							_UpperCAmelCase      =						CanineTokenizer
							_UpperCAmelCase      =						False






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										super().setUp()
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					CanineTokenizer()
										tokenizer.save_pretrained(self.tmpdirname       )






							@cached_property
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										return CanineTokenizer.from_pretrained('google/canine-s'       )






							def 					UpperCamelCase__     (      self			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					self.tokenizer_class.from_pretrained(self.tmpdirname			,						**lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Tuple					=					1_0_2_4
										return tokenizer






							@require_torch
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					self.canine_tokenizer
										SCREAMING_SNAKE_CASE_		:     List[str]					=					['Life is like a box of chocolates.', 'You never know what you\'re gonna get.']
										# fmt: off
										SCREAMING_SNAKE_CASE_		:     Tuple					=					[5_7_3_4_4, 7_6, 1_0_5, 1_0_2, 1_0_1, 3_2, 1_0_5, 1_1_5, 3_2, 1_0_8, 1_0_5, 1_0_7, 1_0_1, 3_2, 9_7, 3_2, 9_8, 1_1_1, 1_2_0, 3_2, 1_1_1, 1_0_2, 3_2, 9_9, 1_0_4, 1_1_1, 9_9, 1_1_1, 1_0_8, 9_7, 1_1_6, 1_0_1, 1_1_5, 4_6, 5_7_3_4_5, 0, 0, 0, 0]
										# fmt: on
										SCREAMING_SNAKE_CASE_		:     Dict					=					tokenizer(lowerCAmelCase__			,						padding=lowerCAmelCase__			,						return_tensors='pt'       )
										self.assertIsInstance(lowerCAmelCase__			,						lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					list(batch.input_ids.numpy()[0]       )

										self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )

										self.assertEqual((2, 3_9)			,						batch.input_ids.shape       )
										self.assertEqual((2, 3_9)			,						batch.attention_mask.shape       )






							@require_torch
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					self.canine_tokenizer
										SCREAMING_SNAKE_CASE_		:     Dict					=					['Once there was a man.', 'He wrote a test in HuggingFace Tranformers.']
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tokenizer(lowerCAmelCase__			,						padding=lowerCAmelCase__			,						return_tensors='pt'       )
										# check if input_ids, attention_mask and token_type_ids are returned
										self.assertIn('input_ids'			,						lowerCAmelCase__       )
										self.assertIn('attention_mask'			,						lowerCAmelCase__       )
										self.assertIn('token_type_ids'			,						lowerCAmelCase__       )






							@require_torch
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					self.canine_tokenizer
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					[
										    'What\'s the weater?',
										    'It\'s about 25 degrees.',
										]
										SCREAMING_SNAKE_CASE_		:     int					=					tokenizer(
										    text_target=lowerCAmelCase__			,						max_length=3_2			,						padding='max_length'			,						truncation=lowerCAmelCase__			,						return_tensors='pt'       )
										self.assertEqual(3_2			,						targets['input_ids'].shape[1]       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_tokenizers()
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																self.assertNotEqual(tokenizer.model_max_length			,						4_2       )

        # Now let's start the test
										SCREAMING_SNAKE_CASE_		:     List[str]					=					self.get_tokenizers()
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																# Isolate this from the other tests because we save additional tokens/etc
																SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					tempfile.mkdtemp()

																SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					' He is very happy, UNwant\u00E9d,running'
																SCREAMING_SNAKE_CASE_		:     List[str]					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																tokenizer.save_pretrained(lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Any					=					tokenizer.__class__.from_pretrained(lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     str					=					after_tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )

																shutil.rmtree(lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_tokenizers(model_max_length=4_2       )
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																# Isolate this from the other tests because we save additional tokens/etc
																SCREAMING_SNAKE_CASE_		:     List[Any]					=					tempfile.mkdtemp()

																SCREAMING_SNAKE_CASE_		:     List[str]					=					' He is very happy, UNwant\u00E9d,running'

																SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					tokenizer.additional_special_tokens

																# We can add a new special token for Canine as follows:
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					chr(0XE_0_0_7       )
																additional_special_tokens.append(lowerCAmelCase__       )
																tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}       )
																SCREAMING_SNAKE_CASE_		:     Any					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																tokenizer.save_pretrained(lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tokenizer.__class__.from_pretrained(lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					after_tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )
																self.assertIn(lowerCAmelCase__			,						after_tokenizer.additional_special_tokens       )
																self.assertEqual(after_tokenizer.model_max_length			,						4_2       )

																SCREAMING_SNAKE_CASE_		:     int					=					tokenizer.__class__.from_pretrained(lowerCAmelCase__			,						model_max_length=4_3       )
																self.assertEqual(tokenizer.model_max_length			,						4_3       )

																shutil.rmtree(lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					self.get_tokenizers(do_lower_case=lowerCAmelCase__       )
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     int					=					self.get_clean_sequence(lowerCAmelCase__       )

																# a special token for Canine can be defined as follows:
																SCREAMING_SNAKE_CASE_		:     Dict					=					0XE_0_0_5
																SCREAMING_SNAKE_CASE_		:     Tuple					=					chr(lowerCAmelCase__       )

																tokenizer.add_special_tokens({'cls_token': special_token}       )
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																self.assertEqual(len(lowerCAmelCase__       )			,						1       )

																SCREAMING_SNAKE_CASE_		:     Tuple					=					tokenizer.decode(ids + encoded_special_token			,						clean_up_tokenization_spaces=lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     List[Any]					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Any					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     str					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																self.assertEqual(lowerCAmelCase__			,						input_encoded + special_token_id       )

																SCREAMING_SNAKE_CASE_		:     Any					=					tokenizer.decode(lowerCAmelCase__			,						skip_special_tokens=lowerCAmelCase__       )
																self.assertTrue(special_token not in decoded       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[str]					=					self.get_tokenizers(do_lower_case=lowerCAmelCase__       )
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																SCREAMING_SNAKE_CASE_		:     List[Any]					=					chr(0XE_0_0_5       )
																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					chr(0XE_0_0_6       )

																# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
																tokenizer.add_tokens([SPECIAL_TOKEN_1]			,						special_tokens=lowerCAmelCase__       )
																# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
																# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
																tokenizer.add_special_tokens({'additional_special_tokens': [SPECIAL_TOKEN_2]}       )

																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tokenizer.tokenize(lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     List[Any]					=					tokenizer.tokenize(lowerCAmelCase__       )

																self.assertEqual(len(lowerCAmelCase__       )			,						1       )
																self.assertEqual(len(lowerCAmelCase__       )			,						1       )
																self.assertEqual(token_a[0]			,						lowerCAmelCase__       )
																self.assertEqual(token_a[0]			,						lowerCAmelCase__       )






							@require_tokenizers
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     int					=					self.get_tokenizers(do_lower_case=lowerCAmelCase__       )
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																# a special token for Canine can be defined as follows:
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					0XE_0_0_6
																SCREAMING_SNAKE_CASE_		:     Tuple					=					chr(lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__       )
																tokenizer.add_special_tokens({'additional_special_tokens': [new_token]}       )

																with tempfile.TemporaryDirectory() as tmp_dir_name:
																			tokenizer.save_pretrained(lowerCAmelCase__       )
																			tokenizer.from_pretrained(lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     int					=					[]
										if self.test_slow_tokenizer:
													tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

										if self.test_rust_tokenizer:
													tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

										for tokenizer_class, tokenizer_utils in tokenizer_list:
													with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer_utils.save_pretrained(lowerCAmelCase__       )

																with open(os.path.join(lowerCAmelCase__			,						'special_tokens_map.json'       )			,						encoding='utf-8'       ) as json_file:
																			SCREAMING_SNAKE_CASE_		:     Any					=					json.load(lowerCAmelCase__       )

																with open(os.path.join(lowerCAmelCase__			,						'tokenizer_config.json'       )			,						encoding='utf-8'       ) as json_file:
																			SCREAMING_SNAKE_CASE_		:     List[str]					=					json.load(lowerCAmelCase__       )

																# a special token for Canine can be defined as follows:
																SCREAMING_SNAKE_CASE_		:     int					=					0XE_0_0_6
																SCREAMING_SNAKE_CASE_		:     List[str]					=					chr(lowerCAmelCase__       )

																SCREAMING_SNAKE_CASE_		:     Any					=					[new_token_a]
																SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					[new_token_a]

																with open(os.path.join(lowerCAmelCase__			,						'special_tokens_map.json'       )			,						'w'			,						encoding='utf-8'       ) as outfile:
																			json.dump(lowerCAmelCase__			,						lowerCAmelCase__       )
																with open(os.path.join(lowerCAmelCase__			,						'tokenizer_config.json'       )			,						'w'			,						encoding='utf-8'       ) as outfile:
																			json.dump(lowerCAmelCase__			,						lowerCAmelCase__       )

																# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																# "special_tokens_map.json" files
																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tokenizer_class.from_pretrained(lowerCAmelCase__			,						extra_ids=0       )
																self.assertIn(lowerCAmelCase__			,						tokenizer_without_change_in_init.additional_special_tokens       )
																# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																self.assertEqual(
																    [new_token_a]			,						tokenizer_without_change_in_init.convert_ids_to_tokens(
																        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]       )       )			,						)

																SCREAMING_SNAKE_CASE_		:     List[str]					=					0XE_0_0_7
																SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					chr(lowerCAmelCase__       )
																# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																SCREAMING_SNAKE_CASE_		:     Dict					=					[AddedToken(lowerCAmelCase__			,						lstrip=lowerCAmelCase__       )]
																SCREAMING_SNAKE_CASE_		:     List[Any]					=					tokenizer_class.from_pretrained(
																    lowerCAmelCase__			,						additional_special_tokens=lowerCAmelCase__			,						extra_ids=0       )

																self.assertIn(lowerCAmelCase__			,						tokenizer.additional_special_tokens       )
																# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
																self.assertEqual(
																    [new_token_a]			,						tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]       )       )       )






							@require_tokenizers
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[str]					=					self.get_tokenizers(do_lower_case=lowerCAmelCase__       )
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																SCREAMING_SNAKE_CASE_		:     int					=					'hello world'
																if self.space_between_special_tokens:
																			SCREAMING_SNAKE_CASE_		:     List[str]					=					'[CLS] hello world [SEP]'
																else:
																			SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					input
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     Dict					=					tokenizer.decode(lowerCAmelCase__			,						spaces_between_special_tokens=self.space_between_special_tokens       )
																self.assertIn(lowerCAmelCase__			,						[output, output.lower()]       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					self.get_tokenizers()
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					[
																    'bos_token',
																    'eos_token',
																    'unk_token',
																    'sep_token',
																    'pad_token',
																    'cls_token',
																    'mask_token',
																]

										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					'a'
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					ord(lowerCAmelCase__       )

										for attr in attributes_list:
													setattr(lowerCAmelCase__			,						attr + '_id'			,						lowerCAmelCase__       )
													self.assertEqual(getattr(lowerCAmelCase__			,						lowerCAmelCase__       )			,						lowerCAmelCase__       )
													self.assertEqual(getattr(lowerCAmelCase__			,						attr + '_id'       )			,						lowerCAmelCase__       )

													setattr(lowerCAmelCase__			,						attr + '_id'			,						lowerCAmelCase__       )
													self.assertEqual(getattr(lowerCAmelCase__			,						lowerCAmelCase__       )			,						lowerCAmelCase__       )
													self.assertEqual(getattr(lowerCAmelCase__			,						attr + '_id'       )			,						lowerCAmelCase__       )

										setattr(lowerCAmelCase__			,						'additional_special_tokens_ids'			,						[]       )
										self.assertListEqual(getattr(lowerCAmelCase__			,						'additional_special_tokens'       )			,						[]       )
										self.assertListEqual(getattr(lowerCAmelCase__			,						'additional_special_tokens_ids'       )			,						[]       )

										SCREAMING_SNAKE_CASE_		:     str					=					0XE_0_0_6
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					chr(lowerCAmelCase__       )
										setattr(lowerCAmelCase__			,						'additional_special_tokens_ids'			,						[additional_special_token_id]       )
										self.assertListEqual(getattr(lowerCAmelCase__			,						'additional_special_tokens'       )			,						[additional_special_token]       )
										self.assertListEqual(getattr(lowerCAmelCase__			,						'additional_special_tokens_ids'       )			,						[additional_special_token_id]       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






'''simple docstring'''


from typing import TYPE_CHECKING

from ...file_utils import _LazyModule, is_torch_available
from ...utils import OptionalDependencyNotAvailable


__snake_case						  =		{
    """configuration_gpt_neox_japanese""": ["""GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GPTNeoXJapaneseConfig"""],
    """tokenization_gpt_neox_japanese""": ["""GPTNeoXJapaneseTokenizer"""],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __snake_case						  =		[
           """GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """GPTNeoXJapaneseForCausalLM""",
           """GPTNeoXJapaneseLayer""",
           """GPTNeoXJapaneseModel""",
           """GPTNeoXJapanesePreTrainedModel""",
       ]


if TYPE_CHECKING:
       from .configuration_gpt_neox_japanese import GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoXJapaneseConfig
       from .tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_gpt_neox_japanese import (
                  GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST,
                  GPTNeoXJapaneseForCausalLM,
                  GPTNeoXJapaneseLayer,
                  GPTNeoXJapaneseModel,
                  GPTNeoXJapanesePreTrainedModel,
              )


else:
       import sys

       __snake_case						  =		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

