






import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast






class 						UpperCAmelCase__ (							unittest.TestCase							):







		def  A__      (					self		):
						_A	:				Optional[int]   =      0







		@slow
		def  A__      (					self		):
						for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
										_A	:				str   =      AutoTokenizer.from_pretrained(A__		)
										self.assertIsNotNone(A__		)
										self.assertIsInstance(A__       ,(BertTokenizer, BertTokenizerFast)		)
										self.assertGreater(len(A__		)       ,0		)

						for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
										_A	:				List[Any]   =      AutoTokenizer.from_pretrained(A__		)
										self.assertIsNotNone(A__		)
										self.assertIsInstance(A__       ,(GPTaTokenizer, GPTaTokenizerFast)		)
										self.assertGreater(len(A__		)       ,0		)







		def  A__      (					self		):
						_A	:				List[str]   =      AutoTokenizer.from_pretrained(A__		)
						self.assertIsInstance(A__       ,(BertTokenizer, BertTokenizerFast)		)
						self.assertEqual(tokenizer.vocab_size       ,12		)







		def  A__      (					self		):
						_A	:				int   =      AutoTokenizer.from_pretrained(A__		)
						self.assertIsInstance(A__       ,(RobertaTokenizer, RobertaTokenizerFast)		)
						self.assertEqual(tokenizer.vocab_size       ,20		)







		def  A__      (					self		):
						_A	:				List[Any]   =      AutoConfig.from_pretrained(A__		)
						self.assertIsInstance(A__       ,A__		)
						# Check that tokenizer_type ≠ model_type
						_A	:				str   =      AutoTokenizer.from_pretrained(A__       ,config=A__		)
						self.assertIsInstance(A__       ,(BertTokenizer, BertTokenizerFast)		)
						self.assertEqual(tokenizer.vocab_size       ,12		)







		def  A__      (					self		):
						with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy('''./tests/fixtures/vocab.txt'''       ,os.path.join(A__       ,'''vocab.txt'''		)		)

										_A	:				Any   =      AutoTokenizer.from_pretrained(A__       ,tokenizer_type='''bert'''       ,use_fast=A__		)
										self.assertIsInstance(A__       ,A__		)

						with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy('''./tests/fixtures/vocab.json'''       ,os.path.join(A__       ,'''vocab.json'''		)		)
										shutil.copy('''./tests/fixtures/merges.txt'''       ,os.path.join(A__       ,'''merges.txt'''		)		)

										_A	:				Dict   =      AutoTokenizer.from_pretrained(A__       ,tokenizer_type='''gpt2'''       ,use_fast=A__		)
										self.assertIsInstance(A__       ,A__		)







		@require_tokenizers
		def  A__      (					self		):
						with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy('''./tests/fixtures/vocab.txt'''       ,os.path.join(A__       ,'''vocab.txt'''		)		)

										_A	:				Any   =      AutoTokenizer.from_pretrained(A__       ,tokenizer_type='''bert'''		)
										self.assertIsInstance(A__       ,A__		)

						with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy('''./tests/fixtures/vocab.json'''       ,os.path.join(A__       ,'''vocab.json'''		)		)
										shutil.copy('''./tests/fixtures/merges.txt'''       ,os.path.join(A__       ,'''merges.txt'''		)		)

										_A	:				List[str]   =      AutoTokenizer.from_pretrained(A__       ,tokenizer_type='''gpt2'''		)
										self.assertIsInstance(A__       ,A__		)







		def  A__      (					self		):
						with pytest.raises(A__		):
										AutoTokenizer.from_pretrained('''./'''       ,tokenizer_type='''xxx'''		)







		@require_tokenizers
		def  A__      (					self		):
						for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
										_A	:				int   =      tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''		)
										self.assertIsInstance(A__       ,(BertTokenizer, BertTokenizerFast)		)

										if isinstance(A__       ,A__		):
														self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,A__		)
										else:
														self.assertEqual(tokenizer.do_lower_case       ,A__		)

										self.assertEqual(tokenizer.model_max_length       ,512		)







		@require_tokenizers
		def  A__      (					self		):
						for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
										with self.assertRaisesRegex(
										    A__       ,'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''       ,):
														_A	:				str   =      tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''		)







		def  A__      (					self		):
						# tests: https://github.com/huggingface/transformers/pull/13251
						# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
						# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
						_A	:				Dict   =      TOKENIZER_MAPPING.values()
						_A	:				List[Any]   =      []

						for slow_tok, fast_tok in tokenizers:
										if slow_tok is not None:
														tokenizer_names.append(slow_tok.__name__		)

										if fast_tok is not None:
														tokenizer_names.append(fast_tok.__name__		)

						for tokenizer_name in tokenizer_names:
										# must find the right class
										tokenizer_class_from_name(A__		)







		@require_tokenizers
		def  A__      (					self		):
						self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''       ,use_fast=A__		)       ,A__		)
						self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''		)       ,A__		)







		@require_tokenizers
		def  A__      (					self		):
						_A	:				Tuple   =      AutoTokenizer.from_pretrained('''distilbert-base-uncased'''       ,do_lower_case=A__		)
						_A	:				Dict   =      '''Hello, world. How are you?'''
						_A	:				int   =      tokenizer.tokenize(A__		)
						self.assertEqual('''[UNK]'''       ,tokens[0]		)

						_A	:				Union[str, Any]   =      AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''       ,do_lower_case=A__		)
						_A	:				Any   =      tokenizer.tokenize(A__		)
						self.assertEqual('''[UNK]'''       ,tokens[0]		)







		@require_tokenizers
		def  A__      (					self		):
						_A	:				Optional[Any]   =      AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''		)
						self.assertEqual(type(A__		)       ,A__		)
						self.assertEqual(tokenizer.model_max_length       ,512		)
						self.assertEqual(tokenizer.vocab_size       ,30000		)
						self.assertEqual(tokenizer.unk_token       ,'''[UNK]'''		)
						self.assertEqual(tokenizer.padding_side       ,'''right'''		)
						self.assertEqual(tokenizer.truncation_side       ,'''right'''		)







		def  A__      (					self		):
						_A	:				Union[str, Any]   =      AutoTokenizer.from_pretrained(A__		)
						self.assertIsInstance(A__       ,(BertTokenizer, BertTokenizerFast)		)
						with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(A__		)
										_A	:				str   =      AutoTokenizer.from_pretrained(A__		)

						self.assertIsInstance(A__       ,tokenizer.__class__		)
						self.assertEqual(tokenizera.vocab_size       ,12		)







		def  A__      (					self		):
						_A	:				str   =      AutoTokenizer.from_pretrained('''ctrl'''		)
						# There is no fast CTRL so this always gives us a slow tokenizer.
						self.assertIsInstance(A__       ,A__		)







		def  A__      (					self		):
						# Check we can load the tokenizer config of an online model.
						_A	:				Union[str, Any]   =      get_tokenizer_config('''bert-base-cased'''		)
						_A	:				List[Any]   =      config.pop('''_commit_hash'''       ,A__		)
						# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
						self.assertEqual(A__       ,{'''do_lower_case''': False}		)

						# This model does not have a tokenizer_config so we get back an empty dict.
						_A	:				Dict   =      get_tokenizer_config(A__		)
						self.assertDictEqual(A__       ,{}		)

						# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
						_A	:				Optional[Any]   =      AutoTokenizer.from_pretrained(A__		)
						with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(A__		)
										_A	:				int   =      get_tokenizer_config(A__		)

						# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
						self.assertEqual(config['''tokenizer_class''']       ,'''BertTokenizer'''		)







		def  A__      (					self		):
						try:
										AutoConfig.register('''custom'''       ,A__		)

										AutoTokenizer.register(A__       ,slow_tokenizer_class=A__		)
										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(A__		):
														AutoTokenizer.register(A__       ,slow_tokenizer_class=A__		)

										_A	:				Any   =      CustomTokenizer.from_pretrained(A__		)
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(A__		)

														_A	:				int   =      AutoTokenizer.from_pretrained(A__		)
														self.assertIsInstance(A__       ,A__		)

						finally:
										if "custom" in CONFIG_MAPPING._extra_content:
														del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
														del TOKENIZER_MAPPING._extra_content[CustomConfig]







		@require_tokenizers
		def  A__      (					self		):
						try:
										AutoConfig.register('''custom'''       ,A__		)

										# Can register in two steps
										AutoTokenizer.register(A__       ,slow_tokenizer_class=A__		)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,(CustomTokenizer, None)		)
										AutoTokenizer.register(A__       ,fast_tokenizer_class=A__		)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,(CustomTokenizer, CustomTokenizerFast)		)

										del TOKENIZER_MAPPING._extra_content[CustomConfig]
										# Can register in one step
										AutoTokenizer.register(
										    A__       ,slow_tokenizer_class=A__       ,fast_tokenizer_class=A__		)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,(CustomTokenizer, CustomTokenizerFast)		)

										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(A__		):
														AutoTokenizer.register(A__       ,fast_tokenizer_class=A__		)

										# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
										# and that model does not have a tokenizer.json
										with tempfile.TemporaryDirectory() as tmp_dir:
														_A	:				Optional[int]   =      BertTokenizerFast.from_pretrained(A__		)
														bert_tokenizer.save_pretrained(A__		)
														_A	:				List[str]   =      CustomTokenizerFast.from_pretrained(A__		)

										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(A__		)

														_A	:				str   =      AutoTokenizer.from_pretrained(A__		)
														self.assertIsInstance(A__       ,A__		)

														_A	:				Any   =      AutoTokenizer.from_pretrained(A__       ,use_fast=A__		)
														self.assertIsInstance(A__       ,A__		)

						finally:
										if "custom" in CONFIG_MAPPING._extra_content:
														del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
														del TOKENIZER_MAPPING._extra_content[CustomConfig]







		def  A__      (					self		):
						# If remote code is not set, we will time out when asking whether to load the model.
						with self.assertRaises(A__		):
										_A	:				Tuple   =      AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''		)
						# If remote code is disabled, we can't load this config.
						with self.assertRaises(A__		):
										_A	:				Any   =      AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''       ,trust_remote_code=A__		)

						_A	:				int   =      AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''       ,trust_remote_code=A__		)
						self.assertTrue(tokenizer.special_attribute_present		)
						# Test tokenizer can be reloaded.
						with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(A__		)
										_A	:				Optional[Any]   =      AutoTokenizer.from_pretrained(A__       ,trust_remote_code=A__		)
						self.assertTrue(reloaded_tokenizer.special_attribute_present		)

						if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizerFast'''		)
										self.assertEqual(reloaded_tokenizer.__class__.__name__       ,'''NewTokenizerFast'''		)

										# Test we can also load the slow version
										_A	:				str   =      AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''       ,trust_remote_code=A__       ,use_fast=A__		)
										self.assertTrue(tokenizer.special_attribute_present		)
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizer'''		)
										# Test tokenizer can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(A__		)
														_A	:				Optional[int]   =      AutoTokenizer.from_pretrained(A__       ,trust_remote_code=A__       ,use_fast=A__		)
										self.assertEqual(reloaded_tokenizer.__class__.__name__       ,'''NewTokenizer'''		)
										self.assertTrue(reloaded_tokenizer.special_attribute_present		)
						else:
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizer'''		)
										self.assertEqual(reloaded_tokenizer.__class__.__name__       ,'''NewTokenizer'''		)







		@require_tokenizers
		def  A__      (					self		):






						class 						UpperCAmelCase__ (							__snake_case							):
								__snake_case      :       int     =   False






						class 						UpperCAmelCase__ (							__snake_case							):
								__snake_case      :       Any     =   NewTokenizer
								__snake_case      :       Optional[int]     =   False

						try:
										AutoConfig.register('''custom'''       ,A__		)
										AutoTokenizer.register(A__       ,slow_tokenizer_class=A__		)
										AutoTokenizer.register(A__       ,fast_tokenizer_class=A__		)
										# If remote code is not set, the default is to use local
										_A	:				Optional[int]   =      AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''		)
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizerFast'''		)
										self.assertFalse(tokenizer.special_attribute_present		)
										_A	:				Tuple   =      AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''       ,use_fast=A__		)
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizer'''		)
										self.assertFalse(tokenizer.special_attribute_present		)

										# If remote code is disabled, we load the local one.
										_A	:				Optional[int]   =      AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''       ,trust_remote_code=A__		)
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizerFast'''		)
										self.assertFalse(tokenizer.special_attribute_present		)
										_A	:				Dict   =      AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''       ,trust_remote_code=A__       ,use_fast=A__		)
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizer'''		)
										self.assertFalse(tokenizer.special_attribute_present		)

										# If remote is enabled, we load from the Hub
										_A	:				Dict   =      AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''       ,trust_remote_code=A__		)
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizerFast'''		)
										self.assertTrue(tokenizer.special_attribute_present		)
										_A	:				Optional[int]   =      AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''       ,trust_remote_code=A__       ,use_fast=A__		)
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizer'''		)
										self.assertTrue(tokenizer.special_attribute_present		)

						finally:
										if "custom" in CONFIG_MAPPING._extra_content:
														del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
														del TOKENIZER_MAPPING._extra_content[CustomConfig]







		def  A__      (					self		):
						_A	:				str   =      AutoTokenizer.from_pretrained(
						    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''       ,trust_remote_code=A__		)
						self.assertTrue(tokenizer.special_attribute_present		)
						if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizerFast'''		)

										# Test we can also load the slow version
										_A	:				Any   =      AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''       ,trust_remote_code=A__       ,use_fast=A__		)
										self.assertTrue(tokenizer.special_attribute_present		)
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizer'''		)
						else:
										self.assertEqual(tokenizer.__class__.__name__       ,'''NewTokenizer'''		)







		def  A__      (					self		):
						with self.assertRaisesRegex(
						    A__       ,'''bert-base is not a local folder and is not a valid model identifier'''		):
										_A	:				Tuple   =      AutoTokenizer.from_pretrained('''bert-base'''		)







		def  A__      (					self		):
						with self.assertRaisesRegex(
						    A__       ,r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''		):
										_A	:				Optional[int]   =      AutoTokenizer.from_pretrained(A__       ,revision='''aaaaaa'''		)







		def  A__      (					self		):
						# Make sure we have cached the tokenizer.
						_A	:				int   =      AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
						with RequestCounter() as counter:
										_A	:				Optional[int]   =      AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
										self.assertEqual(counter.get_request_count       ,0		)
										self.assertEqual(counter.head_request_count       ,1		)
										self.assertEqual(counter.other_request_count       ,0		)











from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCamelCase			:	Optional[int]								=logging.get_logger(__name__)

_UpperCamelCase			:	Tuple								={
    'google/canine-s': 'https://huggingface.co/google/canine-s/resolve/main/config.json',
    # See all CANINE models at https://huggingface.co/models?filter=canine
}






class 						UpperCAmelCase__ (							__snake_case							):
		__snake_case      :       Any     =   "canine"







		def __init__(					self       ,A__=768       ,A__=12       ,A__=12       ,A__=3072       ,A__="gelu"       ,A__=0.1       ,A__=0.1       ,A__=16384       ,A__=16       ,A__=0.02       ,A__=1E-12       ,A__=0       ,A__=0xE_000       ,A__=0xE_001       ,A__=4       ,A__=4       ,A__=8       ,A__=16384       ,A__=128       ,**A__       ,):
						super().__init__(pad_token_id=A__       ,bos_token_id=A__       ,eos_token_id=A__       ,**A__		)

						_A	:				int   =      max_position_embeddings
						_A	:				Dict   =      hidden_size
						_A	:				List[str]   =      num_hidden_layers
						_A	:				Optional[Any]   =      num_attention_heads
						_A	:				Optional[Any]   =      intermediate_size
						_A	:				Optional[Any]   =      hidden_act
						_A	:				str   =      hidden_dropout_prob
						_A	:				Optional[int]   =      attention_probs_dropout_prob
						_A	:				Union[str, Any]   =      initializer_range
						_A	:				Dict   =      type_vocab_size
						_A	:				Optional[Any]   =      layer_norm_eps

						# Character config:
						_A	:				Union[str, Any]   =      downsampling_rate
						_A	:				List[str]   =      upsampling_kernel_size
						_A	:				str   =      num_hash_functions
						_A	:				Optional[Any]   =      num_hash_buckets
						_A	:				Optional[Any]   =      local_transformer_stride



