
import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast




class        _SCREAMING_SNAKE_CASE	( unittest.TestCase		):



	def        SCREAMING_SNAKE_CASE_(       self					)							->			str:
					lowerCamelCase_         =     0



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[Any]:




					for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
									self.assertIsNotNone(_SCREAMING_SNAKE_CASE					)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    (BertTokenizer, BertTokenizerFast)					)
									self.assertGreater(len(_SCREAMING_SNAKE_CASE					)		,    0					)

					for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
									self.assertIsNotNone(_SCREAMING_SNAKE_CASE					)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    (GPTaTokenizer, GPTaTokenizerFast)					)
									self.assertGreater(len(_SCREAMING_SNAKE_CASE					)		,    0					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[Any]:
					lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    (BertTokenizer, BertTokenizerFast)					)
					self.assertEqual(tokenizer.vocab_size		,    12					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			int:
					lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    (RobertaTokenizer, RobertaTokenizerFast)					)
					self.assertEqual(tokenizer.vocab_size		,    20					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[Any]:
					lowerCamelCase_         =     AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE					)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)
					# Check that tokenizer_type ≠ model_type
					lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    config=_SCREAMING_SNAKE_CASE					)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    (BertTokenizer, BertTokenizerFast)					)
					self.assertEqual(tokenizer.vocab_size		,    12					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			str:




					with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("./tests/fixtures/vocab.txt"		,    os.path.join(_SCREAMING_SNAKE_CASE		,    "vocab.txt"					)					)

									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    tokenizer_type="bert"		,    use_fast=_SCREAMING_SNAKE_CASE					)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)

					with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("./tests/fixtures/vocab.json"		,    os.path.join(_SCREAMING_SNAKE_CASE		,    "vocab.json"					)					)
									shutil.copy("./tests/fixtures/merges.txt"		,    os.path.join(_SCREAMING_SNAKE_CASE		,    "merges.txt"					)					)

									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    tokenizer_type="gpt2"		,    use_fast=_SCREAMING_SNAKE_CASE					)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)



	@require_tokenizers
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[int]:




					with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("./tests/fixtures/vocab.txt"		,    os.path.join(_SCREAMING_SNAKE_CASE		,    "vocab.txt"					)					)

									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    tokenizer_type="bert"					)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)

					with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("./tests/fixtures/vocab.json"		,    os.path.join(_SCREAMING_SNAKE_CASE		,    "vocab.json"					)					)
									shutil.copy("./tests/fixtures/merges.txt"		,    os.path.join(_SCREAMING_SNAKE_CASE		,    "merges.txt"					)					)

									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    tokenizer_type="gpt2"					)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Dict:




					with pytest.raises(_SCREAMING_SNAKE_CASE					):
									AutoTokenizer.from_pretrained("./"		,    tokenizer_type="xxx"					)



	@require_tokenizers
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[Any]:




					for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									lowerCamelCase_         =     tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"					)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    (BertTokenizer, BertTokenizerFast)					)

									if isinstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					):
													self.assertEqual(tokenizer.basic_tokenizer.do_lower_case		,    _SCREAMING_SNAKE_CASE					)
									else:
													self.assertEqual(tokenizer.do_lower_case		,    _SCREAMING_SNAKE_CASE					)

									self.assertEqual(tokenizer.model_max_length		,    512					)



	@require_tokenizers
	def        SCREAMING_SNAKE_CASE_(       self					)							->			int:




					for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									with self.assertRaisesRegex(
									    _SCREAMING_SNAKE_CASE		,    "julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"		,    ):
													lowerCamelCase_         =     tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[Any]:
					lowerCamelCase_         =     TOKENIZER_MAPPING.values()
					lowerCamelCase_         =     []

					for slow_tok, fast_tok in tokenizers:
									if slow_tok is not None:
													tokenizer_names.append(slow_tok.__name__					)

									if fast_tok is not None:
													tokenizer_names.append(fast_tok.__name__					)

					for tokenizer_name in tokenizer_names:
									# must find the right class
									tokenizer_class_from_name(_SCREAMING_SNAKE_CASE					)



	@require_tokenizers
	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[str]:
					self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"		,    use_fast=_SCREAMING_SNAKE_CASE					)		,    _SCREAMING_SNAKE_CASE					)
					self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"					)		,    _SCREAMING_SNAKE_CASE					)



	@require_tokenizers
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[Any]:
					lowerCamelCase_         =     AutoTokenizer.from_pretrained("distilbert-base-uncased"		,    do_lower_case=_SCREAMING_SNAKE_CASE					)
					lowerCamelCase_         =     "Hello, world. How are you?"
					lowerCamelCase_         =     tokenizer.tokenize(_SCREAMING_SNAKE_CASE					)
					self.assertEqual("[UNK]"		,    tokens[0]					)

					lowerCamelCase_         =     AutoTokenizer.from_pretrained("microsoft/mpnet-base"		,    do_lower_case=_SCREAMING_SNAKE_CASE					)
					lowerCamelCase_         =     tokenizer.tokenize(_SCREAMING_SNAKE_CASE					)
					self.assertEqual("[UNK]"		,    tokens[0]					)



	@require_tokenizers
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[Any]:
					lowerCamelCase_         =     AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"					)
					self.assertEqual(type(_SCREAMING_SNAKE_CASE					)		,    _SCREAMING_SNAKE_CASE					)
					self.assertEqual(tokenizer.model_max_length		,    512					)
					self.assertEqual(tokenizer.vocab_size		,    30000					)
					self.assertEqual(tokenizer.unk_token		,    "[UNK]"					)
					self.assertEqual(tokenizer.padding_side		,    "right"					)
					self.assertEqual(tokenizer.truncation_side		,    "right"					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[int]:
					lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    (BertTokenizer, BertTokenizerFast)					)
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE					)
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)

					self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    tokenizer.__class__					)
					self.assertEqual(tokenizera.vocab_size		,    12					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[int]:
					lowerCamelCase_         =     AutoTokenizer.from_pretrained("ctrl"					)
					# There is no fast CTRL so this always gives us a slow tokenizer.
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Any:
					lowerCamelCase_         =     get_tokenizer_config("bert-base-cased"					)
					lowerCamelCase_         =     config.pop("_commit_hash"		,    _SCREAMING_SNAKE_CASE					)
					# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
					self.assertEqual(_SCREAMING_SNAKE_CASE		,    {"do_lower_case": False}					)

					# This model does not have a tokenizer_config so we get back an empty dict.
					lowerCamelCase_         =     get_tokenizer_config(_SCREAMING_SNAKE_CASE					)
					self.assertDictEqual(_SCREAMING_SNAKE_CASE		,    {}					)

					# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
					lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE					)
									lowerCamelCase_         =     get_tokenizer_config(_SCREAMING_SNAKE_CASE					)

					# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
					self.assertEqual(config["tokenizer_class"]		,    "BertTokenizer"					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Any:




					try:
									AutoConfig.register("custom"		,    _SCREAMING_SNAKE_CASE					)

									AutoTokenizer.register(_SCREAMING_SNAKE_CASE		,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE					)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(_SCREAMING_SNAKE_CASE					):
													AutoTokenizer.register(_SCREAMING_SNAKE_CASE		,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE					)

									lowerCamelCase_         =     CustomTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE					)

													lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
													self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]



	@require_tokenizers
	def        SCREAMING_SNAKE_CASE_(       self					)							->			int:




					try:
									AutoConfig.register("custom"		,    _SCREAMING_SNAKE_CASE					)

									# Can register in two steps
									AutoTokenizer.register(_SCREAMING_SNAKE_CASE		,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,    (CustomTokenizer, None)					)
									AutoTokenizer.register(_SCREAMING_SNAKE_CASE		,    fast_tokenizer_class=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,    (CustomTokenizer, CustomTokenizerFast)					)

									del TOKENIZER_MAPPING._extra_content[CustomConfig]
									# Can register in one step
									AutoTokenizer.register(
									    _SCREAMING_SNAKE_CASE		,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE		,    fast_tokenizer_class=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,    (CustomTokenizer, CustomTokenizerFast)					)

									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(_SCREAMING_SNAKE_CASE					):
													AutoTokenizer.register(_SCREAMING_SNAKE_CASE		,    fast_tokenizer_class=_SCREAMING_SNAKE_CASE					)

									# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
									# and that model does not have a tokenizer.json
									with tempfile.TemporaryDirectory() as tmp_dir:
													lowerCamelCase_         =     BertTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE					)
													bert_tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE					)
													lowerCamelCase_         =     CustomTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE					)

									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE					)

													lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
													self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)

													lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    use_fast=_SCREAMING_SNAKE_CASE					)
													self.assertIsInstance(_SCREAMING_SNAKE_CASE		,    _SCREAMING_SNAKE_CASE					)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]



	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[Any]:




					with self.assertRaises(_SCREAMING_SNAKE_CASE					):
									lowerCamelCase_         =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"					)
					# If remote code is disabled, we can't load this config.
					with self.assertRaises(_SCREAMING_SNAKE_CASE					):
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"		,    trust_remote_code=_SCREAMING_SNAKE_CASE					)

					lowerCamelCase_         =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"		,    trust_remote_code=_SCREAMING_SNAKE_CASE					)
					self.assertTrue(tokenizer.special_attribute_present					)
					# Test tokenizer can be reloaded.
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE					)
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    trust_remote_code=_SCREAMING_SNAKE_CASE					)
					self.assertTrue(reloaded_tokenizer.special_attribute_present					)

					if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizerFast"					)
									self.assertEqual(reloaded_tokenizer.__class__.__name__		,    "NewTokenizerFast"					)

									# Test we can also load the slow version
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"		,    trust_remote_code=_SCREAMING_SNAKE_CASE		,    use_fast=_SCREAMING_SNAKE_CASE					)
									self.assertTrue(tokenizer.special_attribute_present					)
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizer"					)
									# Test tokenizer can be reloaded.
									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE					)
													lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    trust_remote_code=_SCREAMING_SNAKE_CASE		,    use_fast=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(reloaded_tokenizer.__class__.__name__		,    "NewTokenizer"					)
									self.assertTrue(reloaded_tokenizer.special_attribute_present					)
					else:
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizer"					)
									self.assertEqual(reloaded_tokenizer.__class__.__name__		,    "NewTokenizer"					)



	@require_tokenizers
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[int]:




					class        _SCREAMING_SNAKE_CASE	( __snake_case		):
						lowerCAmelCase__   =					False




					class        _SCREAMING_SNAKE_CASE	( __snake_case		):
						lowerCAmelCase__   =					NewTokenizer
						lowerCAmelCase__   =					False

					try:
									AutoConfig.register("custom"		,    _SCREAMING_SNAKE_CASE					)
									AutoTokenizer.register(_SCREAMING_SNAKE_CASE		,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE					)
									AutoTokenizer.register(_SCREAMING_SNAKE_CASE		,    fast_tokenizer_class=_SCREAMING_SNAKE_CASE					)
									# If remote code is not set, the default is to use local
									lowerCamelCase_         =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"					)
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizerFast"					)
									self.assertFalse(tokenizer.special_attribute_present					)
									lowerCamelCase_         =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"		,    use_fast=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizer"					)
									self.assertFalse(tokenizer.special_attribute_present					)

									# If remote code is disabled, we load the local one.
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"		,    trust_remote_code=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizerFast"					)
									self.assertFalse(tokenizer.special_attribute_present					)
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"		,    trust_remote_code=_SCREAMING_SNAKE_CASE		,    use_fast=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizer"					)
									self.assertFalse(tokenizer.special_attribute_present					)

									# If remote is enabled, we load from the Hub
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"		,    trust_remote_code=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizerFast"					)
									self.assertTrue(tokenizer.special_attribute_present					)
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer"		,    trust_remote_code=_SCREAMING_SNAKE_CASE		,    use_fast=_SCREAMING_SNAKE_CASE					)
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizer"					)
									self.assertTrue(tokenizer.special_attribute_present					)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]



	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[str]:
					lowerCamelCase_         =     AutoTokenizer.from_pretrained(
					    "hf-internal-testing/test_dynamic_tokenizer_legacy"		,    trust_remote_code=_SCREAMING_SNAKE_CASE					)
					self.assertTrue(tokenizer.special_attribute_present					)
					if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizerFast"					)

									# Test we can also load the slow version
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(
									    "hf-internal-testing/test_dynamic_tokenizer_legacy"		,    trust_remote_code=_SCREAMING_SNAKE_CASE		,    use_fast=_SCREAMING_SNAKE_CASE					)
									self.assertTrue(tokenizer.special_attribute_present					)
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizer"					)
					else:
									self.assertEqual(tokenizer.__class__.__name__		,    "NewTokenizer"					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			str:




					with self.assertRaisesRegex(
					    _SCREAMING_SNAKE_CASE		,    "bert-base is not a local folder and is not a valid model identifier"					):
									lowerCamelCase_         =     AutoTokenizer.from_pretrained("bert-base"					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			str:




					with self.assertRaisesRegex(
					    _SCREAMING_SNAKE_CASE		,    R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"					):
									lowerCamelCase_         =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		,    revision="aaaaaa"					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Union[str, Any]:
					lowerCamelCase_         =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"					)
					with RequestCounter() as counter:
									lowerCamelCase_         =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"					)
									self.assertEqual(counter.get_request_count		,    0					)
									self.assertEqual(counter.head_request_count		,    1					)
									self.assertEqual(counter.other_request_count		,    0					)










import torch
from torch import nn





class 					A__     (     nn.Module ):

     '''simple docstring'''






     def __init__( self      :    List[str]   ,						_SCREAMING_SNAKE_CASE      :    List[Any]   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]   ,						_SCREAMING_SNAKE_CASE      :    Tuple   ,						_SCREAMING_SNAKE_CASE      :    str   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=1   ,						_SCREAMING_SNAKE_CASE      :    List[str]=False ):






           """simple docstring"""




           super().__init__()

           UpperCamelCase					       =  n_token
           UpperCamelCase					       =  d_embed
           UpperCamelCase					       =  d_proj

           UpperCamelCase					       =  cutoffs + [n_token]
           UpperCamelCase					       =  [0] + self.cutoffs
           UpperCamelCase					       =  div_val

           UpperCamelCase					       =  self.cutoffs[0]
           UpperCamelCase					       =  len(self.cutoffs ) - 1
           UpperCamelCase					       =  self.shortlist_size + self.n_clusters

           if self.n_clusters > 0:
                 UpperCamelCase					       =  nn.Parameter(torch.zeros(self.n_clusters   ,						self.d_embed ) )
                 UpperCamelCase					       =  nn.Parameter(torch.zeros(self.n_clusters ) )

           UpperCamelCase					       =  nn.ModuleList()
           UpperCamelCase					       =  nn.ParameterList()

           if div_val == 1:
                 for i in range(len(self.cutoffs ) ):
                       if d_proj != d_embed:
                             self.out_projs.append(nn.Parameter(torch.FloatTensor(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE ) ) )
                       else:
                             self.out_projs.append(_SCREAMING_SNAKE_CASE )

                 self.out_layers.append(nn.Linear(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE ) )
           else:
                 for i in range(len(self.cutoffs ) ):
                       UpperCamelCase       ,  UpperCamelCase					       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       UpperCamelCase					       =  d_embed // (div_val**i)

                       self.out_projs.append(nn.Parameter(torch.FloatTensor(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE ) ) )

                       self.out_layers.append(nn.Linear(_SCREAMING_SNAKE_CASE   ,						r_idx - l_idx ) )

           UpperCamelCase					       =  keep_order







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]   ,						_SCREAMING_SNAKE_CASE      :    Dict   ,						_SCREAMING_SNAKE_CASE      :    List[Any]   ,						_SCREAMING_SNAKE_CASE      :    Dict ):






           """simple docstring"""




           if proj is None:
                 UpperCamelCase					       =  nn.functional.linear(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						bias=_SCREAMING_SNAKE_CASE )
           else:
                 # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
                 UpperCamelCase					       =  nn.functional.linear(_SCREAMING_SNAKE_CASE   ,						proj.t().contiguous() )
                 UpperCamelCase					       =  nn.functional.linear(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						bias=_SCREAMING_SNAKE_CASE )
                 # else:
                 #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
                 #     if bias is not None:
                 #         logit = logit + bias

           return logit







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any]   ,						_SCREAMING_SNAKE_CASE      :    Union[str, Any]   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]=None   ,						_SCREAMING_SNAKE_CASE      :    Dict=False ):






           """simple docstring"""




           if labels is not None:
                 # Shift so that tokens < n predict n
                 UpperCamelCase					       =  hidden[..., :-1, :].contiguous()
                 UpperCamelCase					       =  labels[..., 1:].contiguous()
                 UpperCamelCase					       =  hidden.view(-1   ,						hidden.size(-1 ) )
                 UpperCamelCase					       =  labels.view(-1 )
                 if hidden.size(0 ) != labels.size(0 ):
                       raise RuntimeError('Input and labels should have the same size in the batch dimension.' )
           else:
                 UpperCamelCase					       =  hidden.view(-1   ,						hidden.size(-1 ) )

           if self.n_clusters == 0:
                 UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						self.out_layers[0].weight   ,						self.out_layers[0].bias   ,						self.out_projs[0] )
                 if labels is not None:
                       UpperCamelCase					       =  labels != -100
                       UpperCamelCase					       =  torch.zeros_like(_SCREAMING_SNAKE_CASE   ,						dtype=hidden.dtype   ,						device=hidden.device )
                       UpperCamelCase					       =  (
                           -nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=-1 )[mask].gather(1   ,						labels[mask].unsqueeze(1 ) ).squeeze(1 )
                       )
                 else:
                       UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=-1 )
           else:
                 # construct weights and biases
                 UpperCamelCase       ,  UpperCamelCase					       =  [], []
                 for i in range(len(self.cutoffs ) ):
                       if self.div_val == 1:
                             UpperCamelCase       ,  UpperCamelCase					       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                             UpperCamelCase					       =  self.out_layers[0].weight[l_idx:r_idx]
                             UpperCamelCase					       =  self.out_layers[0].bias[l_idx:r_idx]
                       else:
                             UpperCamelCase					       =  self.out_layers[i].weight
                             UpperCamelCase					       =  self.out_layers[i].bias

                       if i == 0:
                             UpperCamelCase					       =  torch.cat([weight_i, self.cluster_weight]   ,						dim=0 )
                             UpperCamelCase					       =  torch.cat([bias_i, self.cluster_bias]   ,						dim=0 )

                       weights.append(_SCREAMING_SNAKE_CASE )
                       biases.append(_SCREAMING_SNAKE_CASE )

                 UpperCamelCase       ,  UpperCamelCase       ,  UpperCamelCase					       =  weights[0], biases[0], self.out_projs[0]

                 UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                 UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=1 )

                 if labels is None:
                       UpperCamelCase					       =  hidden.new_empty((head_logit.size(0 ), self.n_token) )
                 else:
                       UpperCamelCase					       =  torch.zeros_like(_SCREAMING_SNAKE_CASE   ,						dtype=hidden.dtype   ,						device=hidden.device )

                 UpperCamelCase					       =  0
                 UpperCamelCase					       =  [0] + self.cutoffs
                 for i in range(len(_SCREAMING_SNAKE_CASE ) - 1 ):
                       UpperCamelCase       ,  UpperCamelCase					       =  cutoff_values[i], cutoff_values[i + 1]

                       if labels is not None:
                             UpperCamelCase					       =  (labels >= l_idx) & (labels < r_idx)
                             UpperCamelCase					       =  mask_i.nonzero().squeeze()

                             if indices_i.numel() == 0:
                                   continue

                             UpperCamelCase					       =  labels.index_select(0   ,						_SCREAMING_SNAKE_CASE ) - l_idx
                             UpperCamelCase					       =  head_logprob.index_select(0   ,						_SCREAMING_SNAKE_CASE )
                             UpperCamelCase					       =  hidden.index_select(0   ,						_SCREAMING_SNAKE_CASE )
                       else:
                             UpperCamelCase					       =  hidden

                       if i == 0:
                             if labels is not None:
                                   UpperCamelCase					       =  head_logprob_i.gather(1   ,						target_i[:, None] ).squeeze(1 )
                             else:
                                   UpperCamelCase					       =  head_logprob[:, : self.cutoffs[0]]
                       else:
                             UpperCamelCase       ,  UpperCamelCase       ,  UpperCamelCase					       =  weights[i], biases[i], self.out_projs[i]

                             UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                             UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=1 )
                             UpperCamelCase					       =  self.cutoffs[0] + i - 1  # No probability for the head cluster
                             if labels is not None:
                                   UpperCamelCase					       =  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                       1   ,						target_i[:, None] ).squeeze(1 )
                             else:
                                   UpperCamelCase					       =  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                                   UpperCamelCase					       =  logprob_i

                       if labels is not None:
                             if (hasattr(self   ,						'keep_order' ) and self.keep_order) or keep_order:
                                   out.index_copy_(0   ,						_SCREAMING_SNAKE_CASE   ,						-logprob_i )
                             else:
                                   out[offset : offset + logprob_i.size(0 )].copy_(-logprob_i )
                             offset += logprob_i.size(0 )

           return out







     def   _SCREAMING_SNAKE_CASE     ( self      :    int   ,						_SCREAMING_SNAKE_CASE      :    str ):






           """simple docstring"""




           if self.n_clusters == 0:
                 UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						self.out_layers[0].weight   ,						self.out_layers[0].bias   ,						self.out_projs[0] )
                 return nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=-1 )
           else:
                 # construct weights and biases
                 UpperCamelCase       ,  UpperCamelCase					       =  [], []
                 for i in range(len(self.cutoffs ) ):
                       if self.div_val == 1:
                             UpperCamelCase       ,  UpperCamelCase					       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                             UpperCamelCase					       =  self.out_layers[0].weight[l_idx:r_idx]
                             UpperCamelCase					       =  self.out_layers[0].bias[l_idx:r_idx]
                       else:
                             UpperCamelCase					       =  self.out_layers[i].weight
                             UpperCamelCase					       =  self.out_layers[i].bias

                       if i == 0:
                             UpperCamelCase					       =  torch.cat([weight_i, self.cluster_weight]   ,						dim=0 )
                             UpperCamelCase					       =  torch.cat([bias_i, self.cluster_bias]   ,						dim=0 )

                       weights.append(_SCREAMING_SNAKE_CASE )
                       biases.append(_SCREAMING_SNAKE_CASE )

                 UpperCamelCase       ,  UpperCamelCase       ,  UpperCamelCase					       =  weights[0], biases[0], self.out_projs[0]
                 UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

                 UpperCamelCase					       =  hidden.new_empty((head_logit.size(0 ), self.n_token) )
                 UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=1 )

                 UpperCamelCase					       =  [0] + self.cutoffs
                 for i in range(len(_SCREAMING_SNAKE_CASE ) - 1 ):
                       UpperCamelCase       ,  UpperCamelCase					       =  cutoff_values[i], cutoff_values[i + 1]

                       if i == 0:
                             UpperCamelCase					       =  head_logprob[:, : self.cutoffs[0]]
                       else:
                             UpperCamelCase       ,  UpperCamelCase       ,  UpperCamelCase					       =  weights[i], biases[i], self.out_projs[i]

                             UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                             UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=1 )

                             UpperCamelCase					       =  head_logprob[:, -i] + tail_logprob_i
                             UpperCamelCase					       =  logprob_i

                 return out



