




"""simple docstring"""




import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast






class        __magic_name__				(			unittest.TestCase							):






				'''simple docstring'''





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   0





				@slow
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)
										self.assertIsNotNone(_a					)
										self.assertIsInstance(_a						,					(BertTokenizer, BertTokenizerFast)					)
										self.assertGreater(len(_a					)						,					0					)

							for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)
										self.assertIsNotNone(_a					)
										self.assertIsInstance(_a						,					(GPTaTokenizer, GPTaTokenizerFast)					)
										self.assertGreater(len(_a					)						,					0					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)
							self.assertIsInstance(_a						,					(BertTokenizer, BertTokenizerFast)					)
							self.assertEqual(tokenizer.vocab_size						,					12					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)
							self.assertIsInstance(_a						,					(RobertaTokenizer, RobertaTokenizerFast)					)
							self.assertEqual(tokenizer.vocab_size						,					20					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   AutoConfig.from_pretrained(_a					)
							self.assertIsInstance(_a						,					_a					)
							# Check that tokenizer_type ≠ model_type
							lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					config=_a					)
							self.assertIsInstance(_a						,					(BertTokenizer, BertTokenizerFast)					)
							self.assertEqual(tokenizer.vocab_size						,					12					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy("""./tests/fixtures/vocab.txt"""						,					os.path.join(_a						,					"""vocab.txt"""					)					)

										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					tokenizer_type="""bert"""						,					use_fast=_a					)
										self.assertIsInstance(_a						,					_a					)

							with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy("""./tests/fixtures/vocab.json"""						,					os.path.join(_a						,					"""vocab.json"""					)					)
										shutil.copy("""./tests/fixtures/merges.txt"""						,					os.path.join(_a						,					"""merges.txt"""					)					)

										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					tokenizer_type="""gpt2"""						,					use_fast=_a					)
										self.assertIsInstance(_a						,					_a					)





				@require_tokenizers
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy("""./tests/fixtures/vocab.txt"""						,					os.path.join(_a						,					"""vocab.txt"""					)					)

										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					tokenizer_type="""bert"""					)
										self.assertIsInstance(_a						,					_a					)

							with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy("""./tests/fixtures/vocab.json"""						,					os.path.join(_a						,					"""vocab.json"""					)					)
										shutil.copy("""./tests/fixtures/merges.txt"""						,					os.path.join(_a						,					"""merges.txt"""					)					)

										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					tokenizer_type="""gpt2"""					)
										self.assertIsInstance(_a						,					_a					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							with pytest.raises(_a					):
										AutoTokenizer.from_pretrained("""./"""						,					tokenizer_type="""xxx"""					)





				@require_tokenizers
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
										lowerCamelCase      =   tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""					)
										self.assertIsInstance(_a						,					(BertTokenizer, BertTokenizerFast)					)

										if isinstance(_a						,					_a					):
													self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						,					_a					)
										else:
													self.assertEqual(tokenizer.do_lower_case						,					_a					)

										self.assertEqual(tokenizer.model_max_length						,					512					)





				@require_tokenizers
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
										with self.assertRaisesRegex(
										    _a						,					"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""						,					):
													lowerCamelCase      =   tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							# tests: https://github.com/huggingface/transformers/pull/13251
							# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
							# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
							lowerCamelCase      =   TOKENIZER_MAPPING.values()
							lowerCamelCase      =   []

							for slow_tok, fast_tok in tokenizers:
										if slow_tok is not None:
													tokenizer_names.append(slow_tok.__name__					)

										if fast_tok is not None:
													tokenizer_names.append(fast_tok.__name__					)

							for tokenizer_name in tokenizer_names:
										# must find the right class
										tokenizer_class_from_name(_a					)





				@require_tokenizers
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""						,					use_fast=_a					)						,					_a					)
							self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""					)						,					_a					)





				@require_tokenizers
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   AutoTokenizer.from_pretrained("""distilbert-base-uncased"""						,					do_lower_case=_a					)
							lowerCamelCase      =   """Hello, world. How are you?"""
							lowerCamelCase      =   tokenizer.tokenize(_a					)
							self.assertEqual("""[UNK]"""						,					tokens[0]					)

							lowerCamelCase      =   AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""						,					do_lower_case=_a					)
							lowerCamelCase      =   tokenizer.tokenize(_a					)
							self.assertEqual("""[UNK]"""						,					tokens[0]					)





				@require_tokenizers
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""					)
							self.assertEqual(type(_a					)						,					_a					)
							self.assertEqual(tokenizer.model_max_length						,					512					)
							self.assertEqual(tokenizer.vocab_size						,					30_000					)
							self.assertEqual(tokenizer.unk_token						,					"""[UNK]"""					)
							self.assertEqual(tokenizer.padding_side						,					"""right"""					)
							self.assertEqual(tokenizer.truncation_side						,					"""right"""					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)
							self.assertIsInstance(_a						,					(BertTokenizer, BertTokenizerFast)					)
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(_a					)
										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)

							self.assertIsInstance(_a						,					tokenizer.__class__					)
							self.assertEqual(tokenizera.vocab_size						,					12					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   AutoTokenizer.from_pretrained("""ctrl"""					)
							# There is no fast CTRL so this always gives us a slow tokenizer.
							self.assertIsInstance(_a						,					_a					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							# Check we can load the tokenizer config of an online model.
							lowerCamelCase      =   get_tokenizer_config("""bert-base-cased"""					)
							lowerCamelCase      =   config.pop("""_commit_hash"""						,					_a					)
							# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
							self.assertEqual(_a						,					{"""do_lower_case""": False}					)

							# This model does not have a tokenizer_config so we get back an empty dict.
							lowerCamelCase      =   get_tokenizer_config(_a					)
							self.assertDictEqual(_a						,					{}					)

							# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
							lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(_a					)
										lowerCamelCase      =   get_tokenizer_config(_a					)

							# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
							self.assertEqual(config["""tokenizer_class"""]						,					"""BertTokenizer"""					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							try:
										AutoConfig.register("""custom"""						,					_a					)

										AutoTokenizer.register(_a						,					slow_tokenizer_class=_a					)
										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(_a					):
													AutoTokenizer.register(_a						,					slow_tokenizer_class=_a					)

										lowerCamelCase      =   CustomTokenizer.from_pretrained(_a					)
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_a					)

													lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)
													self.assertIsInstance(_a						,					_a					)

							finally:
										if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]





				@require_tokenizers
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							try:
										AutoConfig.register("""custom"""						,					_a					)

										# Can register in two steps
										AutoTokenizer.register(_a						,					slow_tokenizer_class=_a					)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,					(CustomTokenizer, None)					)
										AutoTokenizer.register(_a						,					fast_tokenizer_class=_a					)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,					(CustomTokenizer, CustomTokenizerFast)					)

										del TOKENIZER_MAPPING._extra_content[CustomConfig]
										# Can register in one step
										AutoTokenizer.register(
										    _a						,					slow_tokenizer_class=_a						,					fast_tokenizer_class=_a					)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,					(CustomTokenizer, CustomTokenizerFast)					)

										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(_a					):
													AutoTokenizer.register(_a						,					fast_tokenizer_class=_a					)

										# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
										# and that model does not have a tokenizer.json
										with tempfile.TemporaryDirectory() as tmp_dir:
													lowerCamelCase      =   BertTokenizerFast.from_pretrained(_a					)
													bert_tokenizer.save_pretrained(_a					)
													lowerCamelCase      =   CustomTokenizerFast.from_pretrained(_a					)

										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_a					)

													lowerCamelCase      =   AutoTokenizer.from_pretrained(_a					)
													self.assertIsInstance(_a						,					_a					)

													lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					use_fast=_a					)
													self.assertIsInstance(_a						,					_a					)

							finally:
										if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							# If remote code is not set, we will time out when asking whether to load the model.
							with self.assertRaises(_a					):
										lowerCamelCase      =   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""					)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(_a					):
										lowerCamelCase      =   AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""						,					trust_remote_code=_a					)

							lowerCamelCase      =   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						,					trust_remote_code=_a					)
							self.assertTrue(tokenizer.special_attribute_present					)
							# Test tokenizer can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(_a					)
										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					trust_remote_code=_a					)
							self.assertTrue(reloaded_tokenizer.special_attribute_present					)

							if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizerFast"""					)
										self.assertEqual(reloaded_tokenizer.__class__.__name__						,					"""NewTokenizerFast"""					)

										# Test we can also load the slow version
										lowerCamelCase      =   AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""						,					trust_remote_code=_a						,					use_fast=_a					)
										self.assertTrue(tokenizer.special_attribute_present					)
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizer"""					)
										# Test tokenizer can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_a					)
													lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					trust_remote_code=_a						,					use_fast=_a					)
										self.assertEqual(reloaded_tokenizer.__class__.__name__						,					"""NewTokenizer"""					)
										self.assertTrue(reloaded_tokenizer.special_attribute_present					)
							else:
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizer"""					)
										self.assertEqual(reloaded_tokenizer.__class__.__name__						,					"""NewTokenizer"""					)





				@require_tokenizers
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""






							class        __magic_name__				(			UpperCAmelCase__							):






											'''simple docstring'''
											__UpperCamelCase         =		False






							class        __magic_name__				(			UpperCAmelCase__							):






											'''simple docstring'''
											__UpperCamelCase         =		NewTokenizer
											__UpperCamelCase         =		False

							try:
										AutoConfig.register("""custom"""						,					_a					)
										AutoTokenizer.register(_a						,					slow_tokenizer_class=_a					)
										AutoTokenizer.register(_a						,					fast_tokenizer_class=_a					)
										# If remote code is not set, the default is to use local
										lowerCamelCase      =   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""					)
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizerFast"""					)
										self.assertFalse(tokenizer.special_attribute_present					)
										lowerCamelCase      =   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						,					use_fast=_a					)
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizer"""					)
										self.assertFalse(tokenizer.special_attribute_present					)

										# If remote code is disabled, we load the local one.
										lowerCamelCase      =   AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""						,					trust_remote_code=_a					)
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizerFast"""					)
										self.assertFalse(tokenizer.special_attribute_present					)
										lowerCamelCase      =   AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""						,					trust_remote_code=_a						,					use_fast=_a					)
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizer"""					)
										self.assertFalse(tokenizer.special_attribute_present					)

										# If remote is enabled, we load from the Hub
										lowerCamelCase      =   AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""						,					trust_remote_code=_a					)
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizerFast"""					)
										self.assertTrue(tokenizer.special_attribute_present					)
										lowerCamelCase      =   AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""						,					trust_remote_code=_a						,					use_fast=_a					)
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizer"""					)
										self.assertTrue(tokenizer.special_attribute_present					)

							finally:
										if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   AutoTokenizer.from_pretrained(
							    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						,					trust_remote_code=_a					)
							self.assertTrue(tokenizer.special_attribute_present					)
							if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizerFast"""					)

										# Test we can also load the slow version
										lowerCamelCase      =   AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						,					trust_remote_code=_a						,					use_fast=_a					)
										self.assertTrue(tokenizer.special_attribute_present					)
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizer"""					)
							else:
										self.assertEqual(tokenizer.__class__.__name__						,					"""NewTokenizer"""					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							with self.assertRaisesRegex(
							    _a						,					"""bert-base is not a local folder and is not a valid model identifier"""					):
										lowerCamelCase      =   AutoTokenizer.from_pretrained("""bert-base"""					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							with self.assertRaisesRegex(
							    _a						,					r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""					):
										lowerCamelCase      =   AutoTokenizer.from_pretrained(_a						,					revision="""aaaaaa"""					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							# Make sure we have cached the tokenizer.
							lowerCamelCase      =   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""					)
							with RequestCounter() as counter:
										lowerCamelCase      =   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""					)
										self.assertEqual(counter.get_request_count						,					0					)
										self.assertEqual(counter.head_request_count						,					1					)
										self.assertEqual(counter.other_request_count						,					0					)









"""simple docstring"""




import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
lowerCAmelCase				:     List[str]					=				object()

# For specifying empty leaf dict `{}`
lowerCAmelCase				:     Any					=				object()



def        a__					(							snake_case__			,					snake_case__							)		->   List[Any]:
			lowerCamelCase      =   tuple((re.compile(x + """$"""							) for x in qs)							)
			for i in range(len(snake_case__							) - len(snake_case__							) + 1							):
						lowerCamelCase      =   [x.match(snake_case__							) for x, y in zip(snake_case__			,					ks[i:]							)]
						if matches and all(snake_case__							):
									return True
			return False



def        a__					(							snake_case__							)		->   str:
			def replace(snake_case__			,					snake_case__							):
						for rule, replacement in rules:
									if _match(snake_case__			,					snake_case__							):
												return replacement
						return val

			return replace



def        a__					(							)		->   Union[str, Any]:
			return [
			    # embeddings
			    (("transformer", "wpe", "embedding"), P("""mp"""			,					snake_case__							)),
			    (("transformer", "wte", "embedding"), P("""mp"""			,					snake_case__							)),
			    # atention
			    (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(snake_case__			,					"""mp"""							)),
			    (("attention", "out_proj", "kernel"), P("""mp"""			,					snake_case__							)),
			    (("attention", "out_proj", "bias"), None),
			    # mlp
			    (("mlp", "c_fc", "kernel"), P(snake_case__			,					"""mp"""							)),
			    (("mlp", "c_fc", "bias"), P("""mp"""							)),
			    (("mlp", "c_proj", "kernel"), P("""mp"""			,					snake_case__							)),
			    (("mlp", "c_proj", "bias"), None),
			    # layer norms
			    ((r"ln_\d+", "bias"), None),
			    ((r"\d+", r"ln_\d+", "scale"), None),
			    (("ln_f", "bias"), None),
			    (("ln_f", "scale"), None),
			]



def        a__					(							snake_case__							)		->   Optional[Any]:
			lowerCamelCase      =   _get_partition_rules()
			lowerCamelCase      =   _replacement_rules(snake_case__							)
			lowerCamelCase      =   {k: _unmatched for k in flatten_dict(snake_case__							)}
			lowerCamelCase      =   {k: replace(snake_case__			,					snake_case__							) for k, v in initd.items()}
			assert _unmatched not in result.values(), "Incomplete partition spec."
			return freeze(unflatten_dict(snake_case__							)							)



