

import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast






class      lowerCamelCase_      (	unittest.TestCase	):




	def        lowerCAmelCase_       (     self						:     List[str]							):
		__A :			Union[str, Any]		    =			0




	@slow
	def        lowerCAmelCase_       (     self						:     Dict							):
		for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
			__A :			Optional[Any]		    =			AutoTokenizer.from_pretrained(__A							)
			self.assertIsNotNone(__A							)
			self.assertIsInstance(__A					,      (BertTokenizer, BertTokenizerFast)							)
			self.assertGreater(len(__A							)					,      0							)

		for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
			__A :			int		    =			AutoTokenizer.from_pretrained(__A							)
			self.assertIsNotNone(__A							)
			self.assertIsInstance(__A					,      (GPTaTokenizer, GPTaTokenizerFast)							)
			self.assertGreater(len(__A							)					,      0							)




	def        lowerCAmelCase_       (     self						:     List[str]							):
		__A :			List[str]		    =			AutoTokenizer.from_pretrained(__A							)
		self.assertIsInstance(__A					,      (BertTokenizer, BertTokenizerFast)							)
		self.assertEqual(tokenizer.vocab_size					,      12							)




	def        lowerCAmelCase_       (     self						:     Union[str, Any]							):
		__A :			int		    =			AutoTokenizer.from_pretrained(__A							)
		self.assertIsInstance(__A					,      (RobertaTokenizer, RobertaTokenizerFast)							)
		self.assertEqual(tokenizer.vocab_size					,      20							)




	def        lowerCAmelCase_       (     self						:     List[str]							):
		__A :			int		    =			AutoConfig.from_pretrained(__A							)
		self.assertIsInstance(__A					,      __A							)
		# Check that tokenizer_type ≠ model_type
		__A :			str		    =			AutoTokenizer.from_pretrained(__A					,      config=__A							)
		self.assertIsInstance(__A					,      (BertTokenizer, BertTokenizerFast)							)
		self.assertEqual(tokenizer.vocab_size					,      12							)




	def        lowerCAmelCase_       (     self						:     Optional[int]							):
		with tempfile.TemporaryDirectory() as tmp_dir:
			shutil.copy("""./tests/fixtures/vocab.txt"""					,      os.path.join(__A					,      """vocab.txt"""							)							)

			__A :			List[str]		    =			AutoTokenizer.from_pretrained(__A					,      tokenizer_type="""bert"""					,      use_fast=__A							)
			self.assertIsInstance(__A					,      __A							)

		with tempfile.TemporaryDirectory() as tmp_dir:
			shutil.copy("""./tests/fixtures/vocab.json"""					,      os.path.join(__A					,      """vocab.json"""							)							)
			shutil.copy("""./tests/fixtures/merges.txt"""					,      os.path.join(__A					,      """merges.txt"""							)							)

			__A :			int		    =			AutoTokenizer.from_pretrained(__A					,      tokenizer_type="""gpt2"""					,      use_fast=__A							)
			self.assertIsInstance(__A					,      __A							)




	@require_tokenizers
	def        lowerCAmelCase_       (     self						:     Dict							):
		with tempfile.TemporaryDirectory() as tmp_dir:
			shutil.copy("""./tests/fixtures/vocab.txt"""					,      os.path.join(__A					,      """vocab.txt"""							)							)

			__A :			List[Any]		    =			AutoTokenizer.from_pretrained(__A					,      tokenizer_type="""bert"""							)
			self.assertIsInstance(__A					,      __A							)

		with tempfile.TemporaryDirectory() as tmp_dir:
			shutil.copy("""./tests/fixtures/vocab.json"""					,      os.path.join(__A					,      """vocab.json"""							)							)
			shutil.copy("""./tests/fixtures/merges.txt"""					,      os.path.join(__A					,      """merges.txt"""							)							)

			__A :			int		    =			AutoTokenizer.from_pretrained(__A					,      tokenizer_type="""gpt2"""							)
			self.assertIsInstance(__A					,      __A							)




	def        lowerCAmelCase_       (     self						:     List[Any]							):
		with pytest.raises(__A							):
			AutoTokenizer.from_pretrained("""./"""					,      tokenizer_type="""xxx"""							)




	@require_tokenizers
	def        lowerCAmelCase_       (     self						:     str							):
		for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
			__A :			List[Any]		    =			tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""							)
			self.assertIsInstance(__A					,      (BertTokenizer, BertTokenizerFast)							)

			if isinstance(__A					,      __A							):
				self.assertEqual(tokenizer.basic_tokenizer.do_lower_case					,      __A							)
			else:
				self.assertEqual(tokenizer.do_lower_case					,      __A							)

			self.assertEqual(tokenizer.model_max_length					,      512							)




	@require_tokenizers
	def        lowerCAmelCase_       (     self						:     Tuple							):
		for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
			with self.assertRaisesRegex(
			    __A					,      """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""					,      ):
				__A :			Union[str, Any]		    =			tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""							)




	def        lowerCAmelCase_       (     self						:     List[str]							):
		# tests: https://github.com/huggingface/transformers/pull/13251
		# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
		# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
		__A :			Union[str, Any]		    =			TOKENIZER_MAPPING.values()
		__A :			int		    =			[]

		for slow_tok, fast_tok in tokenizers:
			if slow_tok is not None:
				tokenizer_names.append(slow_tok.__name__							)

			if fast_tok is not None:
				tokenizer_names.append(fast_tok.__name__							)

		for tokenizer_name in tokenizer_names:
			# must find the right class
			tokenizer_class_from_name(__A							)




	@require_tokenizers
	def        lowerCAmelCase_       (     self						:     Union[str, Any]							):
		self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""					,      use_fast=__A							)					,      __A							)
		self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""							)					,      __A							)




	@require_tokenizers
	def        lowerCAmelCase_       (     self						:     Any							):
		__A :			List[str]		    =			AutoTokenizer.from_pretrained("""distilbert-base-uncased"""					,      do_lower_case=__A							)
		__A :			Dict		    =			"""Hello, world. How are you?"""
		__A :			List[Any]		    =			tokenizer.tokenize(__A							)
		self.assertEqual("""[UNK]"""					,      tokens[0]							)

		__A :			Dict		    =			AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""					,      do_lower_case=__A							)
		__A :			str		    =			tokenizer.tokenize(__A							)
		self.assertEqual("""[UNK]"""					,      tokens[0]							)




	@require_tokenizers
	def        lowerCAmelCase_       (     self						:     str							):
		__A :			Optional[Any]		    =			AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""							)
		self.assertEqual(type(__A							)					,      __A							)
		self.assertEqual(tokenizer.model_max_length					,      512							)
		self.assertEqual(tokenizer.vocab_size					,      3_0000							)
		self.assertEqual(tokenizer.unk_token					,      """[UNK]"""							)
		self.assertEqual(tokenizer.padding_side					,      """right"""							)
		self.assertEqual(tokenizer.truncation_side					,      """right"""							)




	def        lowerCAmelCase_       (     self						:     Optional[int]							):
		__A :			Dict		    =			AutoTokenizer.from_pretrained(__A							)
		self.assertIsInstance(__A					,      (BertTokenizer, BertTokenizerFast)							)
		with tempfile.TemporaryDirectory() as tmp_dir:
			tokenizer.save_pretrained(__A							)
			__A :			int		    =			AutoTokenizer.from_pretrained(__A							)

		self.assertIsInstance(__A					,      tokenizer.__class__							)
		self.assertEqual(tokenizera.vocab_size					,      12							)




	def        lowerCAmelCase_       (     self						:     Dict							):
		__A :			List[Any]		    =			AutoTokenizer.from_pretrained("""ctrl"""							)
		# There is no fast CTRL so this always gives us a slow tokenizer.
		self.assertIsInstance(__A					,      __A							)




	def        lowerCAmelCase_       (     self						:     Any							):
		# Check we can load the tokenizer config of an online model.
		__A :			List[Any]		    =			get_tokenizer_config("""bert-base-cased"""							)
		__A :			int		    =			config.pop("""_commit_hash"""					,      __A							)
		# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
		self.assertEqual(__A					,      {"""do_lower_case""": False}							)

		# This model does not have a tokenizer_config so we get back an empty dict.
		__A :			Optional[Any]		    =			get_tokenizer_config(__A							)
		self.assertDictEqual(__A					,      {}							)

		# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
		__A :			str		    =			AutoTokenizer.from_pretrained(__A							)
		with tempfile.TemporaryDirectory() as tmp_dir:
			tokenizer.save_pretrained(__A							)
			__A :			List[Any]		    =			get_tokenizer_config(__A							)

		# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
		self.assertEqual(config["""tokenizer_class"""]					,      """BertTokenizer"""							)




	def        lowerCAmelCase_       (     self						:     List[str]							):
		try:
			AutoConfig.register("""custom"""					,      __A							)

			AutoTokenizer.register(__A					,      slow_tokenizer_class=__A							)
			# Trying to register something existing in the Transformers library will raise an error
			with self.assertRaises(__A							):
				AutoTokenizer.register(__A					,      slow_tokenizer_class=__A							)

			__A :			Union[str, Any]		    =			CustomTokenizer.from_pretrained(__A							)
			with tempfile.TemporaryDirectory() as tmp_dir:
				tokenizer.save_pretrained(__A							)

				__A :			Union[str, Any]		    =			AutoTokenizer.from_pretrained(__A							)
				self.assertIsInstance(__A					,      __A							)

		finally:
			if "custom" in CONFIG_MAPPING._extra_content:
				del CONFIG_MAPPING._extra_content["custom"]
			if CustomConfig in TOKENIZER_MAPPING._extra_content:
				del TOKENIZER_MAPPING._extra_content[CustomConfig]




	@require_tokenizers
	def        lowerCAmelCase_       (     self						:     str							):
		try:
			AutoConfig.register("""custom"""					,      __A							)

			# Can register in two steps
			AutoTokenizer.register(__A					,      slow_tokenizer_class=__A							)
			self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,      (CustomTokenizer, None)							)
			AutoTokenizer.register(__A					,      fast_tokenizer_class=__A							)
			self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,      (CustomTokenizer, CustomTokenizerFast)							)

			del TOKENIZER_MAPPING._extra_content[CustomConfig]
			# Can register in one step
			AutoTokenizer.register(
			    __A					,      slow_tokenizer_class=__A					,      fast_tokenizer_class=__A							)
			self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,      (CustomTokenizer, CustomTokenizerFast)							)

			# Trying to register something existing in the Transformers library will raise an error
			with self.assertRaises(__A							):
				AutoTokenizer.register(__A					,      fast_tokenizer_class=__A							)

			# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
			# and that model does not have a tokenizer.json
			with tempfile.TemporaryDirectory() as tmp_dir:
				__A :			List[str]		    =			BertTokenizerFast.from_pretrained(__A							)
				bert_tokenizer.save_pretrained(__A							)
				__A :			List[str]		    =			CustomTokenizerFast.from_pretrained(__A							)

			with tempfile.TemporaryDirectory() as tmp_dir:
				tokenizer.save_pretrained(__A							)

				__A :			Tuple		    =			AutoTokenizer.from_pretrained(__A							)
				self.assertIsInstance(__A					,      __A							)

				__A :			List[str]		    =			AutoTokenizer.from_pretrained(__A					,      use_fast=__A							)
				self.assertIsInstance(__A					,      __A							)

		finally:
			if "custom" in CONFIG_MAPPING._extra_content:
				del CONFIG_MAPPING._extra_content["custom"]
			if CustomConfig in TOKENIZER_MAPPING._extra_content:
				del TOKENIZER_MAPPING._extra_content[CustomConfig]




	def        lowerCAmelCase_       (     self						:     Optional[Any]							):
		# If remote code is not set, we will time out when asking whether to load the model.
		with self.assertRaises(__A							):
			__A :			List[str]		    =			AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
		# If remote code is disabled, we can't load this config.
		with self.assertRaises(__A							):
			__A :			Optional[int]		    =			AutoTokenizer.from_pretrained(
			    """hf-internal-testing/test_dynamic_tokenizer"""					,      trust_remote_code=__A							)

		__A :			Optional[int]		    =			AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""					,      trust_remote_code=__A							)
		self.assertTrue(tokenizer.special_attribute_present							)
		# Test tokenizer can be reloaded.
		with tempfile.TemporaryDirectory() as tmp_dir:
			tokenizer.save_pretrained(__A							)
			__A :			Tuple		    =			AutoTokenizer.from_pretrained(__A					,      trust_remote_code=__A							)
		self.assertTrue(reloaded_tokenizer.special_attribute_present							)

		if is_tokenizers_available():
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizerFast"""							)
			self.assertEqual(reloaded_tokenizer.__class__.__name__					,      """NewTokenizerFast"""							)

			# Test we can also load the slow version
			__A :			Optional[int]		    =			AutoTokenizer.from_pretrained(
			    """hf-internal-testing/test_dynamic_tokenizer"""					,      trust_remote_code=__A					,      use_fast=__A							)
			self.assertTrue(tokenizer.special_attribute_present							)
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizer"""							)
			# Test tokenizer can be reloaded.
			with tempfile.TemporaryDirectory() as tmp_dir:
				tokenizer.save_pretrained(__A							)
				__A :			Dict		    =			AutoTokenizer.from_pretrained(__A					,      trust_remote_code=__A					,      use_fast=__A							)
			self.assertEqual(reloaded_tokenizer.__class__.__name__					,      """NewTokenizer"""							)
			self.assertTrue(reloaded_tokenizer.special_attribute_present							)
		else:
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizer"""							)
			self.assertEqual(reloaded_tokenizer.__class__.__name__					,      """NewTokenizer"""							)




	@require_tokenizers
	def        lowerCAmelCase_       (     self						:     Any							):






		class      lowerCamelCase_      (	_lowercase	):
			_lowercase		:   int     					=				False






		class      lowerCamelCase_      (	_lowercase	):
			_lowercase		:   Optional[int]     					=				NewTokenizer
			_lowercase		:   Dict     					=				False

		try:
			AutoConfig.register("""custom"""					,      __A							)
			AutoTokenizer.register(__A					,      slow_tokenizer_class=__A							)
			AutoTokenizer.register(__A					,      fast_tokenizer_class=__A							)
			# If remote code is not set, the default is to use local
			__A :			Optional[int]		    =			AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizerFast"""							)
			self.assertFalse(tokenizer.special_attribute_present							)
			__A :			Any		    =			AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""					,      use_fast=__A							)
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizer"""							)
			self.assertFalse(tokenizer.special_attribute_present							)

			# If remote code is disabled, we load the local one.
			__A :			Any		    =			AutoTokenizer.from_pretrained(
			    """hf-internal-testing/test_dynamic_tokenizer"""					,      trust_remote_code=__A							)
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizerFast"""							)
			self.assertFalse(tokenizer.special_attribute_present							)
			__A :			Any		    =			AutoTokenizer.from_pretrained(
			    """hf-internal-testing/test_dynamic_tokenizer"""					,      trust_remote_code=__A					,      use_fast=__A							)
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizer"""							)
			self.assertFalse(tokenizer.special_attribute_present							)

			# If remote is enabled, we load from the Hub
			__A :			Dict		    =			AutoTokenizer.from_pretrained(
			    """hf-internal-testing/test_dynamic_tokenizer"""					,      trust_remote_code=__A							)
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizerFast"""							)
			self.assertTrue(tokenizer.special_attribute_present							)
			__A :			str		    =			AutoTokenizer.from_pretrained(
			    """hf-internal-testing/test_dynamic_tokenizer"""					,      trust_remote_code=__A					,      use_fast=__A							)
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizer"""							)
			self.assertTrue(tokenizer.special_attribute_present							)

		finally:
			if "custom" in CONFIG_MAPPING._extra_content:
				del CONFIG_MAPPING._extra_content["custom"]
			if CustomConfig in TOKENIZER_MAPPING._extra_content:
				del TOKENIZER_MAPPING._extra_content[CustomConfig]




	def        lowerCAmelCase_       (     self						:     str							):
		__A :			Optional[int]		    =			AutoTokenizer.from_pretrained(
		    """hf-internal-testing/test_dynamic_tokenizer_legacy"""					,      trust_remote_code=__A							)
		self.assertTrue(tokenizer.special_attribute_present							)
		if is_tokenizers_available():
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizerFast"""							)

			# Test we can also load the slow version
			__A :			int		    =			AutoTokenizer.from_pretrained(
			    """hf-internal-testing/test_dynamic_tokenizer_legacy"""					,      trust_remote_code=__A					,      use_fast=__A							)
			self.assertTrue(tokenizer.special_attribute_present							)
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizer"""							)
		else:
			self.assertEqual(tokenizer.__class__.__name__					,      """NewTokenizer"""							)




	def        lowerCAmelCase_       (     self						:     Union[str, Any]							):
		with self.assertRaisesRegex(
		    __A					,      """bert-base is not a local folder and is not a valid model identifier"""							):
			__A :			List[Any]		    =			AutoTokenizer.from_pretrained("""bert-base"""							)




	def        lowerCAmelCase_       (     self						:     str							):
		with self.assertRaisesRegex(
		    __A					,      R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
			__A :			Dict		    =			AutoTokenizer.from_pretrained(__A					,      revision="""aaaaaa"""							)




	def        lowerCAmelCase_       (     self						:     int							):
		# Make sure we have cached the tokenizer.
		__A :			Dict		    =			AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
		with RequestCounter() as counter:
			__A :			int		    =			AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
			self.assertEqual(counter.get_request_count					,      0							)
			self.assertEqual(counter.head_request_count					,      1							)
			self.assertEqual(counter.other_request_count					,      0							)








from __future__ import annotations
def 							__SCREAMING_SNAKE_CASE     (   a__    :   List[str]			,a__    :   Dict			,a__    :   Union[str, Any]			,a__    :   Any       )					->				Optional[int]:  # noqa: E741
	while r - l > 1:
		__A :			Any		    =			(l + r) // 2
		if v[m] >= key:
			__A :			Optional[int]		    =			m
		else:
			__A :			List[Any]		    =			m  # noqa: E741
	return r
def 							__SCREAMING_SNAKE_CASE     (   a__    :   list[int]       )					->				int:
	if len(a__       ) == 0:
		return 0

	__A :			str		    =			[0] * len(a__       )
	__A :			List[str]		    =			1

	__A :			List[Any]		    =			v[0]

	for i in range(1			,len(a__       )       ):
		if v[i] < tail[0]:
			__A :			int		    =			v[i]
		elif v[i] > tail[length - 1]:
			__A :			Union[str, Any]		    =			v[i]
			length += 1
		else:
			__A :			Any		    =			v[i]

	return length


if __name__ == "__main__":
			import doctest

			doctest.testmod()





