
import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast







class        lowerCAmelCase__      (   unittest.TestCase	):

						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								_UpperCamelCase						    =	0

						@slow
						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
										self.assertIsNotNone(a				)
										self.assertIsInstance(a       ,							(BertTokenizer, BertTokenizerFast)				)
										self.assertGreater(len(a				)       ,							0				)

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
										self.assertIsNotNone(a				)
										self.assertIsInstance(a       ,							(GPTaTokenizer, GPTaTokenizerFast)				)
										self.assertGreater(len(a				)       ,							0				)

						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
								self.assertIsInstance(a       ,							(BertTokenizer, BertTokenizerFast)				)
								self.assertEqual(tokenizer.vocab_size       ,							12				)

						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
								self.assertIsInstance(a       ,							(RobertaTokenizer, RobertaTokenizerFast)				)
								self.assertEqual(tokenizer.vocab_size       ,							20				)

						def  A_				(			self				)					->      str:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoConfig.from_pretrained(a				)
								self.assertIsInstance(a       ,							a				)
								# Check that tokenizer_type ≠ model_type
								_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							config=a				)
								self.assertIsInstance(a       ,							(BertTokenizer, BertTokenizerFast)				)
								self.assertEqual(tokenizer.vocab_size       ,							12				)

						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy("""./tests/fixtures/vocab.txt"""       ,							os.path.join(a       ,							"""vocab.txt"""				)				)

										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							tokenizer_type="""bert"""       ,							use_fast=a				)
										self.assertIsInstance(a       ,							a				)

								with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy("""./tests/fixtures/vocab.json"""       ,							os.path.join(a       ,							"""vocab.json"""				)				)
										shutil.copy("""./tests/fixtures/merges.txt"""       ,							os.path.join(a       ,							"""merges.txt"""				)				)

										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							tokenizer_type="""gpt2"""       ,							use_fast=a				)
										self.assertIsInstance(a       ,							a				)

						@require_tokenizers
						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy("""./tests/fixtures/vocab.txt"""       ,							os.path.join(a       ,							"""vocab.txt"""				)				)

										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							tokenizer_type="""bert"""				)
										self.assertIsInstance(a       ,							a				)

								with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy("""./tests/fixtures/vocab.json"""       ,							os.path.join(a       ,							"""vocab.json"""				)				)
										shutil.copy("""./tests/fixtures/merges.txt"""       ,							os.path.join(a       ,							"""merges.txt"""				)				)

										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							tokenizer_type="""gpt2"""				)
										self.assertIsInstance(a       ,							a				)

						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								with pytest.raises(a				):
										AutoTokenizer.from_pretrained("""./"""       ,							tokenizer_type="""xxx"""				)

						@require_tokenizers
						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
										_UpperCamelCase						    =	tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""				)
										self.assertIsInstance(a       ,							(BertTokenizer, BertTokenizerFast)				)

										if isinstance(a       ,							a				):
												self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,							a				)
										else:
												self.assertEqual(tokenizer.do_lower_case       ,							a				)

										self.assertEqual(tokenizer.model_max_length       ,							5_12				)

						@require_tokenizers
						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
										with self.assertRaisesRegex(
										    a       ,							"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""       ,							):
												_UpperCamelCase						    =	tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""				)

						def  A_				(			self				)					->      str:


								'''simple docstring'''

								_UpperCamelCase						    =	TOKENIZER_MAPPING.values()
								_UpperCamelCase						    =	[]

								for slow_tok, fast_tok in tokenizers:
										if slow_tok is not None:
												tokenizer_names.append(slow_tok.__name__				)

										if fast_tok is not None:
												tokenizer_names.append(fast_tok.__name__				)

								for tokenizer_name in tokenizer_names:
										# must find the right class
										tokenizer_class_from_name(a				)

						@require_tokenizers
						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""       ,							use_fast=a				)       ,							a				)
								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""				)       ,							a				)

						@require_tokenizers
						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""distilbert-base-uncased"""       ,							do_lower_case=a				)
								_UpperCamelCase						    =	"""Hello, world. How are you?"""
								_UpperCamelCase						    =	tokenizer.tokenize(a				)
								self.assertEqual("""[UNK]"""       ,							tokens[0]				)

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""       ,							do_lower_case=a				)
								_UpperCamelCase						    =	tokenizer.tokenize(a				)
								self.assertEqual("""[UNK]"""       ,							tokens[0]				)

						@require_tokenizers
						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""				)
								self.assertEqual(type(a				)       ,							a				)
								self.assertEqual(tokenizer.model_max_length       ,							5_12				)
								self.assertEqual(tokenizer.vocab_size       ,							3_00_00				)
								self.assertEqual(tokenizer.unk_token       ,							"""[UNK]"""				)
								self.assertEqual(tokenizer.padding_side       ,							"""right"""				)
								self.assertEqual(tokenizer.truncation_side       ,							"""right"""				)

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
								self.assertIsInstance(a       ,							(BertTokenizer, BertTokenizerFast)				)
								with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(a				)
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)

								self.assertIsInstance(a       ,							tokenizer.__class__				)
								self.assertEqual(tokenizera.vocab_size       ,							12				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""ctrl"""				)
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(a       ,							a				)

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	get_tokenizer_config("""bert-base-cased"""				)
								_UpperCamelCase						    =	config.pop("""_commit_hash"""       ,							a				)
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(a       ,							{"""do_lower_case""": False}				)

								# This model does not have a tokenizer_config so we get back an empty dict.
								_UpperCamelCase						    =	get_tokenizer_config(a				)
								self.assertDictEqual(a       ,							{}				)

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
								with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(a				)
										_UpperCamelCase						    =	get_tokenizer_config(a				)

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config["""tokenizer_class"""]       ,							"""BertTokenizer"""				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								try:
										AutoConfig.register("""custom"""       ,							a				)

										AutoTokenizer.register(a       ,							slow_tokenizer_class=a				)
										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(a				):
												AutoTokenizer.register(a       ,							slow_tokenizer_class=a				)

										_UpperCamelCase						    =	CustomTokenizer.from_pretrained(a				)
										with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(a				)

												_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
												self.assertIsInstance(a       ,							a				)

								finally:
										if "custom" in CONFIG_MAPPING._extra_content:
												del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
												del TOKENIZER_MAPPING._extra_content[CustomConfig]

						@require_tokenizers
						def  A_				(			self				)					->      int:


								'''simple docstring'''

								try:
										AutoConfig.register("""custom"""       ,							a				)

										# Can register in two steps
										AutoTokenizer.register(a       ,							slow_tokenizer_class=a				)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,							(CustomTokenizer, None)				)
										AutoTokenizer.register(a       ,							fast_tokenizer_class=a				)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,							(CustomTokenizer, CustomTokenizerFast)				)

										del TOKENIZER_MAPPING._extra_content[CustomConfig]
										# Can register in one step
										AutoTokenizer.register(
										    a       ,							slow_tokenizer_class=a       ,							fast_tokenizer_class=a				)
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,							(CustomTokenizer, CustomTokenizerFast)				)

										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(a				):
												AutoTokenizer.register(a       ,							fast_tokenizer_class=a				)

										# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
										# and that model does not have a tokenizer.json
										with tempfile.TemporaryDirectory() as tmp_dir:
												_UpperCamelCase						    =	BertTokenizerFast.from_pretrained(a				)
												bert_tokenizer.save_pretrained(a				)
												_UpperCamelCase						    =	CustomTokenizerFast.from_pretrained(a				)

										with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(a				)

												_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
												self.assertIsInstance(a       ,							a				)

												_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							use_fast=a				)
												self.assertIsInstance(a       ,							a				)

								finally:
										if "custom" in CONFIG_MAPPING._extra_content:
												del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
												del TOKENIZER_MAPPING._extra_content[CustomConfig]

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								with self.assertRaises(a				):
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""				)
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(a				):
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""       ,							trust_remote_code=a				)

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       ,							trust_remote_code=a				)
								self.assertTrue(tokenizer.special_attribute_present				)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(a				)
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							trust_remote_code=a				)
								self.assertTrue(reloaded_tokenizer.special_attribute_present				)

								if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizerFast"""				)
										self.assertEqual(reloaded_tokenizer.__class__.__name__       ,							"""NewTokenizerFast"""				)

										# Test we can also load the slow version
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""       ,							trust_remote_code=a       ,							use_fast=a				)
										self.assertTrue(tokenizer.special_attribute_present				)
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)
										# Test tokenizer can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(a				)
												_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							trust_remote_code=a       ,							use_fast=a				)
										self.assertEqual(reloaded_tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)
										self.assertTrue(reloaded_tokenizer.special_attribute_present				)
								else:
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)
										self.assertEqual(reloaded_tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)

						@require_tokenizers
						def  A_				(			self				)					->      Optional[int]:


								'''simple docstring'''

								class        lowerCAmelCase__      (   __lowercase	):
														UpperCamelCase_							: int	=		False







								class        lowerCAmelCase__      (   __lowercase	):
														UpperCamelCase_							: Optional[Any]	=		NewTokenizer
														UpperCamelCase_							: Union[str, Any]	=		False

								try:
										AutoConfig.register("""custom"""       ,							a				)
										AutoTokenizer.register(a       ,							slow_tokenizer_class=a				)
										AutoTokenizer.register(a       ,							fast_tokenizer_class=a				)
										# If remote code is not set, the default is to use local
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""				)
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizerFast"""				)
										self.assertFalse(tokenizer.special_attribute_present				)
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       ,							use_fast=a				)
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)
										self.assertFalse(tokenizer.special_attribute_present				)

										# If remote code is disabled, we load the local one.
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""       ,							trust_remote_code=a				)
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizerFast"""				)
										self.assertFalse(tokenizer.special_attribute_present				)
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""       ,							trust_remote_code=a       ,							use_fast=a				)
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)
										self.assertFalse(tokenizer.special_attribute_present				)

										# If remote is enabled, we load from the Hub
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""       ,							trust_remote_code=a				)
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizerFast"""				)
										self.assertTrue(tokenizer.special_attribute_present				)
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer"""       ,							trust_remote_code=a       ,							use_fast=a				)
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)
										self.assertTrue(tokenizer.special_attribute_present				)

								finally:
										if "custom" in CONFIG_MAPPING._extra_content:
												del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
												del TOKENIZER_MAPPING._extra_content[CustomConfig]

						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained(
								    """hf-internal-testing/test_dynamic_tokenizer_legacy"""       ,							trust_remote_code=a				)
								self.assertTrue(tokenizer.special_attribute_present				)
								if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizerFast"""				)

										# Test we can also load the slow version
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(
										    """hf-internal-testing/test_dynamic_tokenizer_legacy"""       ,							trust_remote_code=a       ,							use_fast=a				)
										self.assertTrue(tokenizer.special_attribute_present				)
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)
								else:
										self.assertEqual(tokenizer.__class__.__name__       ,							"""NewTokenizer"""				)

						def  A_				(			self				)					->      Optional[int]:


								'''simple docstring'''

								with self.assertRaisesRegex(
								    a       ,							"""bert-base is not a local folder and is not a valid model identifier"""				):
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""bert-base"""				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								with self.assertRaisesRegex(
								    a       ,							r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""				):
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a       ,							revision="""aaaaaa"""				)







						def  A_				(			self				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""				)
								with RequestCounter() as counter:
										_UpperCamelCase						    =	AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""				)
										self.assertEqual(counter.get_request_count       ,							0				)
										self.assertEqual(counter.head_request_count       ,							1				)
										self.assertEqual(counter.other_request_count       ,							0				)





import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import is_accelerate_available, is_torch_available, is_transformers_available, is_xformers_available
from . import BaseDiffusersCLICommand


def 							__A(lowerCAmelCase )		->				Optional[int]:

		"""simple docstring"""






		return EnvironmentCommand()







class        lowerCAmelCase__      (   __lowercase	):

						@staticmethod
						def  A_				(			a				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	parser.add_parser("""env"""				)
								download_parser.set_defaults(func=a				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	huggingface_hub.__version__

								_UpperCamelCase						    =	"""not installed"""
								_UpperCamelCase						    =	"""NA"""
								if is_torch_available():
										import torch

										_UpperCamelCase						    =	torch.__version__
										_UpperCamelCase						    =	torch.cuda.is_available()

								_UpperCamelCase						    =	"""not installed"""
								if is_transformers_available():
										import transformers

										_UpperCamelCase						    =	transformers.__version__

								_UpperCamelCase						    =	"""not installed"""
								if is_accelerate_available():
										import accelerate

										_UpperCamelCase						    =	accelerate.__version__

								_UpperCamelCase						    =	"""not installed"""
								if is_xformers_available():
										import xformers

										_UpperCamelCase						    =	xformers.__version__

								_UpperCamelCase						    =	{
								    """`diffusers` version""": version,
								    """Platform""": platform.platform(),
								    """Python version""": platform.python_version(),
								    """PyTorch version (GPU?)""": F'{pt_version} ({pt_cuda_available})',
								    """Huggingface_hub version""": hub_version,
								    """Transformers version""": transformers_version,
								    """Accelerate version""": accelerate_version,
								    """xFormers version""": xformers_version,
								    """Using GPU in script?""": """<fill in>""",
								    """Using distributed or parallel set-up in script?""": """<fill in>""",
								}

								print("""\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n"""				)
								print(self.format_dict(a				)				)

								return info







						@staticmethod
						def  A_				(			a				)					->      Dict:


								'''simple docstring'''

								return "\n".join([F'- {prop}: {val}' for prop, val in d.items()]				) + "\n"



