



import argparse
import glob
import logging
import os
import time
from argparse import Namespace

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from torch.utils.data import DataLoader, TensorDataset

from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes, glue_tasks_num_labels
from transformers import glue_processors as processors


_lowerCamelCase    :		Any		      =    logging.getLogger(__name__)


class 						lowercase    (							lowerCAmelCase_):
						'''simple docstring'''


						UpperCAmelCase			:		str		=		'sequence-classification'







						def __init__(  self    :				List[str]       ,	snake_case    :				Optional[Any]	):



											'''simple docstring'''


											if type(__lowerCAmelCase	) == dict:
																SCREAMING_SNAKE_CASE							:     List[str]						=				Namespace(**__lowerCAmelCase	)
											SCREAMING_SNAKE_CASE							:     Dict						=				glue_output_modes[hparams.task]
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				glue_tasks_num_labels[hparams.task]

											super().__init__(__lowerCAmelCase       ,	__lowerCAmelCase       ,	self.mode	)







						def       lowerCamelCase_				(  self    :				Optional[int]       ,	**snake_case    :				Tuple	):



											'''simple docstring'''


											return self.model(**__lowerCAmelCase	)







						def       lowerCamelCase_				(  self    :				Dict       ,	snake_case    :				List[Any]       ,	snake_case    :				List[str]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				{'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}

											if self.config.model_type not in ["distilbert", "bart"]:
																SCREAMING_SNAKE_CASE							:     Optional[Any]						=				batch[2] if self.config.model_type in ['bert', 'xlnet', 'albert'] else None

											SCREAMING_SNAKE_CASE							:     Tuple						=				self(**__lowerCAmelCase	)
											SCREAMING_SNAKE_CASE							:     Dict						=				outputs[0]

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				self.trainer.lr_schedulers[0]['scheduler']
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				{'loss': loss, 'rate': lr_scheduler.get_last_lr()[-1]}
											return {"loss": loss, "log": tensorboard_logs}







						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[int]						=				self.hparams
											SCREAMING_SNAKE_CASE							:     List[Any]						=				processors[args.task]()
											SCREAMING_SNAKE_CASE							:     str						=				processor.get_labels()

											for mode in ["train", "dev"]:
																SCREAMING_SNAKE_CASE							:     str						=				self._feature_file(__lowerCAmelCase	)
																if os.path.exists(__lowerCAmelCase	) and not args.overwrite_cache:
																					logger.info('Loading features from cached file %s'       ,	__lowerCAmelCase	)
																else:
																					logger.info('Creating features from dataset file at %s'       ,	args.data_dir	)
																					SCREAMING_SNAKE_CASE							:     Tuple						=				(
																					    processor.get_dev_examples(args.data_dir	)
																					    if mode == 'dev'
																					    else processor.get_train_examples(args.data_dir	)
																					)
																					SCREAMING_SNAKE_CASE							:     Optional[Any]						=				convert_examples_to_features(
																					    __lowerCAmelCase       ,	self.tokenizer       ,	max_length=args.max_seq_length       ,	label_list=self.labels       ,	output_mode=args.glue_output_mode       ,	)
																					logger.info('Saving features into cached file %s'       ,	__lowerCAmelCase	)
																					torch.save(__lowerCAmelCase       ,	__lowerCAmelCase	)







						def       lowerCamelCase_				(  self    :				int       ,	snake_case    :				str       ,	snake_case    :				int       ,	snake_case    :				bool = False	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Dict						=				'dev' if mode == 'test' else mode

											SCREAMING_SNAKE_CASE							:     str						=				self._feature_file(__lowerCAmelCase	)
											logger.info('Loading features from cached file %s'       ,	__lowerCAmelCase	)
											SCREAMING_SNAKE_CASE							:     Dict						=				torch.load(__lowerCAmelCase	)
											SCREAMING_SNAKE_CASE							:     int						=				torch.tensor([f.input_ids for f in features]       ,	dtype=torch.long	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				torch.tensor([f.attention_mask for f in features]       ,	dtype=torch.long	)
											SCREAMING_SNAKE_CASE							:     str						=				torch.tensor([f.token_type_ids for f in features]       ,	dtype=torch.long	)
											if self.hparams.glue_output_mode == "classification":
																SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				torch.tensor([f.label for f in features]       ,	dtype=torch.long	)
											elif self.hparams.glue_output_mode == "regression":
																SCREAMING_SNAKE_CASE							:     Tuple						=				torch.tensor([f.label for f in features]       ,	dtype=torch.float	)

											return DataLoader(
											    TensorDataset(__lowerCAmelCase       ,	__lowerCAmelCase       ,	__lowerCAmelCase       ,	__lowerCAmelCase	)       ,	batch_size=__lowerCAmelCase       ,	shuffle=__lowerCAmelCase       ,	)







						def       lowerCamelCase_				(  self    :				str       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Tuple	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				{'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}

											if self.config.model_type not in ["distilbert", "bart"]:
																SCREAMING_SNAKE_CASE							:     List[str]						=				batch[2] if self.config.model_type in ['bert', 'xlnet', 'albert'] else None

											SCREAMING_SNAKE_CASE							:     List[Any]						=				self(**__lowerCAmelCase	)
											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				outputs[:2]
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				logits.detach().cpu().numpy()
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				inputs['labels'].detach().cpu().numpy()

											return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}







						def       lowerCamelCase_				(  self    :				str       ,	snake_case    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[int]						=				torch.stack([x['val_loss'] for x in outputs]	).mean().detach().cpu().item()
											SCREAMING_SNAKE_CASE							:     str						=				np.concatenate([x['pred'] for x in outputs]       ,	axis=0	)

											if self.hparams.glue_output_mode == "classification":
																SCREAMING_SNAKE_CASE							:     Optional[Any]						=				np.argmax(__lowerCAmelCase       ,	axis=1	)
											elif self.hparams.glue_output_mode == "regression":
																SCREAMING_SNAKE_CASE							:     Tuple						=				np.squeeze(__lowerCAmelCase	)

											SCREAMING_SNAKE_CASE							:     Dict						=				np.concatenate([x['target'] for x in outputs]       ,	axis=0	)
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				[[] for _ in range(out_label_ids.shape[0]	)]
											SCREAMING_SNAKE_CASE							:     Any						=				[[] for _ in range(out_label_ids.shape[0]	)]

											SCREAMING_SNAKE_CASE							:     List[str]						=				{**{'val_loss': val_loss_mean}, **compute_metrics(self.hparams.task       ,	__lowerCAmelCase       ,	__lowerCAmelCase	)}

											SCREAMING_SNAKE_CASE							:     str						=				dict(results.items()	)
											SCREAMING_SNAKE_CASE							:     List[str]						=				results
											return ret, preds_list, out_label_list







						def       lowerCamelCase_				(  self    :				Union[str, Any]       ,	snake_case    :				list	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				self._eval_end(__lowerCAmelCase	)
											SCREAMING_SNAKE_CASE							:     List[str]						=				ret['log']
											return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}







						def       lowerCamelCase_				(  self    :				int       ,	snake_case    :				Union[str, Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     Optional[int]						=				self._eval_end(__lowerCAmelCase	)
											SCREAMING_SNAKE_CASE							:     int						=				ret['log']
											# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
											return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}







						@staticmethod
						def       lowerCamelCase_				(  snake_case    :				str       ,	snake_case    :				Dict	):



											'''simple docstring'''


											BaseTransformer.add_model_specific_args(__lowerCAmelCase       ,	__lowerCAmelCase	)
											parser.add_argument(
											    '--max_seq_length'       ,	default=128       ,	type=__lowerCAmelCase       ,	help=(
											        'The maximum total input sequence length after tokenization. Sequences longer '
											        'than this will be truncated, sequences shorter will be padded.'
											    )       ,	)

											parser.add_argument(
											    '--task'       ,	default=''       ,	type=__lowerCAmelCase       ,	required=__lowerCAmelCase       ,	help='The GLUE task to run'       ,	)
											parser.add_argument(
											    '--gpus'       ,	default=0       ,	type=__lowerCAmelCase       ,	help='The number of GPUs allocated for this, it is by default 0 meaning none'       ,	)

											parser.add_argument(
											    '--overwrite_cache'       ,	action='store_true'       ,	help='Overwrite the cached training and evaluation sets'	)

											return parser


def        __a			(   )     ->     Dict:
					SCREAMING_SNAKE_CASE							:     Optional[Any]						=				argparse.ArgumentParser()
					add_generic_args(__lowerCAmelCase      ,	os.getcwd()   )
					SCREAMING_SNAKE_CASE							:     Optional[int]						=				GLUETransformer.add_model_specific_args(__lowerCAmelCase      ,	os.getcwd()   )
					SCREAMING_SNAKE_CASE							:     List[str]						=				parser.parse_args()

					# If output_dir not provided, a folder will be generated in pwd
					if args.output_dir is None:
										SCREAMING_SNAKE_CASE							:     str						=				os.path.join(
										    './results'      ,	F'''{args.task}_{time.strftime('%Y%m%d_%H%M%S'   )}'''      ,	)
										os.makedirs(args.output_dir   )

					SCREAMING_SNAKE_CASE							:     int						=				GLUETransformer(__lowerCAmelCase   )
					SCREAMING_SNAKE_CASE							:     Any						=				generic_train(__lowerCAmelCase      ,	__lowerCAmelCase   )

					# Optionally, predict on dev set and write to output_dir
					if args.do_predict:
										SCREAMING_SNAKE_CASE							:     Tuple						=				sorted(glob.glob(os.path.join(args.output_dir      ,	'checkpoint-epoch=*.ckpt'   )      ,	recursive=__lowerCAmelCase   )   )
										SCREAMING_SNAKE_CASE							:     Optional[int]						=				model.load_from_checkpoint(checkpoints[-1]   )
										return trainer.test(__lowerCAmelCase   )


if __name__ == "__main__":
			main()
"""simple docstring"""


import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast


class 							a					(		unittest.TestCase ):


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      _UpperCAmelCase    			=   0


    @slow
    def   lowerCAmelCase_	(       self						:	List[str]       ):
      for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
        self.assertIsNotNone(__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
        self.assertGreater(len(__lowerCAmelCase       ) ,					0       )

      for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
        self.assertIsNotNone(__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					(GPTaTokenizer, GPTaTokenizerFast)       )
        self.assertGreater(len(__lowerCAmelCase       ) ,					0       )


    def   lowerCAmelCase_	(       self						:	Union[str, Any]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(RobertaTokenizer, RobertaTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					20       )


    def   lowerCAmelCase_	(       self						:	Any       ):
      _UpperCAmelCase    			=   AutoConfig.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )
      # Check that tokenizer_type ≠ model_type
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					config=__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""bert""" ,					use_fast=__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.json"""       )       )
        shutil.copy("""./tests/fixtures/merges.txt""" ,					os.path.join(__lowerCAmelCase ,					"""merges.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""gpt2""" ,					use_fast=__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Tuple       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""bert"""       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.json"""       )       )
        shutil.copy("""./tests/fixtures/merges.txt""" ,					os.path.join(__lowerCAmelCase ,					"""merges.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""gpt2"""       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      with pytest.raises(__lowerCAmelCase       ):
        AutoTokenizer.from_pretrained("""./""" ,					tokenizer_type="""xxx"""       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        _UpperCAmelCase    			=   tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""       )
        self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )

        if isinstance(__lowerCAmelCase ,					__lowerCAmelCase       ):
          self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,					__lowerCAmelCase       )
        else:
          self.assertEqual(tokenizer.do_lower_case ,					__lowerCAmelCase       )

        self.assertEqual(tokenizer.model_max_length ,					512       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[str]       ):
      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        with self.assertRaisesRegex(
            __lowerCAmelCase ,					"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier""" ,					):
          _UpperCAmelCase    			=   tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""       )


    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      # tests: https://github.com/huggingface/transformers/pull/13251
      # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
      # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
      _UpperCAmelCase    			=   TOKENIZER_MAPPING.values()
      _UpperCAmelCase    			=   []

      for slow_tok, fast_tok in tokenizers:
        if slow_tok is not None:
          tokenizer_names.append(slow_tok.__name__       )

        if fast_tok is not None:
          tokenizer_names.append(fast_tok.__name__       )

      for tokenizer_name in tokenizer_names:
        # must find the right class
        tokenizer_class_from_name(__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased""" ,					use_fast=__lowerCAmelCase       ) ,					__lowerCAmelCase       )
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""       ) ,					__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	int       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""distilbert-base-uncased""" ,					do_lower_case=__lowerCAmelCase       )
      _UpperCAmelCase    			=   """Hello, world. How are you?"""
      _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
      self.assertEqual("""[UNK]""" ,					tokens[0]       )

      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""microsoft/mpnet-base""" ,					do_lower_case=__lowerCAmelCase       )
      _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
      self.assertEqual("""[UNK]""" ,					tokens[0]       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""       )
      self.assertEqual(type(__lowerCAmelCase       ) ,					__lowerCAmelCase       )
      self.assertEqual(tokenizer.model_max_length ,					512       )
      self.assertEqual(tokenizer.vocab_size ,					3_0000       )
      self.assertEqual(tokenizer.unk_token ,					"""[UNK]"""       )
      self.assertEqual(tokenizer.padding_side ,					"""right"""       )
      self.assertEqual(tokenizer.truncation_side ,					"""right"""       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )

      self.assertIsInstance(__lowerCAmelCase ,					tokenizer.__class__       )
      self.assertEqual(tokenizera.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""ctrl"""       )
      # There is no fast CTRL so this always gives us a slow tokenizer.
      self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	str       ):
      # Check we can load the tokenizer config of an online model.
      _UpperCAmelCase    			=   get_tokenizer_config("""bert-base-cased"""       )
      _UpperCAmelCase    			=   config.pop("""_commit_hash""" ,					__lowerCAmelCase       )
      # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
      self.assertEqual(__lowerCAmelCase ,					{"""do_lower_case""": False}       )

      # This model does not have a tokenizer_config so we get back an empty dict.
      _UpperCAmelCase    			=   get_tokenizer_config(__lowerCAmelCase       )
      self.assertDictEqual(__lowerCAmelCase ,					{}       )

      # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   get_tokenizer_config(__lowerCAmelCase       )

      # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
      self.assertEqual(config["""tokenizer_class"""] ,					"""BertTokenizer"""       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )

        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(__lowerCAmelCase       ):
          AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )

        _UpperCAmelCase    			=   CustomTokenizer.from_pretrained(__lowerCAmelCase       )
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )

        # Can register in two steps
        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, None)       )
        AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, CustomTokenizerFast)       )

        del TOKENIZER_MAPPING._extra_content[CustomConfig]
        # Can register in one step
        AutoTokenizer.register(
            __lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, CustomTokenizerFast)       )

        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(__lowerCAmelCase       ):
          AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )

        # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
        # and that model does not have a tokenizer.json
        with tempfile.TemporaryDirectory() as tmp_dir:
          _UpperCAmelCase    			=   BertTokenizerFast.from_pretrained(__lowerCAmelCase       )
          bert_tokenizer.save_pretrained(__lowerCAmelCase       )
          _UpperCAmelCase    			=   CustomTokenizerFast.from_pretrained(__lowerCAmelCase       )

        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      # If remote code is not set, we will time out when asking whether to load the model.
      with self.assertRaises(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       )
      # If remote code is disabled, we can't load this config.
      with self.assertRaises(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )

      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(tokenizer.special_attribute_present       )
      # Test tokenizer can be reloaded.
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(reloaded_tokenizer.special_attribute_present       )

      if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )

        # Test we can also load the slow version
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertTrue(tokenizer.special_attribute_present       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        # Test tokenizer can be reloaded.
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )
          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertTrue(reloaded_tokenizer.special_attribute_present       )
      else:
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[str]       ):


      class 							a					(		lowerCAmelCase_ ):
          _snake_case  :	Tuple  				=				False



      class 							a					(		lowerCAmelCase_ ):
          _snake_case  :	Optional[int]  				=				NewTokenizer
          _snake_case  :	Any  				=				False

      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )
        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        # If remote code is not set, the default is to use local
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertFalse(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer""" ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertFalse(tokenizer.special_attribute_present       )

        # If remote code is disabled, we load the local one.
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertFalse(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertFalse(tokenizer.special_attribute_present       )

        # If remote is enabled, we load from the Hub
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertTrue(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertTrue(tokenizer.special_attribute_present       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    def   lowerCAmelCase_	(       self						:	Dict       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
          """hf-internal-testing/test_dynamic_tokenizer_legacy""" ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(tokenizer.special_attribute_present       )
      if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )

        # Test we can also load the slow version
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer_legacy""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertTrue(tokenizer.special_attribute_present       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
      else:
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      with self.assertRaisesRegex(
          __lowerCAmelCase ,					"""bert-base is not a local folder and is not a valid model identifier"""       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""bert-base"""       )


    def   lowerCAmelCase_	(       self						:	int       ):
      with self.assertRaisesRegex(
          __lowerCAmelCase ,					R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					revision="""aaaaaa"""       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      # Make sure we have cached the tokenizer.
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
      with RequestCounter() as counter:
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
        self.assertEqual(counter.get_request_count ,					0       )
        self.assertEqual(counter.head_request_count ,					1       )
        self.assertEqual(counter.other_request_count ,					0       )
