



"""simple docstring"""





import argparse
import dataclasses
import json
import logging
import os
import shutil
from typing import List, Optional

import datasets
from accelerate import Accelerator
from datasets import load_dataset
from finetuning import finetune
from tqdm.auto import tqdm

import transformers
from transformers import AutoConfig, set_seed
from transformers.trainer_utils import IntervalStrategy


lowerCamelCase  			=		logging.getLogger(__name__)

lowerCamelCase  			=		"""pytorch_model.bin"""
@dataclasses.dataclass
class  lowercase__  :







       '''simple docstring'''


       UpperCamelCase		=	dataclasses.field(
           metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models.'''} )
       UpperCamelCase		=	dataclasses.field(
           default=SCREAMING_SNAKE_CASE							,   metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co.'''}							,   )



@dataclasses.dataclass
class  lowercase__  :







       '''simple docstring'''


       UpperCamelCase		=	dataclasses.field(metadata={'''help''': '''A csv or a json file containing the training data.'''} )
       UpperCamelCase		=	dataclasses.field(metadata={'''help''': '''A csv or a json file containing the data to predict on.'''} )
       UpperCamelCase		=	dataclasses.field(
           default=SCREAMING_SNAKE_CASE							,   metadata={'''help''': '''A csv or a json file containing the validation data.'''} )
       UpperCamelCase		=	dataclasses.field(
           default=SCREAMING_SNAKE_CASE							,   metadata={'''help''': '''The name of the task to train on.'''}							,   )
       UpperCamelCase		=	dataclasses.field(
           default=SCREAMING_SNAKE_CASE							,   metadata={'''help''': '''The list of labels for the task.'''} )



@dataclasses.dataclass
class  lowercase__  :







       '''simple docstring'''


       UpperCamelCase		=	dataclasses.field(
           metadata={'''help''': '''The output directory where the model predictions and checkpoints will be written.'''} )
       UpperCamelCase		=	dataclasses.field(
           default='''accuracy'''							,   metadata={'''help''': '''The evaluation metric used for the task.'''} )
       UpperCamelCase		=	dataclasses.field(
           default='''no'''							,   metadata={
               '''help''': '''The evaluation strategy to adopt during training. Possible values are: ["no", "step", "epoch]'''
           }							,   )
       UpperCamelCase		=	dataclasses.field(
           default=10							,   metadata={'''help''': '''Number of evaluation calls with no improvement after which training will be stopped.'''}							,   )
       UpperCamelCase		=	dataclasses.field(
           default=0.0							,   metadata={
               '''help''': '''How much the specified evaluation metric must improve to satisfy early stopping conditions.'''
           }							,   )
       UpperCamelCase		=	dataclasses.field(
           default=SCREAMING_SNAKE_CASE							,   metadata={'''help''': '''Whether to filter the pseudo-labeled data based on the confidence score.'''}							,   )
       UpperCamelCase		=	dataclasses.field(
           default=SCREAMING_SNAKE_CASE							,   metadata={'''help''': '''Whether to filter the pseudo-labeled data based on the validation performance.'''}							,   )
       UpperCamelCase		=	dataclasses.field(
           default=SCREAMING_SNAKE_CASE							,   metadata={'''help''': '''Whether to fine-tune on labeled data after pseudo training.'''}							,   )
       UpperCamelCase		=	dataclasses.field(
           default=0.0							,   metadata={'''help''': '''Confidence threshold for pseudo-labeled data filtering.'''}							,   )
       UpperCamelCase		=	dataclasses.field(
           default=1_00							,   metadata={'''help''': '''Number of evaluation calls with no improvement after which training will be stopped.'''}							,   )
       UpperCamelCase		=	dataclasses.field(
           default=SCREAMING_SNAKE_CASE							,   metadata={'''help''': '''Random seed for initialization.'''}							,   )



def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ ):
      UpperCAmelCase_     					=				datasets.concatenate_datasets([infer_input, infer_output]      ,     axis=1 )

      if args.do_filter_by_confidence:
            UpperCAmelCase_     					=				dataset.filter(lambda lowerCAmelCase__ : example["probability"] > args.confidence_threshold )

      if args.do_filter_by_val_performance:
            assert eval_result >= 0.0 and eval_result <= 1.0
            UpperCAmelCase_     					=				int(eval_result * len(lowerCAmelCase__ ) )
            print(lowerCAmelCase__ )
            UpperCAmelCase_     					=				dataset.sort("probability"      ,     reverse=lowerCAmelCase__ )
            UpperCAmelCase_     					=				dataset.select(range(lowerCAmelCase__ ) )

      UpperCAmelCase_     					=				dataset.remove_columns(["label", "probability"] )
      UpperCAmelCase_     					=				dataset.rename_column("prediction"      ,     "label" )
      UpperCAmelCase_     					=				dataset.map(lambda lowerCAmelCase__ : {"label": idalabel[example["label"]]} )
      UpperCAmelCase_     					=				dataset.shuffle(seed=args.seed )

      UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     f"""train_pseudo.{args.data_file_extension}""" )
      if args.data_file_extension == "csv":
            dataset.to_csv(lowerCAmelCase__      ,     index=lowerCAmelCase__ )
      else:
            dataset.to_json(lowerCAmelCase__ )


def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     **lowerCAmelCase__ ):
      UpperCAmelCase_     					=				Accelerator()
      # Make one log on every process with the configuration for debugging.
      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"      ,     datefmt="%m/%d/%Y %H:%M:%S"      ,     level=logging.INFO      ,     )
      logger.info(accelerator.state )

      # Setup logging, we only want one process per machine to log things on the
      # screen. accelerator.is_local_main_process is only True for one process per
      # machine.
      logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR )

      if accelerator.is_local_main_process:
            datasets.utils.logging.set_verbosity_warning()
            transformers.utils.logging.set_verbosity_info()
      else:
            datasets.utils.logging.set_verbosity_error()
            transformers.utils.logging.set_verbosity_error()

      UpperCAmelCase_     					=				STModelArguments(model_name_or_path=lowerCAmelCase__ )
      UpperCAmelCase_     					=				STDataArguments(train_file=lowerCAmelCase__      ,     infer_file=lowerCAmelCase__ )
      UpperCAmelCase_     					=				STTrainingArguments(output_dir=lowerCAmelCase__ )
      UpperCAmelCase_     					=				argparse.Namespace()

      for arg_class in (model_args, data_args, training_args):
            for key, value in vars(lowerCAmelCase__ ).items():
                  setattr(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ )

      for key, value in kwargs.items():
            if hasattr(lowerCAmelCase__      ,     lowerCAmelCase__ ):
                  setattr(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ )

    # Sanity checks
      UpperCAmelCase_     					=				{}
      UpperCAmelCase_     					=				None

      # You need to provide the training data and the data to predict on
      assert args.train_file is not None
      assert args.infer_file is not None
      UpperCAmelCase_     					=				args.train_file
      UpperCAmelCase_     					=				args.infer_file

      if args.evaluation_strategy != IntervalStrategy.NO.value:
            assert args.eval_file is not None
            UpperCAmelCase_     					=				args.eval_file

      for key in data_files:
            UpperCAmelCase_     					=				data_files[key].split("." )[-1]
            assert extension in ["csv", "json"], f"""`{key}_file` should be a csv or a json file."""
            if args.data_file_extension is None:
                  UpperCAmelCase_     					=				extension
            else:
                  assert extension == args.data_file_extension, f"""`{key}_file` should be a {args.data_file_extension} file`."""

      assert (
          args.eval_metric in datasets.list_metrics()
      ), f"""{args.eval_metric} not in the list of supported metrics {datasets.list_metrics()}."""

      # If passed along, set the training seed now.
      if args.seed is not None:
            set_seed(args.seed )

      logger.info("Creating the initial data directory for self-training..." )
      UpperCAmelCase_     					=				f"""{args.output_dir}/self-train_iter-{{}}""".format
      UpperCAmelCase_     					=				data_dir_format(0 )

      if accelerator.is_main_process:
            if args.output_dir is not None:
                  os.makedirs(args.output_dir      ,     exist_ok=lowerCAmelCase__ )
                  os.makedirs(lowerCAmelCase__      ,     exist_ok=lowerCAmelCase__ )
      accelerator.wait_for_everyone()

      UpperCAmelCase_     					=				None
      UpperCAmelCase_     					=				None
      UpperCAmelCase_     					=				0
      UpperCAmelCase_     					=				False
      # Show the progress bar
      UpperCAmelCase_     					=				tqdm(range(args.max_selftrain_iterations )      ,     disable=not accelerator.is_local_main_process )

      # Self-train
      for iteration in range(0      ,     int(args.max_selftrain_iterations ) ):
            UpperCAmelCase_     					=				data_dir_format(lowerCAmelCase__ )
            assert os.path.exists(lowerCAmelCase__ )

            # Stage 1: initial fine-tuning for iteration = 0 or pseudo-training for
            # iteration > 0
            UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     "stage-1" )
            UpperCAmelCase_     					=				{
                "accelerator": accelerator,
                "model_name_or_path": args.model_name_or_path,
                "cache_dir": args.cache_dir,
                "do_train": True,
                "train_file": data_files["train"] if iteration == 0 else data_files["train_pseudo"],
                "do_eval": True if args.eval_file is not None else False,
                "eval_file": data_files["eval"],
                "do_predict": True,
                "infer_file": data_files["infer"],
                "task_name": args.task_name,
                "label_list": args.label_list,
                "output_dir": current_output_dir,
                "eval_metric": args.eval_metric,
                "evaluation_strategy": args.evaluation_strategy,
                "early_stopping_patience": args.early_stopping_patience,
                "early_stopping_threshold": args.early_stopping_threshold,
                "seed": args.seed,
            }
            # Add additional training arguments
            for key, value in kwargs.items():
                  if key not in arguments_dict and not hasattr(lowerCAmelCase__      ,     lowerCAmelCase__ ):
                        arguments_dict.update({key: value} )

            UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     "best-checkpoint"      ,     lowerCAmelCase__ )
            if os.path.exists(lowerCAmelCase__ ):
                  logger.info(
                      "Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 1."      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     )
            else:
                  logger.info("***** Running self-training: iteration: %d, stage: 1 *****"      ,     lowerCAmelCase__ )
                  finetune(**lowerCAmelCase__ )
                  accelerator.wait_for_everyone()
                  assert os.path.exists(lowerCAmelCase__ )
                  logger.info("Self-training job completed: iteration: %d, stage: 1."      ,     lowerCAmelCase__ )

            if iteration > 0 and args.finetune_on_labeled_data:
                  # Stage 2 (optional): fine-tuning on the original labeled data
                  UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     "best-checkpoint" )
                  UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     "stage-2" )
                  # Update arguments_dict
                  UpperCAmelCase_     					=				model_path
                  UpperCAmelCase_     					=				data_files["train"]
                  UpperCAmelCase_     					=				current_output_dir

                  UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     "best-checkpoint"      ,     lowerCAmelCase__ )
                  if os.path.exists(lowerCAmelCase__ ):
                        logger.info(
                            "Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 2."      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     )
                  else:
                        logger.info("***** Running self-training: iteration: %d, stage: 2 *****"      ,     lowerCAmelCase__ )
                        finetune(**lowerCAmelCase__ )
                        accelerator.wait_for_everyone()
                        assert os.path.exists(lowerCAmelCase__ )
                        logger.info("Self-training job completed: iteration: %d, stage: 2."      ,     lowerCAmelCase__ )

            UpperCAmelCase_     					=				iteration
            UpperCAmelCase_     					=				data_dir_format(iteration + 1 )

            UpperCAmelCase_     					=				AutoConfig.from_pretrained(os.path.join(lowerCAmelCase__      ,     "best-checkpoint" ) )
            UpperCAmelCase_     					=				config.idalabel
            UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     "eval_results_best-checkpoint.json" )
            UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     "test_results_best-checkpoint.json" )
            assert os.path.exists(lowerCAmelCase__ )

            with open(lowerCAmelCase__      ,     "r" ) as f:
                  UpperCAmelCase_     					=				float(json.load(lowerCAmelCase__ )[args.eval_metric] )
            UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     "infer_output_best-checkpoint.csv" )
            assert os.path.exists(lowerCAmelCase__ )
            # Loading the dataset from local csv or json files.
            UpperCAmelCase_     					=				load_dataset(args.data_file_extension      ,     data_files={"data": data_files["infer"]} )["data"]
            UpperCAmelCase_     					=				load_dataset("csv"      ,     data_files={"data": infer_output_file} )["data"]

            if accelerator.is_main_process:
                  os.makedirs(lowerCAmelCase__      ,     exist_ok=lowerCAmelCase__ )
                  shutil.copy(lowerCAmelCase__      ,     os.path.join(lowerCAmelCase__      ,     f"""eval_results_iter-{iteration}.json""" ) )
                  if os.path.exists(lowerCAmelCase__ ):
                        shutil.copy(lowerCAmelCase__      ,     os.path.join(lowerCAmelCase__      ,     f"""test_results_iter-{iteration}.json""" ) )
                  create_pseudo_labeled_data(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ )
            accelerator.wait_for_everyone()

            UpperCAmelCase_     					=				os.path.join(lowerCAmelCase__      ,     f"""train_pseudo.{args.data_file_extension}""" )

            if args.evaluation_strategy != IntervalStrategy.NO.value:
                  UpperCAmelCase_     					=				eval_result

                  if best_iteration is None:
                        UpperCAmelCase_     					=				new_iteration
                        UpperCAmelCase_     					=				new_eval_result
                  else:
                        if new_eval_result - best_eval_result > args.early_stopping_threshold:
                              UpperCAmelCase_     					=				new_iteration
                              UpperCAmelCase_     					=				new_eval_result
                              UpperCAmelCase_     					=				0
                        else:
                              if new_eval_result == best_eval_result:
                                    UpperCAmelCase_     					=				new_iteration
                                    UpperCAmelCase_     					=				new_eval_result
                              early_stopping_patience_counter += 1

                        if early_stopping_patience_counter >= args.early_stopping_patience:
                              UpperCAmelCase_     					=				True

            progress_bar.update(1 )

            if should_training_stop:
                  break

      if best_iteration is not None:
            # Save the best iteration
            logger.info("Best iteration: %d"      ,     lowerCAmelCase__ )
            logger.info("Best evaluation result: %s = %f"      ,     args.eval_metric      ,     lowerCAmelCase__ )
            accelerator.wait_for_everyone()
            if accelerator.is_main_process:
                  shutil.copy(
                      os.path.join(lowerCAmelCase__      ,     f"""eval_results_iter-{iteration}.json""" )      ,     os.path.join(lowerCAmelCase__      ,     "eval_results_best-iteration.json" )      ,     )
      else:
            # Assume that the last iteration is the best
            logger.info("Best iteration: %d"      ,     args.max_selftrain_iterations - 1 )
            logger.info("Best evaluation result: %s = %f"      ,     args.eval_metric      ,     lowerCAmelCase__ )
            accelerator.wait_for_everyone()
            if accelerator.is_main_process:
                  shutil.copy(
                      os.path.join(lowerCAmelCase__      ,     f"""eval_results_iter-{args.max_selftrain_iterations - 1}.json""" )      ,     os.path.join(lowerCAmelCase__      ,     "eval_results_best-iteration.json" )      ,     )












import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast





class 					A__     (     unittest.TestCase ):

     '''simple docstring'''






     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           UpperCamelCase					       =  0







     @slow
     def   _SCREAMING_SNAKE_CASE     ( self      :    Any ):






           """simple docstring"""




           for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
                 self.assertIsNotNone(_SCREAMING_SNAKE_CASE )
                 self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						(BertTokenizer, BertTokenizerFast) )
                 self.assertGreater(len(_SCREAMING_SNAKE_CASE )   ,						0 )

           for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
                 self.assertIsNotNone(_SCREAMING_SNAKE_CASE )
                 self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						(GPTaTokenizer, GPTaTokenizerFast) )
                 self.assertGreater(len(_SCREAMING_SNAKE_CASE )   ,						0 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Union[str, Any] ):






           """simple docstring"""




           UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
           self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						(BertTokenizer, BertTokenizerFast) )
           self.assertEqual(tokenizer.vocab_size   ,						12 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
           self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						(RobertaTokenizer, RobertaTokenizerFast) )
           self.assertEqual(tokenizer.vocab_size   ,						20 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           UpperCamelCase					       =  AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE )
           self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           # Check that tokenizer_type ≠ model_type
           UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						config=_SCREAMING_SNAKE_CASE )
           self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						(BertTokenizer, BertTokenizerFast) )
           self.assertEqual(tokenizer.vocab_size   ,						12 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple ):






           """simple docstring"""




           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('./tests/fixtures/vocab.txt'   ,						os.path.join(_SCREAMING_SNAKE_CASE   ,						'vocab.txt' ) )

                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						tokenizer_type='bert'   ,						use_fast=_SCREAMING_SNAKE_CASE )
                 self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('./tests/fixtures/vocab.json'   ,						os.path.join(_SCREAMING_SNAKE_CASE   ,						'vocab.json' ) )
                 shutil.copy('./tests/fixtures/merges.txt'   ,						os.path.join(_SCREAMING_SNAKE_CASE   ,						'merges.txt' ) )

                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						tokenizer_type='gpt2'   ,						use_fast=_SCREAMING_SNAKE_CASE )
                 self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     @require_tokenizers
     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('./tests/fixtures/vocab.txt'   ,						os.path.join(_SCREAMING_SNAKE_CASE   ,						'vocab.txt' ) )

                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						tokenizer_type='bert' )
                 self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('./tests/fixtures/vocab.json'   ,						os.path.join(_SCREAMING_SNAKE_CASE   ,						'vocab.json' ) )
                 shutil.copy('./tests/fixtures/merges.txt'   ,						os.path.join(_SCREAMING_SNAKE_CASE   ,						'merges.txt' ) )

                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						tokenizer_type='gpt2' )
                 self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           with pytest.raises(_SCREAMING_SNAKE_CASE ):
                 AutoTokenizer.from_pretrained('./'   ,						tokenizer_type='xxx' )







     @require_tokenizers
     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple ):






           """simple docstring"""




           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 UpperCamelCase					       =  tokenizer_class.from_pretrained('wietsedv/bert-base-dutch-cased' )
                 self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						(BertTokenizer, BertTokenizerFast) )

                 if isinstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE ):
                       self.assertEqual(tokenizer.basic_tokenizer.do_lower_case   ,						_SCREAMING_SNAKE_CASE )
                 else:
                       self.assertEqual(tokenizer.do_lower_case   ,						_SCREAMING_SNAKE_CASE )

                 self.assertEqual(tokenizer.model_max_length   ,						512 )







     @require_tokenizers
     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 with self.assertRaisesRegex(
                     _SCREAMING_SNAKE_CASE   ,						'julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'   ,						):
                       UpperCamelCase					       =  tokenizer_class.from_pretrained('julien-c/herlolip-not-exists' )







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           UpperCamelCase					       =  TOKENIZER_MAPPING.values()
           UpperCamelCase					       =  []

           for slow_tok, fast_tok in tokenizers:
                 if slow_tok is not None:
                       tokenizer_names.append(slow_tok.__name__ )

                 if fast_tok is not None:
                       tokenizer_names.append(fast_tok.__name__ )

           for tokenizer_name in tokenizer_names:
                 # must find the right class
                 tokenizer_class_from_name(_SCREAMING_SNAKE_CASE )







     @require_tokenizers
     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'   ,						use_fast=_SCREAMING_SNAKE_CASE )   ,						_SCREAMING_SNAKE_CASE )
           self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased' )   ,						_SCREAMING_SNAKE_CASE )







     @require_tokenizers
     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           UpperCamelCase					       =  AutoTokenizer.from_pretrained('distilbert-base-uncased'   ,						do_lower_case=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  'Hello, world. How are you?'
           UpperCamelCase					       =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE )
           self.assertEqual('[UNK]'   ,						tokens[0] )

           UpperCamelCase					       =  AutoTokenizer.from_pretrained('microsoft/mpnet-base'   ,						do_lower_case=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE )
           self.assertEqual('[UNK]'   ,						tokens[0] )







     @require_tokenizers
     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  AutoTokenizer.from_pretrained('robot-test/dummy-tokenizer-fast-with-model-config' )
           self.assertEqual(type(_SCREAMING_SNAKE_CASE )   ,						_SCREAMING_SNAKE_CASE )
           self.assertEqual(tokenizer.model_max_length   ,						512 )
           self.assertEqual(tokenizer.vocab_size   ,						3_0000 )
           self.assertEqual(tokenizer.unk_token   ,						'[UNK]' )
           self.assertEqual(tokenizer.padding_side   ,						'right' )
           self.assertEqual(tokenizer.truncation_side   ,						'right' )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
           self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						(BertTokenizer, BertTokenizerFast) )
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE )
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )

           self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						tokenizer.__class__ )
           self.assertEqual(tokenizera.vocab_size   ,						12 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           UpperCamelCase					       =  AutoTokenizer.from_pretrained('ctrl' )
           # There is no fast CTRL so this always gives us a slow tokenizer.
           self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           UpperCamelCase					       =  get_tokenizer_config('bert-base-cased' )
           UpperCamelCase					       =  config.pop('_commit_hash'   ,						_SCREAMING_SNAKE_CASE )
           # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
           self.assertEqual(_SCREAMING_SNAKE_CASE   ,						{'do_lower_case': False} )

           # This model does not have a tokenizer_config so we get back an empty dict.
           UpperCamelCase					       =  get_tokenizer_config(_SCREAMING_SNAKE_CASE )
           self.assertDictEqual(_SCREAMING_SNAKE_CASE   ,						{} )

           # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
           UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE )
                 UpperCamelCase					       =  get_tokenizer_config(_SCREAMING_SNAKE_CASE )

           # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
           self.assertEqual(config['tokenizer_class']   ,						'BertTokenizer' )







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           try:
                 AutoConfig.register('custom'   ,						_SCREAMING_SNAKE_CASE )

                 AutoTokenizer.register(_SCREAMING_SNAKE_CASE   ,						slow_tokenizer_class=_SCREAMING_SNAKE_CASE )
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(_SCREAMING_SNAKE_CASE ):
                       AutoTokenizer.register(_SCREAMING_SNAKE_CASE   ,						slow_tokenizer_class=_SCREAMING_SNAKE_CASE )

                 UpperCamelCase					       =  CustomTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
                       self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]







     @require_tokenizers
     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           try:
                 AutoConfig.register('custom'   ,						_SCREAMING_SNAKE_CASE )

                 # Can register in two steps
                 AutoTokenizer.register(_SCREAMING_SNAKE_CASE   ,						slow_tokenizer_class=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   ,						(CustomTokenizer, None) )
                 AutoTokenizer.register(_SCREAMING_SNAKE_CASE   ,						fast_tokenizer_class=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   ,						(CustomTokenizer, CustomTokenizerFast) )

                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
                 # Can register in one step
                 AutoTokenizer.register(
                     _SCREAMING_SNAKE_CASE   ,						slow_tokenizer_class=_SCREAMING_SNAKE_CASE   ,						fast_tokenizer_class=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   ,						(CustomTokenizer, CustomTokenizerFast) )

                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(_SCREAMING_SNAKE_CASE ):
                       AutoTokenizer.register(_SCREAMING_SNAKE_CASE   ,						fast_tokenizer_class=_SCREAMING_SNAKE_CASE )

                 # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
                 # and that model does not have a tokenizer.json
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       UpperCamelCase					       =  BertTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE )
                       bert_tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  CustomTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE )

                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE )
                       self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						use_fast=_SCREAMING_SNAKE_CASE )
                       self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           with self.assertRaises(_SCREAMING_SNAKE_CASE ):
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer' )
           # If remote code is disabled, we can't load this config.
           with self.assertRaises(_SCREAMING_SNAKE_CASE ):
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(
                     'hf-internal-testing/test_dynamic_tokenizer'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE )
           self.assertTrue(tokenizer.special_attribute_present )
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE )
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						trust_remote_code=_SCREAMING_SNAKE_CASE )
           self.assertTrue(reloaded_tokenizer.special_attribute_present )

           if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizerFast' )
                 self.assertEqual(reloaded_tokenizer.__class__.__name__   ,						'NewTokenizerFast' )

                 # Test we can also load the slow version
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(
                     'hf-internal-testing/test_dynamic_tokenizer'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE   ,						use_fast=_SCREAMING_SNAKE_CASE )
                 self.assertTrue(tokenizer.special_attribute_present )
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizer' )
                 # Test tokenizer can be reloaded.
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						trust_remote_code=_SCREAMING_SNAKE_CASE   ,						use_fast=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(reloaded_tokenizer.__class__.__name__   ,						'NewTokenizer' )
                 self.assertTrue(reloaded_tokenizer.special_attribute_present )
           else:
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizer' )
                 self.assertEqual(reloaded_tokenizer.__class__.__name__   ,						'NewTokenizer' )







     @require_tokenizers
     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           class 					A__     (     __snake_case ):

                '''simple docstring'''






                snake_case__					=					False





           class 					A__     (     __snake_case ):

                '''simple docstring'''






                snake_case__					=					NewTokenizer
                snake_case__					=					False

           try:
                 AutoConfig.register('custom'   ,						_SCREAMING_SNAKE_CASE )
                 AutoTokenizer.register(_SCREAMING_SNAKE_CASE   ,						slow_tokenizer_class=_SCREAMING_SNAKE_CASE )
                 AutoTokenizer.register(_SCREAMING_SNAKE_CASE   ,						fast_tokenizer_class=_SCREAMING_SNAKE_CASE )
                 # If remote code is not set, the default is to use local
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer' )
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizerFast' )
                 self.assertFalse(tokenizer.special_attribute_present )
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'   ,						use_fast=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizer' )
                 self.assertFalse(tokenizer.special_attribute_present )

                 # If remote code is disabled, we load the local one.
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(
                     'hf-internal-testing/test_dynamic_tokenizer'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizerFast' )
                 self.assertFalse(tokenizer.special_attribute_present )
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(
                     'hf-internal-testing/test_dynamic_tokenizer'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE   ,						use_fast=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizer' )
                 self.assertFalse(tokenizer.special_attribute_present )

                 # If remote is enabled, we load from the Hub
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(
                     'hf-internal-testing/test_dynamic_tokenizer'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizerFast' )
                 self.assertTrue(tokenizer.special_attribute_present )
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(
                     'hf-internal-testing/test_dynamic_tokenizer'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE   ,						use_fast=_SCREAMING_SNAKE_CASE )
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizer' )
                 self.assertTrue(tokenizer.special_attribute_present )

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  AutoTokenizer.from_pretrained(
               'hf-internal-testing/test_dynamic_tokenizer_legacy'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE )
           self.assertTrue(tokenizer.special_attribute_present )
           if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizerFast' )

                 # Test we can also load the slow version
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(
                     'hf-internal-testing/test_dynamic_tokenizer_legacy'   ,						trust_remote_code=_SCREAMING_SNAKE_CASE   ,						use_fast=_SCREAMING_SNAKE_CASE )
                 self.assertTrue(tokenizer.special_attribute_present )
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizer' )
           else:
                 self.assertEqual(tokenizer.__class__.__name__   ,						'NewTokenizer' )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple ):






           """simple docstring"""




           with self.assertRaisesRegex(
               _SCREAMING_SNAKE_CASE   ,						'bert-base is not a local folder and is not a valid model identifier' ):
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained('bert-base' )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           with self.assertRaisesRegex(
               _SCREAMING_SNAKE_CASE   ,						r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)' ):
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   ,						revision='aaaaaa' )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           UpperCamelCase					       =  AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert' )
           with RequestCounter() as counter:
                 UpperCamelCase					       =  AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert' )
                 self.assertEqual(counter.get_request_count   ,						0 )
                 self.assertEqual(counter.head_request_count   ,						1 )
                 self.assertEqual(counter.other_request_count   ,						0 )



