

'''simple docstring'''






import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


snake_case_       =       1_6
snake_case_       =       3_2


def 					__lowerCamelCase			(      SCREAMING_SNAKE_CASE_     :	Dict    )							->   List[str]:
      """simple docstring"""

      return int(x / 2**2_0    )
class 	SCREAMING_SNAKE_CASE__			:




    def __enter__(					self			):






          """simple docstring"""






          gc.collect()
          torch.cuda.empty_cache()
          torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
          SCREAMING_SNAKE_CASE_      :			str     =       torch.cuda.memory_allocated()
          SCREAMING_SNAKE_CASE_      :			str     =       psutil.Process()

          SCREAMING_SNAKE_CASE_      :			List[Any]     =       self.cpu_mem_used()
          SCREAMING_SNAKE_CASE_      :			Any     =       True
          SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       threading.Thread(target=self.peak_monitor_func			)
          SCREAMING_SNAKE_CASE_      :			Dict     =       True
          peak_monitor_thread.start()
          return self




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          return self.process.memory_info().rss




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			str     =       -1

          while True:
                SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       max(self.cpu_mem_used()				,     self.cpu_peak			)

                # can't sleep or will not catch the peak right (this comment is here on purpose)
                # time.sleep(0.001) # 1msec

                if not self.peak_monitoring:
                      break







    def __exit__(					self				,     *lowercase__			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			Tuple     =       False

          gc.collect()
          torch.cuda.empty_cache()
          SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       torch.cuda.memory_allocated()
          SCREAMING_SNAKE_CASE_      :			Optional[int]     =       torch.cuda.max_memory_allocated()
          SCREAMING_SNAKE_CASE_      :			int     =       bamb(self.end - self.begin			)
          SCREAMING_SNAKE_CASE_      :			Tuple     =       bamb(self.peak - self.begin			)

          SCREAMING_SNAKE_CASE_      :			Any     =       self.cpu_mem_used()
          SCREAMING_SNAKE_CASE_      :			Tuple     =       bamb(self.cpu_end - self.cpu_begin			)
          SCREAMING_SNAKE_CASE_      :			str     =       bamb(self.cpu_peak - self.cpu_begin			)
          # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
      from accelerate.test_utils.training import mocked_dataloaders

      snake_case_       =       mocked_dataloaders  # noqa: F811


def 					__lowerCamelCase			(      SCREAMING_SNAKE_CASE_     :	int  ,  SCREAMING_SNAKE_CASE_     :	Optional[int]    )							->   Optional[Any]:
      """simple docstring"""

      if os.environ.get("TESTING_MOCKED_DATALOADERS"  ,  _lowercase    ) == "1":
            SCREAMING_SNAKE_CASE_      :			Optional[int]     =       2

      # New Code #
      # Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
      SCREAMING_SNAKE_CASE_      :			Dict     =       FullyShardedDataParallelPlugin(
          state_dict_config=FullStateDictConfig(offload_to_cpu=_lowercase  ,  ranka_only=_lowercase    )  ,  optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=_lowercase  ,  ranka_only=_lowercase    )  ,  )

      # Initialize accelerator
      if args.with_tracking:
            SCREAMING_SNAKE_CASE_      :			List[str]     =       Accelerator(
                cpu=args.cpu  ,  mixed_precision=args.mixed_precision  ,  log_with="wandb"  ,  project_dir=args.logging_dir  ,  fsdp_plugin=_lowercase  ,  )
      else:
            SCREAMING_SNAKE_CASE_      :			List[Any]     =       Accelerator(fsdp_plugin=_lowercase    )
      accelerator.print(accelerator.distributed_type    )

      if hasattr(args.checkpointing_steps  ,  "isdigit"    ):
            if args.checkpointing_steps == "epoch":
                  SCREAMING_SNAKE_CASE_      :			Dict     =       args.checkpointing_steps
            elif args.checkpointing_steps.isdigit():
                  SCREAMING_SNAKE_CASE_      :			List[str]     =       int(args.checkpointing_steps    )
            else:
                  raise ValueError(
                      F"Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."    )
      else:
            SCREAMING_SNAKE_CASE_      :			int     =       None
      # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
      SCREAMING_SNAKE_CASE_      :			str     =       config["lr"]
      SCREAMING_SNAKE_CASE_      :			Tuple     =       int(config["num_epochs"]    )
      SCREAMING_SNAKE_CASE_      :			List[str]     =       int(config["seed"]    )
      SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       int(config["batch_size"]    )

      # We need to initialize the trackers we use, and also store our configuration
      if args.with_tracking:
            SCREAMING_SNAKE_CASE_      :			Tuple     =       vars(_lowercase    )
            accelerator.init_trackers("fsdp_glue_no_trainer"  ,  _lowercase    )

      SCREAMING_SNAKE_CASE_      :			Dict     =       AutoTokenizer.from_pretrained(args.model_name_or_path    )
      SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       load_dataset("glue"  ,  "mrpc"    )
      SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       evaluate.load("glue"  ,  "mrpc"    )

      def tokenize_function(SCREAMING_SNAKE_CASE_     :	Any    ):
            # max_length=None => use the model max length (it's actually the default)
            SCREAMING_SNAKE_CASE_      :			List[Any]     =       tokenizer(examples["sentence1"]  ,  examples["sentence2"]  ,  truncation=_lowercase  ,  max_length=_lowercase    )
            return outputs

      # Apply the method we just defined to all the examples in all the splits of the dataset
      # starting with the main process first:
      with accelerator.main_process_first():
            SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       datasets.map(
                _lowercase  ,  batched=_lowercase  ,  remove_columns=["idx", "sentence1", "sentence2"]  ,  )

      # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
      # transformers library
      SCREAMING_SNAKE_CASE_      :			Dict     =       tokenized_datasets.rename_column("label"  ,  "labels"    )

      # If the batch size is too big we use gradient accumulation
      SCREAMING_SNAKE_CASE_      :			Any     =       1
      if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
            SCREAMING_SNAKE_CASE_      :			int     =       batch_size // MAX_GPU_BATCH_SIZE
            SCREAMING_SNAKE_CASE_      :			Tuple     =       MAX_GPU_BATCH_SIZE

      def collate_fn(SCREAMING_SNAKE_CASE_     :	Dict    ):
            # On TPU it's best to pad everything to the same length or training will be very slow.
            SCREAMING_SNAKE_CASE_      :			Dict     =       1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
            # When using mixed precision we want round multiples of 8/16
            if accelerator.mixed_precision == "fp8":
                  SCREAMING_SNAKE_CASE_      :			Any     =       1_6
            elif accelerator.mixed_precision != "no":
                  SCREAMING_SNAKE_CASE_      :			Dict     =       8
            else:
                  SCREAMING_SNAKE_CASE_      :			int     =       None

            return tokenizer.pad(
                _lowercase  ,  padding="longest"  ,  max_length=_lowercase  ,  pad_to_multiple_of=_lowercase  ,  return_tensors="pt"  ,  )

      # Instantiate dataloaders.
      SCREAMING_SNAKE_CASE_      :			List[Any]     =       DataLoader(
          tokenized_datasets["train"]  ,  shuffle=_lowercase  ,  collate_fn=_lowercase  ,  batch_size=_lowercase    )
      SCREAMING_SNAKE_CASE_      :			Dict     =       DataLoader(
          tokenized_datasets["validation"]  ,  shuffle=_lowercase  ,  collate_fn=_lowercase  ,  batch_size=_lowercase    )

      set_seed(_lowercase    )

      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      SCREAMING_SNAKE_CASE_      :			Optional[int]     =       AutoModelForSequenceClassification.from_pretrained(
          args.model_name_or_path  ,  return_dict=_lowercase  ,  low_cpu_mem_usage=_lowercase    )

      # New Code #
      # For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
      SCREAMING_SNAKE_CASE_      :			List[Any]     =       accelerator.prepare(_lowercase    )
      accelerator.print(_lowercase    )

      # Instantiate optimizer
      # New Code #
      # For FSDP feature, at present it doesn't support multiple parameter groups,
      # so we need to create a single parameter group for the whole model
      SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       torch.optim.AdamW(params=model.parameters()  ,  lr=_lowercase  ,  weight_decay=2E-4    )

      # Instantiate scheduler
      SCREAMING_SNAKE_CASE_      :			int     =       get_linear_schedule_with_warmup(
          optimizer=_lowercase  ,  num_warmup_steps=1_0  ,  num_training_steps=(len(_lowercase    ) * num_epochs) // gradient_accumulation_steps  ,  )

      # New Code #
      # For FSDP feature, prepare everything except the model as we have already prepared the model
      # before creating the optimizer
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      SCREAMING_SNAKE_CASE_,						SCREAMING_SNAKE_CASE_,						SCREAMING_SNAKE_CASE_,						SCREAMING_SNAKE_CASE_      :			Optional[int]     =       accelerator.prepare(
          _lowercase  ,  _lowercase  ,  _lowercase  ,  _lowercase    )

      SCREAMING_SNAKE_CASE_      :			Any     =       0

      # Potentially load in the weights and states from a previous save
      if args.resume_from_checkpoint:
            if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
                  accelerator.print(F"Resumed from checkpoint: {args.resume_from_checkpoint}"    )
                  accelerator.load_state(args.resume_from_checkpoint    )
                  SCREAMING_SNAKE_CASE_      :			Dict     =       os.path.basename(args.resume_from_checkpoint    )
            else:
                  # Get the most recent checkpoint
                  SCREAMING_SNAKE_CASE_      :			Dict     =       [f.name for f in os.scandir(os.getcwd()    ) if f.is_dir()]
                  dirs.sort(key=os.path.getctime    )
                  SCREAMING_SNAKE_CASE_      :			str     =       dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
            # Extract `epoch_{i}` or `step_{i}`
            SCREAMING_SNAKE_CASE_      :			Any     =       os.path.splitext(_lowercase    )[0]

            if "epoch" in training_difference:
                  num_epochs -= int(training_difference.replace("epoch_"  ,  ""    )    )
                  SCREAMING_SNAKE_CASE_      :			str     =       None
            else:
                  SCREAMING_SNAKE_CASE_      :			Optional[int]     =       int(training_difference.replace("step_"  ,  ""    )    )
                  num_epochs -= resume_step // len(_lowercase    )
                  # If resuming by step, we also need to know exactly how far into the DataLoader we went
                  SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       (num_epochs * len(_lowercase    )) - resume_step

    # Now we train the model
      for epoch in range(_lowercase    ):
            # New Code #
            # context manager to track the peak memory usage during the training epoch
            with TorchTracemalloc() as tracemalloc:
                  model.train()
                  if args.with_tracking:
                        SCREAMING_SNAKE_CASE_      :			Optional[int]     =       0
                  for step, batch in enumerate(_lowercase    ):
                        # We need to skip steps until we reach the resumed step
                        if args.resume_from_checkpoint and epoch == 0:
                              if resume_step is not None and step < resume_step:
                                    pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device    )
                        SCREAMING_SNAKE_CASE_      :			Dict     =       model(**_lowercase    )
                        SCREAMING_SNAKE_CASE_      :			List[str]     =       outputs.loss
                        SCREAMING_SNAKE_CASE_      :			Any     =       loss / gradient_accumulation_steps
                        # We keep track of the loss at each epoch
                        if args.with_tracking:
                              total_loss += loss.detach().float()
                        accelerator.backward(_lowercase    )
                        if step % gradient_accumulation_steps == 0:
                              optimizer.step()
                              lr_scheduler.step()
                              optimizer.zero_grad()
                              # accelerator.print(lr_scheduler.get_lr())

                        overall_step += 1

                        if isinstance(_lowercase  ,  _lowercase    ):
                              SCREAMING_SNAKE_CASE_      :			Tuple     =       F"step_{overall_step}"
                              if overall_step % checkpointing_steps == 0:
                                    if args.output_dir is not None:
                                          SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       os.path.join(args.output_dir  ,  _lowercase    )
                                    accelerator.save_state(_lowercase    )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
            accelerator.print("Memory before entering the train : {}".format(bamb(tracemalloc.begin    )    )    )
            accelerator.print("Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.used    )    )
            accelerator.print("Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.peaked    )    )
            accelerator.print(
                "Total Peak Memory consumed during the train (max): {}".format(
                    tracemalloc.peaked + bamb(tracemalloc.begin    )    )    )
            # Logging the peak memory usage of the GPU to the tracker
            if args.with_tracking:
                  accelerator.log(
                      {
                          "train_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin    ),
                      }  ,  step=_lowercase  ,  )

            # New Code #
            # context manager to track the peak memory usage during the evaluation
            with TorchTracemalloc() as tracemalloc:
                  model.eval()
                  for step, batch in enumerate(_lowercase    ):
                        # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device    )
                        with torch.no_grad():
                              SCREAMING_SNAKE_CASE_      :			List[Any]     =       model(**_lowercase    )
                        SCREAMING_SNAKE_CASE_      :			str     =       outputs.logits.argmax(dim=-1    )
                        SCREAMING_SNAKE_CASE_,						SCREAMING_SNAKE_CASE_      :			Dict     =       accelerator.gather_for_metrics((predictions, batch["labels"])    )
                        metric.add_batch(
                            predictions=_lowercase  ,  references=_lowercase  ,  )

                  SCREAMING_SNAKE_CASE_      :			str     =       metric.compute()
                  # Use accelerator.print to print only on the main process.
                  accelerator.print(F"epoch {epoch}:"  ,  _lowercase    )
                  if args.with_tracking:
                        accelerator.log(
                            {
                                "accuracy": eval_metric["accuracy"],
                                "f1": eval_metric["f1"],
                                "train_loss": total_loss.item() / len(_lowercase    ),
                            }  ,  step=_lowercase  ,  )

                  if checkpointing_steps == "epoch":
                        SCREAMING_SNAKE_CASE_      :			List[Any]     =       F"epoch_{epoch}"
                        if args.output_dir is not None:
                              SCREAMING_SNAKE_CASE_      :			Optional[int]     =       os.path.join(args.output_dir  ,  _lowercase    )
                        accelerator.save_state(_lowercase    )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
            accelerator.print("Memory before entering the eval : {}".format(bamb(tracemalloc.begin    )    )    )
            accelerator.print("Memory consumed at the end of the eval (end-begin): {}".format(tracemalloc.used    )    )
            accelerator.print("Peak Memory consumed during the eval (max-begin): {}".format(tracemalloc.peaked    )    )
            accelerator.print(
                "Total Peak Memory consumed during the eval (max): {}".format(tracemalloc.peaked + bamb(tracemalloc.begin    )    )    )
            # Logging the peak memory usage of the GPU to the tracker
            if args.with_tracking:
                  accelerator.log(
                      {
                          "eval_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin    ),
                      }  ,  step=_lowercase  ,  )

      if args.with_tracking:
            accelerator.end_training()


def 					__lowerCamelCase			(      )							->   List[Any]:
      """simple docstring"""

      SCREAMING_SNAKE_CASE_      :			Tuple     =       argparse.ArgumentParser(description="Simple example of training script."    )
      parser.add_argument(
          "--mixed_precision"  ,  type=_lowercase  ,  default=_lowercase  ,  choices=["no", "fp16", "bf16", "fp8"]  ,  help="Whether to use mixed precision. Choose"
          "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
          "and an Nvidia Ampere GPU."  ,  )
      parser.add_argument("--cpu"  ,  action="store_true"  ,  help="If passed, will train on the CPU."    )
      parser.add_argument(
          "--checkpointing_steps"  ,  type=_lowercase  ,  default=_lowercase  ,  help="Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch."  ,  )
      parser.add_argument(
          "--resume_from_checkpoint"  ,  type=_lowercase  ,  default=_lowercase  ,  help="If the training should continue from a checkpoint folder."  ,  )
      parser.add_argument(
          "--with_tracking"  ,  action="store_true"  ,  help="Whether to load in all available experiment trackers from the environment and use them for logging."  ,  )
      parser.add_argument(
          "--output_dir"  ,  type=_lowercase  ,  default="."  ,  help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory."  ,  )
      parser.add_argument(
          "--logging_dir"  ,  type=_lowercase  ,  default="logs"  ,  help="Location on where to store experiment tracking logs`"  ,  )
      parser.add_argument(
          "--model_name_or_path"  ,  type=_lowercase  ,  help="Path to pretrained model or model identifier from huggingface.co/models."  ,  required=_lowercase  ,  )
      SCREAMING_SNAKE_CASE_      :			Dict     =       parser.parse_args()
      SCREAMING_SNAKE_CASE_      :			int     =       {"lr": 2E-5, "num_epochs": 3, "seed": 4_2, "batch_size": 1_6}
      training_function(_lowercase  ,  _lowercase    )


if __name__ == "__main__":
      main()









import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


__A          = logging.get_logger(__name__)

__A          = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__A          = {
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

__A          = {
    'allenai/longformer-base-4096': 4096,
    'allenai/longformer-large-4096': 4096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4096,
}





class   SCREAMING_SNAKE_CASE (       snake_case    ):





							"""simple docstring"""

							A_       =    VOCAB_FILES_NAMES
							A_       =    PRETRAINED_VOCAB_FILES_MAP
							A_       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_       =    ["input_ids", "attention_mask"]
							A_       =    LongformerTokenizer

							def __init__(     self:      List[str]						,   __A:      Dict=None						,   __A:      int=None						,   __A:      Tuple=None						,   __A:      Any="replace"						,   __A:      List[str]="<s>"						,   __A:      int="</s>"						,   __A:      Dict="</s>"						,   __A:      Tuple="<s>"						,   __A:      Optional[int]="<unk>"						,   __A:      Optional[int]="<pad>"						,   __A:      List[Any]="<mask>"						,   __A:      Tuple=False						,   __A:      Dict=True						,   **__A:      List[Any]						,   )			-> List[Any]:
												super().__init__(
												    __A						,   __A						,   tokenizer_file=__A						,   errors=__A						,   bos_token=__A						,   eos_token=__A						,   sep_token=__A						,   cls_token=__A						,   unk_token=__A						,   pad_token=__A						,   mask_token=__A						,   add_prefix_space=__A						,   trim_offsets=__A						,   **__A						,   )

												_A									=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
												if pre_tok_state.get('''add_prefix_space'''						,   __A      ) != add_prefix_space:
																	_A									=       getattr(__A						,   pre_tok_state.pop('''type'''      )      )
																	_A									=       add_prefix_space
																	_A									=       pre_tok_class(**__A      )

												_A									=       add_prefix_space

												_A									=       '''post_processor'''
												_A									=       getattr(self.backend_tokenizer						,   __A						,   __A      )
												if tokenizer_component_instance:
																	_A									=       json.loads(tokenizer_component_instance.__getstate__()      )

																	# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																	if "sep" in state:
																						_A									=       tuple(state['''sep''']      )
																	if "cls" in state:
																						_A									=       tuple(state['''cls''']      )

																	_A									=       False

																	if state.get('''add_prefix_space'''						,   __A      ) != add_prefix_space:
																						_A									=       add_prefix_space
																						_A									=       True

																	if state.get('''trim_offsets'''						,   __A      ) != trim_offsets:
																						_A									=       trim_offsets
																						_A									=       True

																	if changes_to_apply:
																						_A									=       getattr(__A						,   state.pop('''type'''      )      )
																						_A									=       component_class(**__A      )
																						setattr(self.backend_tokenizer						,   __A						,   __A      )

							@property
							def 					__A       (     self:      Union[str, Any]      )			-> str:
												if self._mask_token is None:
																	if self.verbose:
																						logger.error('''Using mask_token, but it is not set yet.'''      )
																	return None
												return str(self._mask_token      )

							@mask_token.setter
							def 					__A       (     self:      Optional[int]						,   __A:      str      )			-> Union[str, Any]:
												_A									=       AddedToken(__A						,   lstrip=__A						,   rstrip=__A      ) if isinstance(__A						,   __A      ) else value
												_A									=       value

							def 					__A       (     self:      Optional[Any]						,   *__A:      Any						,   **__A:      List[Any]      )			-> BatchEncoding:
												_A									=       kwargs.get('''is_split_into_words'''						,   __A      )
												assert self.add_prefix_space or not is_split_into_words, (
												    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
												    "to use it with pretokenized inputs."
												)

												return super()._batch_encode_plus(*__A						,   **__A      )

							def 					__A       (     self:      Optional[Any]						,   *__A:      List[str]						,   **__A:      List[Any]      )			-> BatchEncoding:
												_A									=       kwargs.get('''is_split_into_words'''						,   __A      )

												assert self.add_prefix_space or not is_split_into_words, (
												    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
												    "to use it with pretokenized inputs."
												)

												return super()._encode_plus(*__A						,   **__A      )

							def 					__A       (     self:      List[str]						,   __A:      str						,   __A:      Optional[str] = None      )			-> Tuple[str]:
												_A									=       self._tokenizer.model.save(__A						,   name=__A      )
												return tuple(__A      )

							def 					__A       (     self:      Tuple						,   __A:      Dict						,   __A:      Optional[Any]=None      )			-> Optional[int]:
												_A									=       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
												if token_ids_a is None:
																	return output

												return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



							def 					__A       (     self:      Optional[int]						,   __A:      List[int]						,   __A:      Optional[List[int]] = None      )			-> List[int]:
												_A									=       [self.sep_token_id]
												_A									=       [self.cls_token_id]

												if token_ids_a is None:
																	return len(cls + token_ids_a + sep      ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]



