


'''simple docstring'''






import argparse
import logging
import os
import time
import timeit

import datasets
import numpy as np
import pycuda.autoinit  # noqa: F401
import pycuda.driver as cuda
import tensorrt as trt
import torch
from absl import logging as absl_logging
from accelerate import Accelerator
from datasets import load_dataset, load_metric
from torch.utils.data import DataLoader
from utils_qa import postprocess_qa_predictions

import transformers
from transformers import AutoTokenizer, EvalPrediction, default_data_collator, set_seed
from transformers.trainer_pt_utils import nested_concat, nested_truncate


__UpperCAmelCase					       =							trt.Logger(trt.Logger.WARNING)
__UpperCAmelCase					       =							absl_logging.get_absl_logger()
absl_logger.setLevel(logging.WARNING)

__UpperCAmelCase					       =							logging.getLogger(__name__)

__UpperCAmelCase					       =							argparse.ArgumentParser()

# Required parameters
parser.add_argument(
    '''--onnx_model_path''',
    default=None,
    type=str,
    required=True,
    help='''Path to ONNX model: ''',
)

parser.add_argument(
    '''--output_dir''',
    default=None,
    type=str,
    required=True,
    help='''The output directory where the model checkpoints and predictions will be written.''',
)

# Other parameters

parser.add_argument(
    '''--tokenizer_name''',
    default='''''',
    type=str,
    required=True,
    help='''Pretrained tokenizer name or path if not the same as model_name''',
)

parser.add_argument(
    '''--version_2_with_negative''',
    action='''store_true''',
    help='''If true, the SQuAD examples contain some that do not have an answer.''',
)
parser.add_argument(
    '''--null_score_diff_threshold''',
    type=float,
    default=0.0,
    help='''If null_score - best_non_null is greater than the threshold predict null.''',
)

parser.add_argument(
    '''--max_seq_length''',
    default=384,
    type=int,
    help=(
        '''The maximum total input sequence length after WordPiece tokenization. Sequences '''
        '''longer than this will be truncated, and sequences shorter than this will be padded.'''
    ),
)
parser.add_argument(
    '''--doc_stride''',
    default=128,
    type=int,
    help='''When splitting up a long document into chunks, how much stride to take between chunks.''',
)

parser.add_argument('''--per_device_eval_batch_size''', default=8, type=int, help='''Batch size per GPU/CPU for evaluation.''')

parser.add_argument(
    '''--n_best_size''',
    default=20,
    type=int,
    help='''The total number of n-best predictions to generate in the nbest_predictions.json output file.''',
)
parser.add_argument(
    '''--max_answer_length''',
    default=30,
    type=int,
    help=(
        '''The maximum length of an answer that can be generated. This is needed because the start '''
        '''and end predictions are not conditioned on one another.'''
    ),
)

parser.add_argument('''--seed''', type=int, default=42, help='''random seed for initialization''')

parser.add_argument(
    '''--dataset_name''',
    type=str,
    default=None,
    required=True,
    help='''The name of the dataset to use (via the datasets library).''',
)
parser.add_argument(
    '''--dataset_config_name''',
    type=str,
    default=None,
    help='''The configuration name of the dataset to use (via the datasets library).''',
)
parser.add_argument(
    '''--preprocessing_num_workers''', type=int, default=4, help='''A csv or a json file containing the training data.'''
)
parser.add_argument('''--overwrite_cache''', action='''store_true''', help='''Overwrite the cached training and evaluation sets''')
parser.add_argument(
    '''--fp16''',
    action='''store_true''',
    help='''Whether to use 16-bit (mixed) precision instead of 32-bit''',
)
parser.add_argument(
    '''--int8''',
    action='''store_true''',
    help='''Whether to use INT8''',
)

__UpperCAmelCase					       =							parser.parse_args()

if args.tokenizer_name:
		__UpperCAmelCase					       =							AutoTokenizer.from_pretrained(args.tokenizer_name, use_fast=True)
else:
		raise ValueError(
		    '''You are instantiating a new tokenizer from scratch. This is not supported by this script.'''
		    '''You can do it from another script, save it, and load it from here, using --tokenizer_name.'''
		)

logger.info('''Training/evaluation parameters %s''', args)

__UpperCAmelCase					       =							args.per_device_eval_batch_size

__UpperCAmelCase					       =							(args.eval_batch_size, args.max_seq_length)

# TRT Engine properties
__UpperCAmelCase					       =							True

__UpperCAmelCase					       =							'''temp_engine/bert-fp32.engine'''
if args.fpaa:
		__UpperCAmelCase					       =							'''temp_engine/bert-fp16.engine'''
if args.inta:
		__UpperCAmelCase					       =							'''temp_engine/bert-int8.engine'''

# import ONNX file
if not os.path.exists('''temp_engine'''):
		os.makedirs('''temp_engine''')

__UpperCAmelCase					       =							1 << (int)(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
with trt.Builder(TRT_LOGGER) as builder, builder.create_network(EXPLICIT_BATCH) as network, trt.OnnxParser(
    network, TRT_LOGGER
) as parser:
		with open(args.onnx_model_path, '''rb''') as model:
				if not parser.parse(model.read()):
						for error in range(parser.num_errors):
								print(parser.get_error(error))

    # Query input names and shapes from parsed TensorRT network
		__UpperCAmelCase					       =							[network.get_input(i) for i in range(network.num_inputs)]
		__UpperCAmelCase					       =							[_input.name for _input in network_inputs]  # ex: ["actual_input1"]

		with builder.create_builder_config() as config:
				__UpperCAmelCase					       =							1 << 50
				if STRICT_TYPES:
						config.set_flag(trt.BuilderFlag.STRICT_TYPES)
				if args.fpaa:
						config.set_flag(trt.BuilderFlag.FPaa)
				if args.inta:
						config.set_flag(trt.BuilderFlag.INTa)
				__UpperCAmelCase					       =							builder.create_optimization_profile()
				config.add_optimization_profile(profile)
				for i in range(len(input_names)):
						profile.set_shape(input_names[i], INPUT_SHAPE, INPUT_SHAPE, INPUT_SHAPE)
				__UpperCAmelCase					       =							builder.build_engine(network, config)

				# serialize_engine and store in file (can be directly loaded and deserialized):
				with open(engine_name, '''wb''') as f:
						f.write(engine.serialize())


def       _snake_case     (  A				,					A				,					A				,					A				,					A				,					A				,					A				,					A )		->					Optional[Any]:
							lowerCAmelCase__      =			np.asarray(inputs['''input_ids''']				,					dtype=np.intaa )
							lowerCAmelCase__      =			np.asarray(inputs['''attention_mask''']				,					dtype=np.intaa )
							lowerCAmelCase__      =			np.asarray(inputs['''token_type_ids''']				,					dtype=np.intaa )

							# Copy inputs
							cuda.memcpy_htod_async(d_inputs[0]				,					input_ids.ravel()				,					snake_case__ )
							cuda.memcpy_htod_async(d_inputs[1]				,					attention_mask.ravel()				,					snake_case__ )
							cuda.memcpy_htod_async(d_inputs[2]				,					token_type_ids.ravel()				,					snake_case__ )
							# start time
							lowerCAmelCase__      =			time.time()
							# Run inference
							context.execute_async(
							    bindings=[int(snake_case__ ) for d_inp in d_inputs] + [int(snake_case__ ), int(snake_case__ )]				,					stream_handle=stream.handle )
							# Transfer predictions back from GPU
							cuda.memcpy_dtoh_async(snake_case__				,					snake_case__				,					snake_case__ )
							cuda.memcpy_dtoh_async(snake_case__				,					snake_case__				,					snake_case__ )
							# Synchronize the stream and take time
							stream.synchronize()
							# end time
							lowerCAmelCase__      =			time.time()
							lowerCAmelCase__      =			end_time - start_time
							lowerCAmelCase__      =			(h_outputa, h_outputa)
							# print(outputs)
							return outputs, infer_time


# Initialize the accelerator. We will let the accelerator handle device placement for us in this example.
__UpperCAmelCase					       =							Accelerator()
# Make one log on every process with the configuration for debugging.
logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''',
    datefmt='''%m/%d/%Y %H:%M:%S''',
    level=logging.INFO,
)

# Setup logging, we only want one process per machine to log things on the screen.
# accelerator.is_local_main_process is only True for one process per machine.
logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR)
if accelerator.is_local_main_process:
		datasets.utils.logging.set_verbosity_warning()
		transformers.utils.logging.set_verbosity_info()
else:
		datasets.utils.logging.set_verbosity_error()
		transformers.utils.logging.set_verbosity_error()

# If passed along, set the training seed now.
if args.seed is not None:
		set_seed(args.seed)

# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
# (the dataset will be downloaded automatically from the datasets Hub).
#
# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
# 'text' is found. You can easily tweak this behavior (see below).
if args.dataset_name is not None:
		# Downloading and loading a dataset from the hub.
		__UpperCAmelCase					       =							load_dataset(args.dataset_name, args.dataset_config_name)
else:
		raise ValueError('''Evaluation requires a dataset name''')
# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
# https://huggingface.co/docs/datasets/loading_datasets.html.

# Preprocessing the datasets.
# Preprocessing is slighlty different for training and evaluation.

__UpperCAmelCase					       =							raw_datasets['''validation'''].column_names

__UpperCAmelCase					       =							'''question''' if '''question''' in column_names else column_names[0]
__UpperCAmelCase					       =							'''context''' if '''context''' in column_names else column_names[1]
__UpperCAmelCase					       =							'''answers''' if '''answers''' in column_names else column_names[2]

# Padding side determines if we do (question|context) or (context|question).
__UpperCAmelCase					       =							tokenizer.padding_side == '''right'''

if args.max_seq_length > tokenizer.model_max_length:
		logger.warning(
		    f"""The max_seq_length passed ({args.max_seq_length}) is larger than the maximum length for the"""
		    f"""model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."""
		)

__UpperCAmelCase					       =							min(args.max_seq_length, tokenizer.model_max_length)


def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			[q.lstrip() for q in examples[question_column_name]]

							# Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results
							# in one example possible giving several features when a context is long, each of those features having a
							# context that overlaps a bit the context of the previous feature.
							lowerCAmelCase__      =			tokenizer(
							    examples[question_column_name if pad_on_right else context_column_name]				,					examples[context_column_name if pad_on_right else question_column_name]				,					truncation='''only_second''' if pad_on_right else '''only_first'''				,					max_length=snake_case__				,					stride=args.doc_stride				,					return_overflowing_tokens=snake_case__				,					return_offsets_mapping=snake_case__				,					padding='''max_length'''				,					)

							# Since one example might give us several features if it has a long context, we need a map from a feature to
							# its corresponding example. This key gives us just that.
							lowerCAmelCase__      =			tokenized_examples.pop('''overflow_to_sample_mapping''' )

							# For evaluation, we will need to convert our predictions to substrings of the context, so we keep the
							# corresponding example_id and we will store the offset mappings.
							lowerCAmelCase__      =			[]

							for i in range(len(tokenized_examples['''input_ids'''] ) ):
														# Grab the sequence corresponding to that example (to know what is the context and what is the question).
														lowerCAmelCase__      =			tokenized_examples.sequence_ids(snake_case__ )
														lowerCAmelCase__      =			1 if pad_on_right else 0

														# One example can give several spans, this is the index of the example containing this span of text.
														lowerCAmelCase__      =			sample_mapping[i]
														tokenized_examples["example_id"].append(examples['''id'''][sample_index] )

														# Set to None the offset_mapping that are not part of the context so it's easy to determine if a token
														# position is part of the context or not.
														lowerCAmelCase__      =			[
														    (o if sequence_ids[k] == context_index else None)
														    for k, o in enumerate(tokenized_examples['''offset_mapping'''][i] )
														]

							return tokenized_examples


__UpperCAmelCase					       =							raw_datasets['''validation''']
# Validation Feature Creation
__UpperCAmelCase					       =							eval_examples.map(
    prepare_validation_features,
    batched=True,
    num_proc=args.preprocessing_num_workers,
    remove_columns=column_names,
    load_from_cache_file=not args.overwrite_cache,
    desc='''Running tokenizer on validation dataset''',
)

__UpperCAmelCase					       =							default_data_collator

__UpperCAmelCase					       =							eval_dataset.remove_columns(['''example_id''', '''offset_mapping'''])
__UpperCAmelCase					       =							DataLoader(
    eval_dataset_for_model, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size
)


def       _snake_case     (  A				,					A				,					A				,					A="eval" )		->					Dict:
							lowerCAmelCase__      =			postprocess_qa_predictions(
							    examples=snake_case__				,					features=snake_case__				,					predictions=snake_case__				,					version_2_with_negative=args.version_2_with_negative				,					n_best_size=args.n_best_size				,					max_answer_length=args.max_answer_length				,					null_score_diff_threshold=args.null_score_diff_threshold				,					output_dir=args.output_dir				,					prefix=snake_case__				,					)
							# Format the result to the format the metric expects.
							if args.version_2_with_negative:
														lowerCAmelCase__      =			[
														    {"""id""": k, """prediction_text""": v, """no_answer_probability""": 0.0} for k, v in predictions.items()
														]
							else:
														lowerCAmelCase__      =			[{"""id""": k, """prediction_text""": v} for k, v in predictions.items()]

							lowerCAmelCase__      =			[{"""id""": ex["""id"""], """answers""": ex[answer_column_name]} for ex in examples]
							return EvalPrediction(predictions=snake_case__				,					label_ids=snake_case__ )


__UpperCAmelCase					       =							load_metric('''squad_v2''' if args.version_2_with_negative else '''squad''')

# Evaluation!
logger.info('''Loading ONNX model %s for evaluation''', args.onnx_model_path)
with open(engine_name, '''rb''') as f, trt.Runtime(TRT_LOGGER) as runtime, runtime.deserialize_cuda_engine(
    f.read()
) as engine, engine.create_execution_context() as context:
		# setup for TRT inferrence
		for i in range(len(input_names)):
				context.set_binding_shape(i, INPUT_SHAPE)
		assert context.all_binding_shapes_specified


		def       _snake_case     (  A )		->					List[Any]:
									return trt.volume(engine.get_binding_shape(snake_case__ ) ) * engine.get_binding_dtype(snake_case__ ).itemsize

		# Allocate device memory for inputs and outputs.
		__UpperCAmelCase					       =							[cuda.mem_alloc(binding_nbytes(binding)) for binding in engine if engine.binding_is_input(binding)]

		# Allocate output buffer
		__UpperCAmelCase					       =							cuda.pagelocked_empty(tuple(context.get_binding_shape(3)), dtype=np.floataa)
		__UpperCAmelCase					       =							cuda.pagelocked_empty(tuple(context.get_binding_shape(4)), dtype=np.floataa)
		__UpperCAmelCase					       =							cuda.mem_alloc(h_outputa.nbytes)
		__UpperCAmelCase					       =							cuda.mem_alloc(h_outputa.nbytes)

		# Create a stream in which to copy inputs/outputs and run inference.
		__UpperCAmelCase					       =							cuda.Stream()

		# Evaluation
		logger.info('''***** Running Evaluation *****''')
		logger.info(f"""  Num examples = {len(eval_dataset)}""")
		logger.info(f"""  Batch size = {args.per_device_eval_batch_size}""")

		__UpperCAmelCase					       =							0.0
		__UpperCAmelCase					       =							0
		__UpperCAmelCase					       =							timeit.default_timer()

		__UpperCAmelCase					       =							None
		for step, batch in enumerate(eval_dataloader):
				__UpperCAmelCase       ,    __UpperCAmelCase					       =							model_infer(batch, context, d_inputs, h_outputa, h_outputa, d_outputa, d_outputa, stream)
				total_time += infer_time
				niter += 1

				__UpperCAmelCase       ,    __UpperCAmelCase					       =							outputs
				__UpperCAmelCase					       =							torch.tensor(start_logits)
				__UpperCAmelCase					       =							torch.tensor(end_logits)

				# necessary to pad predictions and labels for being gathered
				__UpperCAmelCase					       =							accelerator.pad_across_processes(start_logits, dim=1, pad_index=-100)
				__UpperCAmelCase					       =							accelerator.pad_across_processes(end_logits, dim=1, pad_index=-100)

				__UpperCAmelCase					       =							(accelerator.gather(start_logits).cpu().numpy(), accelerator.gather(end_logits).cpu().numpy())
				__UpperCAmelCase					       =							logits if all_preds is None else nested_concat(all_preds, logits, padding_index=-100)

		if all_preds is not None:
				__UpperCAmelCase					       =							nested_truncate(all_preds, len(eval_dataset))

		__UpperCAmelCase					       =							timeit.default_timer() - start_time
		logger.info('''  Evaluation done in total %f secs (%f sec per example)''', evalTime, evalTime / len(eval_dataset))
		# Inference time from TRT
		logger.info('''Average Inference Time = {:.3f} ms'''.format(total_time * 1_000 / niter))
		logger.info('''Total Inference Time =  {:.3f} ms'''.format(total_time * 1_000))






		logger.info('''Total Number of Inference =  %d''', niter)

__UpperCAmelCase					       =							post_processing_function(eval_examples, eval_dataset, all_preds)
__UpperCAmelCase					       =							metric.compute(predictions=prediction.predictions, references=prediction.label_ids)






logger.info(f"""Evaluation metrics: {eval_metric}""")



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCAmelCase					       =							{
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/tokenizer.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/tokenizer.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json''',
        '''roberta-base-openai-detector''': (
            '''https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'''
        ),
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'''
        ),
    },
}

__UpperCAmelCase					       =							{
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}




class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     Optional[Any]					=   VOCAB_FILES_NAMES
							lowercase__    :     int					=   PRETRAINED_VOCAB_FILES_MAP
							lowercase__    :     Any					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__    :     List[Any]					=   ["input_ids", "attention_mask"]
							lowercase__    :     Dict					=   RobertaTokenizer







							def __init__(	self      ,					lowerCamelCase_=None      ,					lowerCamelCase_=None      ,					lowerCamelCase_=None      ,					lowerCamelCase_="replace"      ,					lowerCamelCase_="<s>"      ,					lowerCamelCase_="</s>"      ,					lowerCamelCase_="</s>"      ,					lowerCamelCase_="<s>"      ,					lowerCamelCase_="<unk>"      ,					lowerCamelCase_="<pad>"      ,					lowerCamelCase_="<mask>"      ,					lowerCamelCase_=False      ,					lowerCamelCase_=True      ,					**lowerCamelCase_      ,					) ->							Tuple:
														super().__init__(
														    lowerCamelCase_      ,					lowerCamelCase_      ,					tokenizer_file=lowerCamelCase_      ,					errors=lowerCamelCase_      ,					bos_token=lowerCamelCase_      ,					eos_token=lowerCamelCase_      ,					sep_token=lowerCamelCase_      ,					cls_token=lowerCamelCase_      ,					unk_token=lowerCamelCase_      ,					pad_token=lowerCamelCase_      ,					mask_token=lowerCamelCase_      ,					add_prefix_space=lowerCamelCase_      ,					trim_offsets=lowerCamelCase_      ,					**lowerCamelCase_      ,					)

														lowerCAmelCase__      =			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()     )
														if pre_tok_state.get('''add_prefix_space'''      ,					lowerCamelCase_     ) != add_prefix_space:
																					lowerCAmelCase__      =			getattr(lowerCamelCase_      ,					pre_tok_state.pop('''type'''     )     )
																					lowerCAmelCase__      =			add_prefix_space
																					lowerCAmelCase__      =			pre_tok_class(**lowerCamelCase_     )

														lowerCAmelCase__      =			add_prefix_space

														lowerCAmelCase__      =			'''post_processor'''
														lowerCAmelCase__      =			getattr(self.backend_tokenizer      ,					lowerCamelCase_      ,					lowerCamelCase_     )
														if tokenizer_component_instance:
																					lowerCAmelCase__      =			json.loads(tokenizer_component_instance.__getstate__()     )

																					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																					if "sep" in state:
																												lowerCAmelCase__      =			tuple(state['''sep''']     )
																					if "cls" in state:
																												lowerCAmelCase__      =			tuple(state['''cls''']     )

																					lowerCAmelCase__      =			False

																					if state.get('''add_prefix_space'''      ,					lowerCamelCase_     ) != add_prefix_space:
																												lowerCAmelCase__      =			add_prefix_space
																												lowerCAmelCase__      =			True

																					if state.get('''trim_offsets'''      ,					lowerCamelCase_     ) != trim_offsets:
																												lowerCAmelCase__      =			trim_offsets
																												lowerCAmelCase__      =			True

																					if changes_to_apply:
																												lowerCAmelCase__      =			getattr(lowerCamelCase_      ,					state.pop('''type'''     )     )
																												lowerCAmelCase__      =			component_class(**lowerCamelCase_     )
																												setattr(self.backend_tokenizer      ,					lowerCamelCase_      ,					lowerCamelCase_     )







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														if self._mask_token is None:
																					if self.verbose:
																												logger.error('''Using mask_token, but it is not set yet.'''     )
																					return None
														return str(self._mask_token     )







							@mask_token.setter
							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Optional[Any]:
														lowerCAmelCase__      =			AddedToken(lowerCamelCase_      ,					lstrip=lowerCamelCase_      ,					rstrip=lowerCamelCase_     ) if isinstance(lowerCamelCase_      ,					lowerCamelCase_     ) else value
														lowerCAmelCase__      =			value







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							BatchEncoding:
														lowerCAmelCase__      =			kwargs.get('''is_split_into_words'''      ,					lowerCamelCase_     )
														assert self.add_prefix_space or not is_split_into_words, (
														    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._batch_encode_plus(*lowerCamelCase_      ,					**lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							BatchEncoding:
														lowerCAmelCase__      =			kwargs.get('''is_split_into_words'''      ,					lowerCamelCase_     )

														assert self.add_prefix_space or not is_split_into_words, (
														    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._encode_plus(*lowerCamelCase_      ,					**lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							Tuple[str]:
														lowerCAmelCase__      =			self._tokenizer.model.save(lowerCamelCase_      ,					name=lowerCamelCase_     )
														return tuple(lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_=None     ) ->							Tuple:
														lowerCAmelCase__      =			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
														if token_ids_a is None:
																					return output

														return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							List[int]:
														lowerCAmelCase__      =			[self.sep_token_id]
														lowerCAmelCase__      =			[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep     ) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]