
'''simple docstring'''



import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import numpy as np
from utils_multiple_choice import MultipleChoiceDataset, Split, processors

import transformers
from transformers import (
    AutoConfig,
    AutoModelForMultipleChoice,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import is_main_process


_lowercase          =						logging.getLogger(__name__)


def A (__lowerCamelCase :Tuple					,      __lowerCamelCase :Dict						):
						return (preds == labels).mean()




@dataclass
class     UpperCAmelCase_	:






						'''simple docstring'''






						_lowercase						: str										=  field(
						    metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}					)
						_lowercase						: Optional[str]										=  field(
						    default=_SCREAMING_SNAKE_CASE			,				metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}					)
						_lowercase						: Optional[str]										=  field(
						    default=_SCREAMING_SNAKE_CASE			,				metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}					)
						_lowercase						: Optional[str]										=  field(
						    default=_SCREAMING_SNAKE_CASE			,				metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}			,				)






@dataclass
class     UpperCAmelCase_	:






						'''simple docstring'''






						_lowercase						: str										=  field(metadata={'''help''': '''The name of the task to train on: ''' + ''', '''.join(processors.keys()					)}					)
						_lowercase						: str										=  field(metadata={'''help''': '''Should contain the data files for the task.'''}					)
						_lowercase						: int										=  field(
						    default=1_2_8			,				metadata={
						        '''help''': (
						            '''The maximum total input sequence length after tokenization. Sequences longer '''
						            '''than this will be truncated, sequences shorter will be padded.'''
						        )
						    }			,				)
						_lowercase						: bool										=  field(
						    default=_SCREAMING_SNAKE_CASE			,				metadata={'''help''': '''Overwrite the cached training and evaluation sets'''}					)






def A ():
						# See all possible arguments in src/transformers/training_args.py
						# or by passing the --help flag to this script.
						# We now keep distinct sets of args, for a cleaner separation of concerns.

						_lowerCAmelCase    		=					HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)						)
						_lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase    		=					parser.parse_args_into_dataclasses()

						if (
						    os.path.exists(training_args.output_dir						)
						    and os.listdir(training_args.output_dir						)
						    and training_args.do_train
						    and not training_args.overwrite_output_dir
						):
												raise ValueError(
												    f'Output directory ({training_args.output_dir}) already exists and is not empty. Use'
												    """ --overwrite_output_dir to overcome."""						)

						# Setup logging
						logging.basicConfig(
						    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""					,      datefmt="""%m/%d/%Y %H:%M:%S"""					,      level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN					,      )
						logger.warning(
						    """Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"""					,      training_args.local_rank					,      training_args.device					,      training_args.n_gpu					,      bool(training_args.local_rank != -1						)					,      training_args.fpaa					,      )
						# Set the verbosity to info of the Transformers logger (on main process only):
						if is_main_process(training_args.local_rank						):
												transformers.utils.logging.set_verbosity_info()
												transformers.utils.logging.enable_default_handler()
												transformers.utils.logging.enable_explicit_format()
						logger.info("""Training/evaluation parameters %s"""					,      __lowerCamelCase						)

						# Set seed
						set_seed(training_args.seed						)

						try:
												_lowerCAmelCase    		=					processors[data_args.task_name]()
												_lowerCAmelCase    		=					processor.get_labels()
												_lowerCAmelCase    		=					len(__lowerCamelCase						)
						except KeyError:
												raise ValueError("""Task not found: %s""" % (data_args.task_name)						)

						# Load pretrained model and tokenizer
						#
						# Distributed training:
						# The .from_pretrained methods guarantee that only one local process can concurrently
						# download model & vocab.

						_lowerCAmelCase    		=					AutoConfig.from_pretrained(
						    model_args.config_name if model_args.config_name else model_args.model_name_or_path					,      num_labels=__lowerCamelCase					,      finetuning_task=data_args.task_name					,      cache_dir=model_args.cache_dir					,      )
						_lowerCAmelCase    		=					AutoTokenizer.from_pretrained(
						    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path					,      cache_dir=model_args.cache_dir					,      )
						_lowerCAmelCase    		=					AutoModelForMultipleChoice.from_pretrained(
						    model_args.model_name_or_path					,      from_tf=bool(""".ckpt""" in model_args.model_name_or_path						)					,      config=__lowerCamelCase					,      cache_dir=model_args.cache_dir					,      )

						# Get datasets
						_lowerCAmelCase    		=					(
						    MultipleChoiceDataset(
						        data_dir=data_args.data_dir					,      tokenizer=__lowerCamelCase					,      task=data_args.task_name					,      max_seq_length=data_args.max_seq_length					,      overwrite_cache=data_args.overwrite_cache					,      mode=Split.train					,      )
						    if training_args.do_train
						    else None
						)
						_lowerCAmelCase    		=					(
						    MultipleChoiceDataset(
						        data_dir=data_args.data_dir					,      tokenizer=__lowerCamelCase					,      task=data_args.task_name					,      max_seq_length=data_args.max_seq_length					,      overwrite_cache=data_args.overwrite_cache					,      mode=Split.dev					,      )
						    if training_args.do_eval
						    else None
						)

						def compute_metrics(__lowerCamelCase :EvalPrediction						) -> Dict:
												_lowerCAmelCase    		=					np.argmax(p.predictions					,      axis=1						)
												return {"acc": simple_accuracy(__lowerCamelCase					,      p.label_ids						)}

						# Data collator
						_lowerCAmelCase    		=					DataCollatorWithPadding(__lowerCamelCase					,      pad_to_multiple_of=8						) if training_args.fpaa else None

						# Initialize our Trainer
						_lowerCAmelCase    		=					Trainer(
						    model=__lowerCamelCase					,      args=__lowerCamelCase					,      train_dataset=__lowerCamelCase					,      eval_dataset=__lowerCamelCase					,      compute_metrics=__lowerCamelCase					,      data_collator=__lowerCamelCase					,      )

						# Training
						if training_args.do_train:
												trainer.train(
												    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path						) else None						)
												trainer.save_model()
												# For convenience, we also re-save the tokenizer to the same directory,
												# so that you can share your model easily on huggingface.co/models =)
												if trainer.is_world_master():
																		tokenizer.save_pretrained(training_args.output_dir						)

    # Evaluation
						_lowerCAmelCase    		=					{}
						if training_args.do_eval:
												logger.info("""*** Evaluate ***"""						)

												_lowerCAmelCase    		=					trainer.evaluate()

												_lowerCAmelCase    		=					os.path.join(training_args.output_dir					,      """eval_results.txt"""						)
												if trainer.is_world_master():
																		with open(__lowerCamelCase					,      """w"""						) as writer:
																								logger.info("""***** Eval results *****"""						)
																								for key, value in result.items():
																														logger.info("""  %s = %s"""					,      __lowerCamelCase					,      __lowerCamelCase						)
																														writer.write("""%s = %s\n""" % (key, value)						)

																								results.update(__lowerCamelCase						)

						return results


def A (__lowerCamelCase :int						):
						# For xla_spawn (TPUs)
						main()


if __name__ == "__main__":
						main()





"""simple docstring"""







import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class       _a  ( SCREAMING_SNAKE_CASE__     ,	unittest.TestCase):
			__magic_name__     					=     RobertaTokenizer
			__magic_name__     					=     RobertaTokenizerFast
			__magic_name__     					=     True
			__magic_name__     					=     {"""cls_token""": """<s>"""}



			def 				__lowercase					(			self							:				Any		)							->					int:
									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									snake_case   :			int						    = [
									    "l",
									    "o",
									    "w",
									    "e",
									    "r",
									    "s",
									    "t",
									    "i",
									    "d",
									    "n",
									    "\u0120",
									    "\u0120l",
									    "\u0120n",
									    "\u0120lo",
									    "\u0120low",
									    "er",
									    "\u0120lowest",
									    "\u0120newer",
									    "\u0120wider",
									    "<unk>",
									]
									snake_case   :			Optional[int]						    = dict(zip(_lowercase   ,	range(len(_lowercase		)		)		)		)
									snake_case   :			int						    = ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
									snake_case   :			Union[str, Any]						    = {"unk_token": "<unk>"}

									snake_case   :			Optional[Any]						    = os.path.join(self.tmpdirname   ,	VOCAB_FILES_NAMES["vocab_file"]		)
									snake_case   :			str						    = os.path.join(self.tmpdirname   ,	VOCAB_FILES_NAMES["merges_file"]		)
									with open(self.vocab_file   ,	"w"   ,	encoding="utf-8"		) as fp:
															fp.write(json.dumps(_lowercase		) + "\n"		)
									with open(self.merges_file   ,	"w"   ,	encoding="utf-8"		) as fp:
															fp.write("\n".join(_lowercase		)		)



			def 				__lowercase					(			self							:				Optional[Any]   ,	**_lowercase							:				Union[str, Any]		)							->					Dict:
									kwargs.update(self.special_tokens_map		)
									return self.tokenizer_class.from_pretrained(self.tmpdirname   ,	**_lowercase		)



			def 				__lowercase					(			self							:				List[str]   ,	**_lowercase							:				Union[str, Any]		)							->					str:
									kwargs.update(self.special_tokens_map		)
									return RobertaTokenizerFast.from_pretrained(self.tmpdirname   ,	**_lowercase		)



			def 				__lowercase					(			self							:				Union[str, Any]   ,	_lowercase							:				str		)							->					Optional[Any]:
									snake_case   :			Any						    = "lower newer"
									snake_case   :			Union[str, Any]						    = "lower newer"
									return input_text, output_text



			def 				__lowercase					(			self							:				Optional[int]		)							->					Any:
									snake_case   :			Any						    = self.tokenizer_class(self.vocab_file   ,	self.merges_file   ,	**self.special_tokens_map		)
									snake_case   :			Dict						    = "lower newer"
									snake_case   :			Dict						    = ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
									snake_case   :			Any						    = tokenizer.tokenize(_lowercase		)  # , add_prefix_space=True)
									self.assertListEqual(_lowercase   ,	_lowercase		)

									snake_case   :			List[str]						    = tokens + [tokenizer.unk_token]
									snake_case   :			List[Any]						    = [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase		)   ,	_lowercase		)



			def 				__lowercase					(			self							:				int		)							->					str:
									snake_case   :			List[str]						    = self.get_tokenizer()

									self.assertListEqual(tokenizer.encode("Hello world!"   ,	add_special_tokens=_lowercase		)   ,	[0, 31414, 232, 328, 2]		)
									self.assertListEqual(
									    tokenizer.encode("Hello world! cécé herlolip 418"   ,	add_special_tokens=_lowercase		)   ,	[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]   ,	)



			@slow
			def 				__lowercase					(			self							:				Dict		)							->					Optional[int]:
									snake_case   :			Tuple						    = self.tokenizer_class.from_pretrained("roberta-base"		)

									snake_case   :			Tuple						    = tokenizer.encode("sequence builders"   ,	add_special_tokens=_lowercase		)
									snake_case   :			Optional[int]						    = tokenizer.encode("multi-sequence build"   ,	add_special_tokens=_lowercase		)

									snake_case   :			Optional[Any]						    = tokenizer.encode(
									    "sequence builders"   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Optional[int]						    = tokenizer.encode(
									    "sequence builders"   ,	"multi-sequence build"   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)

									snake_case   :			Optional[Any]						    = tokenizer.build_inputs_with_special_tokens(_lowercase		)
									snake_case   :			Any						    = tokenizer.build_inputs_with_special_tokens(_lowercase   ,	_lowercase		)

									assert encoded_sentence == encoded_text_from_decode
									assert encoded_pair == encoded_pair_from_decode



			def 				__lowercase					(			self							:				int		)							->					Any:
									snake_case   :			Any						    = self.get_tokenizer()

									snake_case   :			Optional[Any]						    = "Encode this sequence."
									snake_case   :			List[Any]						    = tokenizer.byte_encoder[" ".encode("utf-8"		)[0]]

									# Testing encoder arguments
									snake_case   :			Any						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Any						    = tokenizer.convert_ids_to_tokens(encoded[0]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)

									snake_case   :			Dict						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Any						    = tokenizer.convert_ids_to_tokens(encoded[0]		)[0]
									self.assertEqual(_lowercase   ,	_lowercase		)

									tokenizer.add_special_tokens({"bos_token": "<s>"}		)
									snake_case   :			Union[str, Any]						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase		)
									snake_case   :			List[str]						    = tokenizer.convert_ids_to_tokens(encoded[1]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)

									# Testing spaces after special tokens
									snake_case   :			Optional[int]						    = "<mask>"
									tokenizer.add_special_tokens(
									    {"mask_token": AddedToken(_lowercase   ,	lstrip=_lowercase   ,	rstrip=_lowercase		)}		)  # mask token has a left space
									snake_case   :			Optional[int]						    = tokenizer.convert_tokens_to_ids(_lowercase		)

									snake_case   :			Tuple						    = "Encode <mask> sequence"
									snake_case   :			Union[str, Any]						    = "Encode <mask>sequence"

									snake_case   :			Union[str, Any]						    = tokenizer.encode(_lowercase		)
									snake_case   :			Optional[int]						    = encoded.index(_lowercase		)
									snake_case   :			int						    = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]		)[0]
									self.assertEqual(_lowercase   ,	_lowercase		)

									snake_case   :			int						    = tokenizer.encode(_lowercase		)
									snake_case   :			Optional[int]						    = encoded.index(_lowercase		)
									snake_case   :			List[Any]						    = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)



			def 				__lowercase					(			self							:				Dict		)							->					Union[str, Any]:
									pass



			def 				__lowercase					(			self							:				List[str]		)							->					str:
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																					snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(_lowercase   ,	**_lowercase		)
																					snake_case   :			Optional[int]						    = self.tokenizer_class.from_pretrained(_lowercase   ,	**_lowercase		)
																					snake_case   :			Optional[int]						    = "A, <mask> AllenNLP sentence."
																					snake_case   :			Tuple						    = tokenizer_r.encode_plus(_lowercase   ,	add_special_tokens=_lowercase   ,	return_token_type_ids=_lowercase		)
																					snake_case   :			Optional[Any]						    = tokenizer_p.encode_plus(_lowercase   ,	add_special_tokens=_lowercase   ,	return_token_type_ids=_lowercase		)

																					# token_type_ids should put 0 everywhere
																					self.assertEqual(sum(tokens_r["token_type_ids"]		)   ,	sum(tokens_p["token_type_ids"]		)		)

																					# attention_mask should put 1 everywhere, so sum over length should be 1
																					self.assertEqual(
																					    sum(tokens_r["attention_mask"]		) / len(tokens_r["attention_mask"]		)   ,	sum(tokens_p["attention_mask"]		) / len(tokens_p["attention_mask"]		)   ,	)

																					snake_case   :			List[str]						    = tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]		)
																					snake_case   :			List[Any]						    = tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]		)

																					# Rust correctly handles the space before the mask while python doesnt
																					self.assertSequenceEqual(tokens_p["input_ids"]   ,	[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]		)
																					self.assertSequenceEqual(tokens_r["input_ids"]   ,	[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]		)

																					self.assertSequenceEqual(
																					    _lowercase   ,	["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]		)
																					self.assertSequenceEqual(
																					    _lowercase   ,	["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]		)



			def 				__lowercase					(			self							:				Tuple		)							->					List[Any]:
									for trim_offsets, add_prefix_space in itertools.product([True, False]   ,	repeat=2		):
															snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(
															    self.tmpdirname   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)

															snake_case   :			Optional[int]						    = json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()		)
															snake_case   :			Union[str, Any]						    = json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()		)

															self.assertEqual(pre_tokenizer_state["add_prefix_space"]   ,	_lowercase		)

															self.assertEqual(post_processor_state["add_prefix_space"]   ,	_lowercase		)
															self.assertEqual(post_processor_state["trim_offsets"]   ,	_lowercase		)



			def 				__lowercase					(			self							:				int		)							->					List[str]:
									# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
									# `trim_offsets`
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																					snake_case   :			List[Any]						    = "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
																					snake_case   :			List[str]						    = F'''{text_of_1_token} {text_of_1_token}'''

																					snake_case   :			Optional[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Union[str, Any]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		) + 1, len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Tuple						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		) + 1, len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Dict						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			int						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		), len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Optional[int]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Optional[int]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		), len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Dict						    = F''' {text}'''

																					# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																					#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																					# )
																					# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																					# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																					# self.assertEqual(
																					#     encoding.offset_mapping[1],
																					#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																					# )

																					snake_case   :			List[str]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			List[str]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(1, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		) + 1, 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Union[str, Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Optional[int]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		), 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Optional[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Union[str, Any]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		), 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)



