



"""simple docstring"""






import numpy as np
import torch
from torch.utils.data import Dataset, IterableDataset

from ..utils.generic import ModelOutput






class 			__snake_case			(      SCREAMING_SNAKE_CASE__):




  def __init__(					self	:		Optional[Any]		,   __lowerCAmelCase	:		str		,   __lowerCAmelCase	:		List[Any]		,   __lowerCAmelCase	:		Union[str, Any]						):






         """simple docstring"""






         _lowerCamelCase		:							str            =					dataset
         _lowerCamelCase		:							Dict            =					process
         _lowerCamelCase		:							Optional[Any]            =					params




  def __len__(					self	:		Tuple						):






         """simple docstring"""






         return len(self.dataset						)




  def __getitem__(					self	:		Union[str, Any]		,   __lowerCAmelCase	:		Optional[Any]						):






         """simple docstring"""






         _lowerCamelCase		:							Optional[Any]            =					self.dataset[i]
         _lowerCamelCase		:							Dict            =					self.process(_lowercase		,   **self.params						)
         return processed






class 			__snake_case			(      SCREAMING_SNAKE_CASE__):




  def __init__(					self	:		Tuple		,   __lowerCAmelCase	:		Tuple		,   __lowerCAmelCase	:		List[Any]		,   __lowerCAmelCase	:		Any		,   __lowerCAmelCase	:		List[Any]=None						):






         """simple docstring"""






         _lowerCamelCase		:							List[str]            =					loader
         _lowerCamelCase		:							Dict            =					infer
         _lowerCamelCase		:							Tuple            =					params
         if loader_batch_size == 1:
                # Let's spare some time by deactivating altogether
                _lowerCamelCase		:							Union[str, Any]            =					None
         _lowerCamelCase		:							Dict            =					loader_batch_size

         # Internal bookkeeping
         _lowerCamelCase		:							Tuple            =					None
         _lowerCamelCase		:							List[Any]            =					None




  def __len__(					self	:		Dict						):






         """simple docstring"""






         return len(self.loader						)




  def __iter__(					self	:		Tuple						):






         """simple docstring"""






         _lowerCamelCase		:							List[str]            =					iter(self.loader						)
         return self




  def 		SCREAMING_SNAKE_CASE							(					self	:		Optional[int]						):






         """simple docstring"""






         if isinstance(self._loader_batch_data		,   torch.Tensor						):
                # Batch data is simple tensor, just fetch the slice
                _lowerCamelCase		:							Tuple            =					self._loader_batch_data[self._loader_batch_index]
         else:
                # Batch data is assumed to be BaseModelOutput (or dict)
                _lowerCamelCase		:							List[str]            =					{}
                for k, element in self._loader_batch_data.items():
                       if isinstance(_lowercase		,   _lowercase						):
                              # Convert ModelOutput to tuple first
                              _lowerCamelCase		:							Any            =					element.to_tuple()
                              if isinstance(element[0]		,   torch.Tensor						):
                                     _lowerCamelCase		:							List[Any]            =					tuple(el[self._loader_batch_index].unsqueeze(0						) for el in element						)
                              elif isinstance(element[0]		,   np.ndarray						):
                                     _lowerCamelCase		:							str            =					tuple(np.expand_dims(el[self._loader_batch_index]		,   0						) for el in element						)
                              continue
                       if k in {"hidden_states", "past_key_values", "attentions"} and isinstance(_lowercase		,   _lowercase						):
                              # Those are stored as lists of tensors so need specific unbatching.
                              if isinstance(element[0]		,   torch.Tensor						):
                                     _lowerCamelCase		:							Optional[int]            =					tuple(el[self._loader_batch_index].unsqueeze(0						) for el in element						)
                              elif isinstance(element[0]		,   np.ndarray						):
                                     _lowerCamelCase		:							str            =					tuple(np.expand_dims(el[self._loader_batch_index]		,   0						) for el in element						)
                              continue
                       if element is None:
                              # This can happen for optional data that get passed around
                              _lowerCamelCase		:							Tuple            =					None
                       elif isinstance(element[self._loader_batch_index]		,   torch.Tensor						):
                              # Take correct batch data, but make it looked like batch_size=1
                              # For compatibility with other methods within transformers

                              _lowerCamelCase		:							Tuple            =					element[self._loader_batch_index].unsqueeze(0						)
                       elif isinstance(element[self._loader_batch_index]		,   np.ndarray						):
                              # Take correct batch data, but make it looked like batch_size=1
                              # For compatibility with other methods within transformers
                              _lowerCamelCase		:							List[str]            =					np.expand_dims(element[self._loader_batch_index]		,   0						)
                       else:
                              # This is typically a list, so no need to `unsqueeze`.
                              _lowerCamelCase		:							Any            =					element[self._loader_batch_index]
            # Recreate the element by reusing the original class to make it look
            # batch_size=1
                _lowerCamelCase		:							Union[str, Any]            =					self._loader_batch_data.__class__(_lowercase						)
         self._loader_batch_index += 1
         return result




  def 		SCREAMING_SNAKE_CASE							(					self	:		Dict						):






         """simple docstring"""






         if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
                # We are currently unrolling a batch so we just need to return
                # the current item within a batch
                return self.loader_batch_item()

         # We're out of items within a batch
         _lowerCamelCase		:							Optional[int]            =					next(self.iterator						)
         _lowerCamelCase		:							str            =					self.infer(_lowercase		,   **self.params						)
         # We now have a batch of "inferred things".
         if self.loader_batch_size is not None:
                # Try to infer the size of the batch
                if isinstance(_lowercase		,   torch.Tensor						):
                       _lowerCamelCase		:							Optional[int]            =					processed
                else:
                       _lowerCamelCase		:							str            =					list(processed.keys()						)[0]
                       _lowerCamelCase		:							Tuple            =					processed[key]
                if isinstance(_lowercase		,   _lowercase						):
                       _lowerCamelCase		:							List[str]            =					len(_lowercase						)
                else:
                       _lowerCamelCase		:							Dict            =					first_tensor.shape[0]
                if 0 < observed_batch_size < self.loader_batch_size:
                       # could be last batch so we can't unroll as many
                       # elements.
                       _lowerCamelCase		:							List[Any]            =					observed_batch_size
                # Setting internal index to unwrap the batch
                _lowerCamelCase		:							Tuple            =					processed
                _lowerCamelCase		:							Tuple            =					0
                return self.loader_batch_item()
         else:
                # We're not unrolling batches
                return processed






class 			__snake_case			(      SCREAMING_SNAKE_CASE__):




  def __init__(					self	:		Tuple		,   __lowerCAmelCase	:		int		,   __lowerCAmelCase	:		Optional[Any]		,   __lowerCAmelCase	:		Optional[int]		,   __lowerCAmelCase	:		List[str]=None						):






         """simple docstring"""






         super().__init__(_lowercase		,   _lowercase		,   _lowercase						)




  def __iter__(					self	:		Optional[int]						):






         """simple docstring"""






         _lowerCamelCase		:							List[Any]            =					iter(self.loader						)
         _lowerCamelCase		:							Any            =					None
         return self




  def 		SCREAMING_SNAKE_CASE							(					self	:		Any						):






         """simple docstring"""






         if self.subiterator is None:
                _lowerCamelCase		:							Dict            =					self.infer(next(self.iterator						)		,   **self.params						)
         try:
                # Try to return next item
                _lowerCamelCase		:							Optional[int]            =					next(self.subiterator						)
         except StopIteration:
                # When a preprocess iterator ends, we can start lookig at the next item
                # ChunkIterator will keep feeding until ALL elements of iterator
                # all have created their subiterator and have been iterating against.
                #
                # Another way to look at it, is we're basically flattening lists of lists
                # into a single list, but with generators
                _lowerCamelCase		:							int            =					self.infer(next(self.iterator						)		,   **self.params						)
                _lowerCamelCase		:							List[str]            =					next(self.subiterator						)
         return processed






class 			__snake_case			(      SCREAMING_SNAKE_CASE__):




  def __iter__(					self	:		Optional[Any]						):






         """simple docstring"""






         _lowerCamelCase		:							Optional[int]            =					iter(self.loader						)
         return self




  def 		SCREAMING_SNAKE_CASE							(					self	:		int						):






         """simple docstring"""






         _lowerCamelCase		:							Optional[int]            =					False
         _lowerCamelCase		:							Dict            =					[]
         if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
                while self._loader_batch_index < self.loader_batch_size:
                       _lowerCamelCase		:							Union[str, Any]            =					self.loader_batch_item()
                       _lowerCamelCase		:							Tuple            =					item.pop('''is_last'''						)
                       accumulator.append(_lowercase						)
                       if is_last:
                              return accumulator

         while not is_last:
                _lowerCamelCase		:							List[str]            =					self.infer(next(self.iterator						)		,   **self.params						)
                if self.loader_batch_size is not None:
                       if isinstance(_lowercase		,   torch.Tensor						):
                              _lowerCamelCase		:							Dict            =					processed
                       else:
                              _lowerCamelCase		:							Optional[int]            =					list(processed.keys()						)[0]
                              _lowerCamelCase		:							Union[str, Any]            =					processed[key]
                       if isinstance(_lowercase		,   _lowercase						):
                              _lowerCamelCase		:							Union[str, Any]            =					len(_lowercase						)
                       else:
                              _lowerCamelCase		:							str            =					first_tensor.shape[0]
                       if 0 < observed_batch_size < self.loader_batch_size:
                              # could be last batch so we can't unroll as many
                              # elements.
                              _lowerCamelCase		:							Optional[int]            =					observed_batch_size
                       _lowerCamelCase		:							Tuple            =					processed
                       _lowerCamelCase		:							List[Any]            =					0
                       while self._loader_batch_index < self.loader_batch_size:
                              _lowerCamelCase		:							Optional[Any]            =					self.loader_batch_item()
                              _lowerCamelCase		:							List[Any]            =					item.pop('''is_last'''						)
                              accumulator.append(_lowercase						)
                              if is_last:
                                     return accumulator
                else:
                       _lowerCamelCase		:							str            =					processed
                       _lowerCamelCase		:							Union[str, Any]            =					item.pop('''is_last'''						)
                       accumulator.append(_lowercase						)
         return accumulator






class 			__snake_case			(      SCREAMING_SNAKE_CASE__):




  def __init__(					self	:		Union[str, Any]		,   __lowerCAmelCase	:		Dataset		,   __lowerCAmelCase	:		str						):






         """simple docstring"""






         _lowerCamelCase		:							Optional[int]            =					dataset
         _lowerCamelCase		:							str            =					key




  def __len__(					self	:		Dict						):






         """simple docstring"""






         return len(self.dataset						)




  def __getitem__(					self	:		int		,   __lowerCAmelCase	:		Union[str, Any]						):






         """simple docstring"""






         return self.dataset[i][self.key]






class 			__snake_case			(      SCREAMING_SNAKE_CASE__):




  def __init__(					self	:		Tuple		,   __lowerCAmelCase	:		Dataset		,   __lowerCAmelCase	:		str		,   __lowerCAmelCase	:		str						):






         """simple docstring"""






         _lowerCamelCase		:							Tuple            =					dataset
         _lowerCamelCase		:							List[Any]            =					keya
         _lowerCamelCase		:							List[Any]            =					keya




  def __len__(					self	:		int						):






         """simple docstring"""






         return len(self.dataset						)




  def __getitem__(					self	:		Tuple		,   __lowerCAmelCase	:		Optional[int]						):






         """simple docstring"""






         return {"text": self.dataset[i][self.keya], "text_pair": self.dataset[i][self.keya]}







"""simple docstring"""







import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class       _a  ( SCREAMING_SNAKE_CASE__     ,	unittest.TestCase):
			__magic_name__     					=     RobertaTokenizer
			__magic_name__     					=     RobertaTokenizerFast
			__magic_name__     					=     True
			__magic_name__     					=     {"""cls_token""": """<s>"""}



			def 				__lowercase					(			self							:				Any		)							->					int:
									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									snake_case   :			int						    = [
									    "l",
									    "o",
									    "w",
									    "e",
									    "r",
									    "s",
									    "t",
									    "i",
									    "d",
									    "n",
									    "\u0120",
									    "\u0120l",
									    "\u0120n",
									    "\u0120lo",
									    "\u0120low",
									    "er",
									    "\u0120lowest",
									    "\u0120newer",
									    "\u0120wider",
									    "<unk>",
									]
									snake_case   :			Optional[int]						    = dict(zip(_lowercase   ,	range(len(_lowercase		)		)		)		)
									snake_case   :			int						    = ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
									snake_case   :			Union[str, Any]						    = {"unk_token": "<unk>"}

									snake_case   :			Optional[Any]						    = os.path.join(self.tmpdirname   ,	VOCAB_FILES_NAMES["vocab_file"]		)
									snake_case   :			str						    = os.path.join(self.tmpdirname   ,	VOCAB_FILES_NAMES["merges_file"]		)
									with open(self.vocab_file   ,	"w"   ,	encoding="utf-8"		) as fp:
															fp.write(json.dumps(_lowercase		) + "\n"		)
									with open(self.merges_file   ,	"w"   ,	encoding="utf-8"		) as fp:
															fp.write("\n".join(_lowercase		)		)



			def 				__lowercase					(			self							:				Optional[Any]   ,	**_lowercase							:				Union[str, Any]		)							->					Dict:
									kwargs.update(self.special_tokens_map		)
									return self.tokenizer_class.from_pretrained(self.tmpdirname   ,	**_lowercase		)



			def 				__lowercase					(			self							:				List[str]   ,	**_lowercase							:				Union[str, Any]		)							->					str:
									kwargs.update(self.special_tokens_map		)
									return RobertaTokenizerFast.from_pretrained(self.tmpdirname   ,	**_lowercase		)



			def 				__lowercase					(			self							:				Union[str, Any]   ,	_lowercase							:				str		)							->					Optional[Any]:
									snake_case   :			Any						    = "lower newer"
									snake_case   :			Union[str, Any]						    = "lower newer"
									return input_text, output_text



			def 				__lowercase					(			self							:				Optional[int]		)							->					Any:
									snake_case   :			Any						    = self.tokenizer_class(self.vocab_file   ,	self.merges_file   ,	**self.special_tokens_map		)
									snake_case   :			Dict						    = "lower newer"
									snake_case   :			Dict						    = ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
									snake_case   :			Any						    = tokenizer.tokenize(_lowercase		)  # , add_prefix_space=True)
									self.assertListEqual(_lowercase   ,	_lowercase		)

									snake_case   :			List[str]						    = tokens + [tokenizer.unk_token]
									snake_case   :			List[Any]						    = [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase		)   ,	_lowercase		)



			def 				__lowercase					(			self							:				int		)							->					str:
									snake_case   :			List[str]						    = self.get_tokenizer()

									self.assertListEqual(tokenizer.encode("Hello world!"   ,	add_special_tokens=_lowercase		)   ,	[0, 31414, 232, 328, 2]		)
									self.assertListEqual(
									    tokenizer.encode("Hello world! cécé herlolip 418"   ,	add_special_tokens=_lowercase		)   ,	[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]   ,	)



			@slow
			def 				__lowercase					(			self							:				Dict		)							->					Optional[int]:
									snake_case   :			Tuple						    = self.tokenizer_class.from_pretrained("roberta-base"		)

									snake_case   :			Tuple						    = tokenizer.encode("sequence builders"   ,	add_special_tokens=_lowercase		)
									snake_case   :			Optional[int]						    = tokenizer.encode("multi-sequence build"   ,	add_special_tokens=_lowercase		)

									snake_case   :			Optional[Any]						    = tokenizer.encode(
									    "sequence builders"   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Optional[int]						    = tokenizer.encode(
									    "sequence builders"   ,	"multi-sequence build"   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)

									snake_case   :			Optional[Any]						    = tokenizer.build_inputs_with_special_tokens(_lowercase		)
									snake_case   :			Any						    = tokenizer.build_inputs_with_special_tokens(_lowercase   ,	_lowercase		)

									assert encoded_sentence == encoded_text_from_decode
									assert encoded_pair == encoded_pair_from_decode



			def 				__lowercase					(			self							:				int		)							->					Any:
									snake_case   :			Any						    = self.get_tokenizer()

									snake_case   :			Optional[Any]						    = "Encode this sequence."
									snake_case   :			List[Any]						    = tokenizer.byte_encoder[" ".encode("utf-8"		)[0]]

									# Testing encoder arguments
									snake_case   :			Any						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Any						    = tokenizer.convert_ids_to_tokens(encoded[0]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)

									snake_case   :			Dict						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Any						    = tokenizer.convert_ids_to_tokens(encoded[0]		)[0]
									self.assertEqual(_lowercase   ,	_lowercase		)

									tokenizer.add_special_tokens({"bos_token": "<s>"}		)
									snake_case   :			Union[str, Any]						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase		)
									snake_case   :			List[str]						    = tokenizer.convert_ids_to_tokens(encoded[1]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)

									# Testing spaces after special tokens
									snake_case   :			Optional[int]						    = "<mask>"
									tokenizer.add_special_tokens(
									    {"mask_token": AddedToken(_lowercase   ,	lstrip=_lowercase   ,	rstrip=_lowercase		)}		)  # mask token has a left space
									snake_case   :			Optional[int]						    = tokenizer.convert_tokens_to_ids(_lowercase		)

									snake_case   :			Tuple						    = "Encode <mask> sequence"
									snake_case   :			Union[str, Any]						    = "Encode <mask>sequence"

									snake_case   :			Union[str, Any]						    = tokenizer.encode(_lowercase		)
									snake_case   :			Optional[int]						    = encoded.index(_lowercase		)
									snake_case   :			int						    = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]		)[0]
									self.assertEqual(_lowercase   ,	_lowercase		)

									snake_case   :			int						    = tokenizer.encode(_lowercase		)
									snake_case   :			Optional[int]						    = encoded.index(_lowercase		)
									snake_case   :			List[Any]						    = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)



			def 				__lowercase					(			self							:				Dict		)							->					Union[str, Any]:
									pass



			def 				__lowercase					(			self							:				List[str]		)							->					str:
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																					snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(_lowercase   ,	**_lowercase		)
																					snake_case   :			Optional[int]						    = self.tokenizer_class.from_pretrained(_lowercase   ,	**_lowercase		)
																					snake_case   :			Optional[int]						    = "A, <mask> AllenNLP sentence."
																					snake_case   :			Tuple						    = tokenizer_r.encode_plus(_lowercase   ,	add_special_tokens=_lowercase   ,	return_token_type_ids=_lowercase		)
																					snake_case   :			Optional[Any]						    = tokenizer_p.encode_plus(_lowercase   ,	add_special_tokens=_lowercase   ,	return_token_type_ids=_lowercase		)

																					# token_type_ids should put 0 everywhere
																					self.assertEqual(sum(tokens_r["token_type_ids"]		)   ,	sum(tokens_p["token_type_ids"]		)		)

																					# attention_mask should put 1 everywhere, so sum over length should be 1
																					self.assertEqual(
																					    sum(tokens_r["attention_mask"]		) / len(tokens_r["attention_mask"]		)   ,	sum(tokens_p["attention_mask"]		) / len(tokens_p["attention_mask"]		)   ,	)

																					snake_case   :			List[str]						    = tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]		)
																					snake_case   :			List[Any]						    = tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]		)

																					# Rust correctly handles the space before the mask while python doesnt
																					self.assertSequenceEqual(tokens_p["input_ids"]   ,	[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]		)
																					self.assertSequenceEqual(tokens_r["input_ids"]   ,	[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]		)

																					self.assertSequenceEqual(
																					    _lowercase   ,	["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]		)
																					self.assertSequenceEqual(
																					    _lowercase   ,	["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]		)



			def 				__lowercase					(			self							:				Tuple		)							->					List[Any]:
									for trim_offsets, add_prefix_space in itertools.product([True, False]   ,	repeat=2		):
															snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(
															    self.tmpdirname   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)

															snake_case   :			Optional[int]						    = json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()		)
															snake_case   :			Union[str, Any]						    = json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()		)

															self.assertEqual(pre_tokenizer_state["add_prefix_space"]   ,	_lowercase		)

															self.assertEqual(post_processor_state["add_prefix_space"]   ,	_lowercase		)
															self.assertEqual(post_processor_state["trim_offsets"]   ,	_lowercase		)



			def 				__lowercase					(			self							:				int		)							->					List[str]:
									# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
									# `trim_offsets`
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																					snake_case   :			List[Any]						    = "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
																					snake_case   :			List[str]						    = F'''{text_of_1_token} {text_of_1_token}'''

																					snake_case   :			Optional[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Union[str, Any]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		) + 1, len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Tuple						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		) + 1, len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Dict						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			int						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		), len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Optional[int]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Optional[int]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		), len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Dict						    = F''' {text}'''

																					# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																					#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																					# )
																					# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																					# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																					# self.assertEqual(
																					#     encoding.offset_mapping[1],
																					#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																					# )

																					snake_case   :			List[str]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			List[str]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(1, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		) + 1, 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Union[str, Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Optional[int]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		), 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Optional[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Union[str, Any]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		), 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)



