





import torch
from torch import nn




class        SCREAMING_SNAKE_CASE__					(		nn.Module ):



 def __init__(				self,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase=1,__lowerCamelCase=False					):
        super().__init__()

        A__		   =		n_token
        A__		   =		d_embed
        A__		   =		d_proj

        A__		   =		cutoffs + [n_token]
        A__		   =		[0] + self.cutoffs
        A__		   =		div_val

        A__		   =		self.cutoffs[0]
        A__		   =		len(self.cutoffs					) - 1
        A__		   =		self.shortlist_size + self.n_clusters

        if self.n_clusters > 0:
               A__		   =		nn.Parameter(torch.zeros(self.n_clusters,self.d_embed					)					)
               A__		   =		nn.Parameter(torch.zeros(self.n_clusters					)					)

        A__		   =		nn.ModuleList()
        A__		   =		nn.ParameterList()

        if div_val == 1:
               for i in range(len(self.cutoffs					)					):
                      if d_proj != d_embed:
                             self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowerCamelCase,__lowerCamelCase					)					)					)
                      else:
                             self.out_projs.append(__lowerCamelCase					)

               self.out_layers.append(nn.Linear(__lowerCamelCase,__lowerCamelCase					)					)
        else:
               for i in range(len(self.cutoffs					)					):
                      A__		   =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
                      A__		   =		d_embed // (div_val**i)

                      self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowerCamelCase,__lowerCamelCase					)					)					)

                      self.out_layers.append(nn.Linear(__lowerCamelCase,r_idx - l_idx					)					)

        A__		   =		keep_order



 def 							UpperCamelCase     (				self,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					):
        if proj is None:
               A__		   =		nn.functional.linear(__lowerCamelCase,__lowerCamelCase,bias=__lowerCamelCase					)
        else:
               # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
               A__		   =		nn.functional.linear(__lowerCamelCase,proj.t().contiguous()					)
               A__		   =		nn.functional.linear(__lowerCamelCase,__lowerCamelCase,bias=__lowerCamelCase					)
               # else:
               #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
               #     if bias is not None:
               #         logit = logit + bias

        return logit



 def 							UpperCamelCase     (				self,__lowerCamelCase,__lowerCamelCase=None,__lowerCamelCase=False					):

        if labels is not None:
               # Shift so that tokens < n predict n
               A__		   =		hidden[..., :-1, :].contiguous()
               A__		   =		labels[..., 1:].contiguous()
               A__		   =		hidden.view(-1,hidden.size(-1					)					)
               A__		   =		labels.view(-1					)
               if hidden.size(0					) != labels.size(0					):
                      raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''					)
        else:
               A__		   =		hidden.view(-1,hidden.size(-1					)					)

        if self.n_clusters == 0:
               A__		   =		self._compute_logit(__lowerCamelCase,self.out_layers[0].weight,self.out_layers[0].bias,self.out_projs[0]					)
               if labels is not None:
                      A__		   =		labels != -100
                      A__		   =		torch.zeros_like(__lowerCamelCase,dtype=hidden.dtype,device=hidden.device					)
                      A__		   =		(
                          -nn.functional.log_softmax(__lowerCamelCase,dim=-1					)[mask].gather(1,labels[mask].unsqueeze(1					)					).squeeze(1					)
                      )
               else:
                      A__		   =		nn.functional.log_softmax(__lowerCamelCase,dim=-1					)
        else:
               # construct weights and biases
               A__		   =		[], []
               for i in range(len(self.cutoffs					)					):
                      if self.div_val == 1:
                             A__		   =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
                             A__		   =		self.out_layers[0].weight[l_idx:r_idx]
                             A__		   =		self.out_layers[0].bias[l_idx:r_idx]
                      else:
                             A__		   =		self.out_layers[i].weight
                             A__		   =		self.out_layers[i].bias

                      if i == 0:
                             A__		   =		torch.cat([weight_i, self.cluster_weight],dim=0					)
                             A__		   =		torch.cat([bias_i, self.cluster_bias],dim=0					)

                      weights.append(__lowerCamelCase					)
                      biases.append(__lowerCamelCase					)

               A__		   =		weights[0], biases[0], self.out_projs[0]

               A__		   =		self._compute_logit(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					)
               A__		   =		nn.functional.log_softmax(__lowerCamelCase,dim=1					)

               if labels is None:
                      A__		   =		hidden.new_empty((head_logit.size(0					), self.n_token)					)
               else:
                      A__		   =		torch.zeros_like(__lowerCamelCase,dtype=hidden.dtype,device=hidden.device					)

               A__		   =		0
               A__		   =		[0] + self.cutoffs
               for i in range(len(__lowerCamelCase					) - 1					):
                      A__		   =		cutoff_values[i], cutoff_values[i + 1]

                      if labels is not None:
                             A__		   =		(labels >= l_idx) & (labels < r_idx)
                             A__		   =		mask_i.nonzero().squeeze()

                             if indices_i.numel() == 0:
                                    continue

                             A__		   =		labels.index_select(0,__lowerCamelCase					) - l_idx
                             A__		   =		head_logprob.index_select(0,__lowerCamelCase					)
                             A__		   =		hidden.index_select(0,__lowerCamelCase					)
                      else:
                             A__		   =		hidden

                      if i == 0:
                             if labels is not None:
                                    A__		   =		head_logprob_i.gather(1,target_i[:, None]					).squeeze(1					)
                             else:
                                    A__		   =		head_logprob[:, : self.cutoffs[0]]
                      else:
                             A__		   =		weights[i], biases[i], self.out_projs[i]

                             A__		   =		self._compute_logit(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					)
                             A__		   =		nn.functional.log_softmax(__lowerCamelCase,dim=1					)
                             A__		   =		self.cutoffs[0] + i - 1  # No probability for the head cluster
                             if labels is not None:
                                    A__		   =		head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                        1,target_i[:, None]					).squeeze(1					)
                             else:
                                    A__		   =		head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                                    A__		   =		logprob_i

                      if labels is not None:
                             if (hasattr(self,'''keep_order'''					) and self.keep_order) or keep_order:
                                    out.index_copy_(0,__lowerCamelCase,-logprob_i					)
                             else:
                                    out[offset : offset + logprob_i.size(0					)].copy_(-logprob_i					)
                             offset += logprob_i.size(0					)

        return out




 def 							UpperCamelCase     (				self,__lowerCamelCase					):
        if self.n_clusters == 0:
               A__		   =		self._compute_logit(__lowerCamelCase,self.out_layers[0].weight,self.out_layers[0].bias,self.out_projs[0]					)
               return nn.functional.log_softmax(__lowerCamelCase,dim=-1					)
        else:
               # construct weights and biases
               A__		   =		[], []
               for i in range(len(self.cutoffs					)					):
                      if self.div_val == 1:
                             A__		   =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
                             A__		   =		self.out_layers[0].weight[l_idx:r_idx]
                             A__		   =		self.out_layers[0].bias[l_idx:r_idx]
                      else:
                             A__		   =		self.out_layers[i].weight
                             A__		   =		self.out_layers[i].bias

                      if i == 0:
                             A__		   =		torch.cat([weight_i, self.cluster_weight],dim=0					)
                             A__		   =		torch.cat([bias_i, self.cluster_bias],dim=0					)

                      weights.append(__lowerCamelCase					)
                      biases.append(__lowerCamelCase					)

               A__		   =		weights[0], biases[0], self.out_projs[0]
               A__		   =		self._compute_logit(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					)

               A__		   =		hidden.new_empty((head_logit.size(0					), self.n_token)					)
               A__		   =		nn.functional.log_softmax(__lowerCamelCase,dim=1					)

               A__		   =		[0] + self.cutoffs
               for i in range(len(__lowerCamelCase					) - 1					):
                      A__		   =		cutoff_values[i], cutoff_values[i + 1]

                      if i == 0:
                             A__		   =		head_logprob[:, : self.cutoffs[0]]
                      else:
                             A__		   =		weights[i], biases[i], self.out_projs[i]

                             A__		   =		self._compute_logit(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					)
                             A__		   =		nn.functional.log_softmax(__lowerCamelCase,dim=1					)

                             A__		   =		head_logprob[:, -i] + tail_logprob_i
                             A__		   =		logprob_i

               return out













"""simple docstring"""







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      UpperCamelCase		(__snake_case						,						unittest.TestCase     ):
						_SCREAMING_SNAKE_CASE  : List[Any]												=		LongformerTokenizer
						_SCREAMING_SNAKE_CASE  : int												=		True
						_SCREAMING_SNAKE_CASE  : List[str]												=		LongformerTokenizerFast
						_SCREAMING_SNAKE_CASE  : Dict												=		True






						def      __snake_case      (		self  :Dict				)      ->Tuple:
										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										lowercase    :      Dict       = [
										    """l""",
										    """o""",
										    """w""",
										    """e""",
										    """r""",
										    """s""",
										    """t""",
										    """i""",
										    """d""",
										    """n""",
										    """\u0120""",
										    """\u0120l""",
										    """\u0120n""",
										    """\u0120lo""",
										    """\u0120low""",
										    """er""",
										    """\u0120lowest""",
										    """\u0120newer""",
										    """\u0120wider""",
										    """<unk>""",
										]
										lowercase    :      Tuple       = dict(zip(__magic_name__			, range(len(__magic_name__				)				)				)				)
										lowercase    :      Tuple       = ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
										lowercase    :      Dict       = {"""unk_token""": """<unk>"""}

										lowercase    :      Dict       = os.path.join(self.tmpdirname			, VOCAB_FILES_NAMES["""vocab_file"""]				)
										lowercase    :      Optional[int]       = os.path.join(self.tmpdirname			, VOCAB_FILES_NAMES["""merges_file"""]				)
										with open(self.vocab_file			, """w"""			, encoding="""utf-8"""				) as fp:
														fp.write(json.dumps(__magic_name__				) + """\n"""				)
										with open(self.merges_file			, """w"""			, encoding="""utf-8"""				) as fp:
														fp.write("""\n""".join(__magic_name__				)				)






						def      __snake_case      (		self  :List[Any]			, **__magic_name__  :Any				)      ->Tuple:
										kwargs.update(self.special_tokens_map				)
										return self.tokenizer_class.from_pretrained(self.tmpdirname			, **__magic_name__				)






						def      __snake_case      (		self  :Optional[Any]			, **__magic_name__  :Optional[Any]				)      ->Dict:
										kwargs.update(self.special_tokens_map				)
										return self.rust_tokenizer_class.from_pretrained(self.tmpdirname			, **__magic_name__				)






						def      __snake_case      (		self  :Tuple			, __magic_name__  :Dict				)      ->str:
										lowercase    :      List[str]       = """lower newer"""
										lowercase    :      Any       = """lower newer"""
										return input_text, output_text






						def      __snake_case      (		self  :Tuple				)      ->Union[str, Any]:
										lowercase    :      Union[str, Any]       = self.tokenizer_class(self.vocab_file			, self.merges_file			, **self.special_tokens_map				)
										lowercase    :      List[str]       = """lower newer"""
										lowercase    :      Tuple       = ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
										lowercase    :      Optional[int]       = tokenizer.tokenize(__magic_name__				)  # , add_prefix_space=True)
										self.assertListEqual(__magic_name__			, __magic_name__				)

										lowercase    :      str       = tokens + [tokenizer.unk_token]
										lowercase    :      Any       = [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__				)			, __magic_name__				)






						def      __snake_case      (		self  :Any				)      ->str:
										lowercase    :      int       = self.get_tokenizer()

										self.assertListEqual(tokenizer.encode("""Hello world!"""			, add_special_tokens=__magic_name__				)			, [0, 31_414, 232, 328, 2]				)
										self.assertListEqual(
										    tokenizer.encode("""Hello world! cécé herlolip 418"""			, add_special_tokens=__magic_name__				)			, [0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]			, )






						@slow
						def      __snake_case      (		self  :Tuple				)      ->Union[str, Any]:
										lowercase    :      Any       = self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""				)

										lowercase    :      Tuple       = tokenizer.encode("""sequence builders"""			, add_special_tokens=__magic_name__				)
										lowercase    :      List[Any]       = tokenizer.encode("""multi-sequence build"""			, add_special_tokens=__magic_name__				)

										lowercase    :      Union[str, Any]       = tokenizer.encode(
										    """sequence builders"""			, add_special_tokens=__magic_name__			, add_prefix_space=__magic_name__				)
										lowercase    :      List[Any]       = tokenizer.encode(
										    """sequence builders"""			, """multi-sequence build"""			, add_special_tokens=__magic_name__			, add_prefix_space=__magic_name__				)

										lowercase    :      int       = tokenizer.build_inputs_with_special_tokens(__magic_name__				)
										lowercase    :      List[Any]       = tokenizer.build_inputs_with_special_tokens(__magic_name__			, __magic_name__				)

										assert encoded_sentence == encoded_text_from_decode
										assert encoded_pair == encoded_pair_from_decode






						def      __snake_case      (		self  :Optional[Any]				)      ->int:
										lowercase    :      Optional[int]       = self.get_tokenizer()

										lowercase    :      Tuple       = """Encode this sequence."""
										lowercase    :      Dict       = tokenizer.byte_encoder[""" """.encode("""utf-8"""				)[0]]

										# Testing encoder arguments
										lowercase    :      List[Any]       = tokenizer.encode(__magic_name__			, add_special_tokens=__magic_name__			, add_prefix_space=__magic_name__				)
										lowercase    :      int       = tokenizer.convert_ids_to_tokens(encoded[0]				)[0]
										self.assertNotEqual(__magic_name__			, __magic_name__				)

										lowercase    :      Optional[Any]       = tokenizer.encode(__magic_name__			, add_special_tokens=__magic_name__			, add_prefix_space=__magic_name__				)
										lowercase    :      Tuple       = tokenizer.convert_ids_to_tokens(encoded[0]				)[0]
										self.assertEqual(__magic_name__			, __magic_name__				)

										tokenizer.add_special_tokens({"""bos_token""": """<s>"""}				)
										lowercase    :      Union[str, Any]       = tokenizer.encode(__magic_name__			, add_special_tokens=__magic_name__				)
										lowercase    :      List[str]       = tokenizer.convert_ids_to_tokens(encoded[1]				)[0]
										self.assertNotEqual(__magic_name__			, __magic_name__				)

										# Testing spaces after special tokens
										lowercase    :      Any       = """<mask>"""
										tokenizer.add_special_tokens(
										    {"""mask_token""": AddedToken(__magic_name__			, lstrip=__magic_name__			, rstrip=__magic_name__				)}				)  # mask token has a left space
										lowercase    :      Any       = tokenizer.convert_tokens_to_ids(__magic_name__				)

										lowercase    :      Any       = """Encode <mask> sequence"""
										lowercase    :      str       = """Encode <mask>sequence"""

										lowercase    :      Optional[int]       = tokenizer.encode(__magic_name__				)
										lowercase    :      List[str]       = encoded.index(__magic_name__				)
										lowercase    :      Tuple       = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]				)[0]
										self.assertEqual(__magic_name__			, __magic_name__				)

										lowercase    :      Tuple       = tokenizer.encode(__magic_name__				)
										lowercase    :      List[str]       = encoded.index(__magic_name__				)
										lowercase    :      Optional[Any]       = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]				)[0]
										self.assertNotEqual(__magic_name__			, __magic_name__				)






						def      __snake_case      (		self  :Any				)      ->int:
										pass






						def      __snake_case      (		self  :List[Any]				)      ->str:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																		lowercase    :      int       = self.rust_tokenizer_class.from_pretrained(__magic_name__			, **__magic_name__				)
																		lowercase    :      List[str]       = self.tokenizer_class.from_pretrained(__magic_name__			, **__magic_name__				)
																		lowercase    :      Optional[int]       = """A, <mask> AllenNLP sentence."""
																		lowercase    :      Any       = tokenizer_r.encode_plus(__magic_name__			, add_special_tokens=__magic_name__			, return_token_type_ids=__magic_name__				)
																		lowercase    :      str       = tokenizer_p.encode_plus(__magic_name__			, add_special_tokens=__magic_name__			, return_token_type_ids=__magic_name__				)

																		# token_type_ids should put 0 everywhere
																		self.assertEqual(sum(tokens_r["""token_type_ids"""]				)			, sum(tokens_p["""token_type_ids"""]				)				)

																		# attention_mask should put 1 everywhere, so sum over length should be 1
																		self.assertEqual(
																		    sum(tokens_r["""attention_mask"""]				) / len(tokens_r["""attention_mask"""]				)			, sum(tokens_p["""attention_mask"""]				) / len(tokens_p["""attention_mask"""]				)			, )

																		lowercase    :      Union[str, Any]       = tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]				)
																		lowercase    :      Dict       = tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]				)

																		# Rust correctly handles the space before the mask while python doesnt
																		self.assertSequenceEqual(tokens_p["""input_ids"""]			, [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]				)
																		self.assertSequenceEqual(tokens_r["""input_ids"""]			, [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]				)

																		self.assertSequenceEqual(
																		    __magic_name__			, ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]				)
																		self.assertSequenceEqual(
																		    __magic_name__			, ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]				)






						def      __snake_case      (		self  :List[str]				)      ->Tuple:
										for trim_offsets, add_prefix_space in itertools.product([True, False]			, repeat=2				):
														lowercase    :      str       = self.rust_tokenizer_class.from_pretrained(
														    self.tmpdirname			, use_fast=__magic_name__			, add_prefix_space=__magic_name__			, trim_offsets=__magic_name__				)

														lowercase    :      str       = json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()				)
														lowercase    :      Optional[Any]       = json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()				)

														self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]			, __magic_name__				)

														self.assertEqual(post_processor_state["""add_prefix_space"""]			, __magic_name__				)
														self.assertEqual(post_processor_state["""trim_offsets"""]			, __magic_name__				)






						def      __snake_case      (		self  :Dict				)      ->List[str]:
										# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
										# `trim_offsets`
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																		lowercase    :      Optional[Any]       = """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
																		lowercase    :      Optional[Any]       = f"""{text_of_1_token} {text_of_1_token}"""

																		lowercase    :      List[str]       = self.rust_tokenizer_class.from_pretrained(
																		    __magic_name__			, use_fast=__magic_name__			, add_prefix_space=__magic_name__			, trim_offsets=__magic_name__				)
																		lowercase    :      List[Any]       = tokenizer_r(__magic_name__			, return_offsets_mapping=__magic_name__			, add_special_tokens=__magic_name__				)
																		self.assertEqual(encoding.offset_mapping[0]			, (0, len(__magic_name__				))				)
																		self.assertEqual(
																		    encoding.offset_mapping[1]			, (len(__magic_name__				) + 1, len(__magic_name__				) + 1 + len(__magic_name__				))			, )

																		lowercase    :      List[Any]       = self.rust_tokenizer_class.from_pretrained(
																		    __magic_name__			, use_fast=__magic_name__			, add_prefix_space=__magic_name__			, trim_offsets=__magic_name__				)
																		lowercase    :      List[Any]       = tokenizer_r(__magic_name__			, return_offsets_mapping=__magic_name__			, add_special_tokens=__magic_name__				)
																		self.assertEqual(encoding.offset_mapping[0]			, (0, len(__magic_name__				))				)
																		self.assertEqual(
																		    encoding.offset_mapping[1]			, (len(__magic_name__				) + 1, len(__magic_name__				) + 1 + len(__magic_name__				))			, )

																		lowercase    :      Tuple       = self.rust_tokenizer_class.from_pretrained(
																		    __magic_name__			, use_fast=__magic_name__			, add_prefix_space=__magic_name__			, trim_offsets=__magic_name__				)
																		lowercase    :      List[str]       = tokenizer_r(__magic_name__			, return_offsets_mapping=__magic_name__			, add_special_tokens=__magic_name__				)
																		self.assertEqual(encoding.offset_mapping[0]			, (0, len(__magic_name__				))				)
																		self.assertEqual(
																		    encoding.offset_mapping[1]			, (len(__magic_name__				), len(__magic_name__				) + 1 + len(__magic_name__				))			, )

																		lowercase    :      Optional[Any]       = self.rust_tokenizer_class.from_pretrained(
																		    __magic_name__			, use_fast=__magic_name__			, add_prefix_space=__magic_name__			, trim_offsets=__magic_name__				)
																		lowercase    :      Optional[int]       = tokenizer_r(__magic_name__			, return_offsets_mapping=__magic_name__			, add_special_tokens=__magic_name__				)
																		self.assertEqual(encoding.offset_mapping[0]			, (0, len(__magic_name__				))				)
																		self.assertEqual(
																		    encoding.offset_mapping[1]			, (len(__magic_name__				), len(__magic_name__				) + 1 + len(__magic_name__				))			, )

																		lowercase    :      Optional[int]       = f""" {text}"""

																		# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																		#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																		# )
																		# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																		# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																		# self.assertEqual(
																		#     encoding.offset_mapping[1],
																		#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																		# )

																		lowercase    :      Union[str, Any]       = self.rust_tokenizer_class.from_pretrained(
																		    __magic_name__			, use_fast=__magic_name__			, add_prefix_space=__magic_name__			, trim_offsets=__magic_name__				)
																		lowercase    :      Union[str, Any]       = tokenizer_r(__magic_name__			, return_offsets_mapping=__magic_name__			, add_special_tokens=__magic_name__				)
																		self.assertEqual(encoding.offset_mapping[0]			, (1, 1 + len(__magic_name__				))				)
																		self.assertEqual(
																		    encoding.offset_mapping[1]			, (1 + len(__magic_name__				) + 1, 1 + len(__magic_name__				) + 1 + len(__magic_name__				))			, )

																		lowercase    :      int       = self.rust_tokenizer_class.from_pretrained(
																		    __magic_name__			, use_fast=__magic_name__			, add_prefix_space=__magic_name__			, trim_offsets=__magic_name__				)
																		lowercase    :      Optional[int]       = tokenizer_r(__magic_name__			, return_offsets_mapping=__magic_name__			, add_special_tokens=__magic_name__				)
																		self.assertEqual(encoding.offset_mapping[0]			, (0, 1 + len(__magic_name__				))				)
																		self.assertEqual(
																		    encoding.offset_mapping[1]			, (1 + len(__magic_name__				), 1 + len(__magic_name__				) + 1 + len(__magic_name__				))			, )

																		lowercase    :      str       = self.rust_tokenizer_class.from_pretrained(
																		    __magic_name__			, use_fast=__magic_name__			, add_prefix_space=__magic_name__			, trim_offsets=__magic_name__				)
																		lowercase    :      Any       = tokenizer_r(__magic_name__			, return_offsets_mapping=__magic_name__			, add_special_tokens=__magic_name__				)
																		self.assertEqual(encoding.offset_mapping[0]			, (0, 1 + len(__magic_name__				))				)
																		self.assertEqual(
																		    encoding.offset_mapping[1]			, (1 + len(__magic_name__				), 1 + len(__magic_name__				) + 1 + len(__magic_name__				))			, )
