



"""simple docstring"""






from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


snake_case   					=				logging.get_logger(__name__)



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):



   def __init__(       self    :	str							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	float							,       **__lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      feature_size
     _snake_case   	=      sampling_rate
     _snake_case   	=      padding_value

     _snake_case   	=      kwargs.pop('''padding_side'''							,       '''right'''				)
     _snake_case   	=      kwargs.pop('''return_attention_mask'''							,       __lowerCamelCase				)

     super().__init__(**__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	Union[
           BatchFeature,
           List[BatchFeature],
           Dict[str, BatchFeature],
           Dict[str, List[BatchFeature]],
           List[Dict[str, BatchFeature]],
       ]							,       __lowerCamelCase    :	Union[bool, str, PaddingStrategy] = True							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	bool = False							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	Optional[bool] = None							,       __lowerCamelCase    :	Optional[Union[str, TensorType]] = None							,       ):



     """simple docstring"""
     # If we have a list of dicts, let's convert it in a dict of lists
     # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
     if isinstance(__lowerCamelCase							,       (list, tuple)				) and isinstance(processed_features[0]							,       (dict, BatchFeature)				):
       _snake_case   	=      {
           key: [example[key] for example in processed_features] for key in processed_features[0].keys()
       }

     # The model's main input name, usually `input_values`, has be passed for padding
     if self.model_input_names[0] not in processed_features:
       raise ValueError(
           '''You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`'''
           f""" to this method that includes {self.model_input_names[0]}, but you provided"""
           f""" {list(processed_features.keys()				)}"""				)

     _snake_case   	=      processed_features[self.model_input_names[0]]
     _snake_case   	=      (
         return_attention_mask if return_attention_mask is not None else self.return_attention_mask
     )

     if len(__lowerCamelCase				) == 0:
       if return_attention_mask:
         _snake_case   	=      []
       return processed_features

     # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
     # and rebuild them afterwards if no return_tensors is specified
     # Note that we lose the specific device the tensor may be on for PyTorch

     _snake_case   	=      required_input[0]
     if isinstance(__lowerCamelCase							,       (list, tuple)				):
       # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
       _snake_case   	=      0
       while len(required_input[index]				) == 0:
         index += 1
       if index < len(__lowerCamelCase				):
         _snake_case   	=      required_input[index][0]

     if return_tensors is None:
       if is_tf_tensor(__lowerCamelCase				):
         _snake_case   	=      '''tf'''
       elif is_torch_tensor(__lowerCamelCase				):
         _snake_case   	=      '''pt'''
       elif isinstance(__lowerCamelCase							,       (int, float, list, tuple, np.ndarray)				):
         _snake_case   	=      '''np'''
       else:
         raise ValueError(
             f"""type of {first_element} unknown: {type(__lowerCamelCase				)}. """
             '''Should be one of a python, numpy, pytorch or tensorflow object.'''				)

     for key, value in processed_features.items():
       if isinstance(value[0]							,       (int, float)				):
         _snake_case   	=      to_numpy(__lowerCamelCase				)
       else:
         _snake_case   	=      [to_numpy(__lowerCamelCase				) for v in value]

        # Convert padding_strategy in PaddingStrategy
     _snake_case   	=      self._get_padding_strategies(padding=__lowerCamelCase							,       max_length=__lowerCamelCase				)

     _snake_case   	=      processed_features[self.model_input_names[0]]

     _snake_case   	=      len(__lowerCamelCase				)
     if not all(len(__lowerCamelCase				) == batch_size for v in processed_features.values()				):
       raise ValueError('''Some items in the output dictionary have a different batch size than others.'''				)

     _snake_case   	=      []
     for i in range(__lowerCamelCase				):
       _snake_case   	=      {k: v[i] for k, v in processed_features.items()}
       # truncation
       _snake_case   	=      self._truncate(
           __lowerCamelCase							,       max_length=__lowerCamelCase							,       pad_to_multiple_of=__lowerCamelCase							,       truncation=__lowerCamelCase							,       )
       truncated_inputs.append(__lowerCamelCase				)

     if padding_strategy == PaddingStrategy.LONGEST:
       # make sure that `max_length` cannot be longer than the longest truncated length
       _snake_case   	=      max(len(input_slice[self.model_input_names[0]]				) for input_slice in truncated_inputs				)
       _snake_case   	=      PaddingStrategy.MAX_LENGTH

     _snake_case   	=      {}
     for i in range(__lowerCamelCase				):
       # padding
       _snake_case   	=      self._pad(
           truncated_inputs[i]							,       max_length=__lowerCamelCase							,       padding_strategy=__lowerCamelCase							,       pad_to_multiple_of=__lowerCamelCase							,       return_attention_mask=__lowerCamelCase							,       )

       for key, value in outputs.items():
         if key not in batch_outputs:
           _snake_case   	=      []
         if value.dtype is np.dtype(np.floataa				):
           _snake_case   	=      value.astype(np.floataa				)
         batch_outputs[key].append(__lowerCamelCase				)

     return BatchFeature(__lowerCamelCase							,       tensor_type=__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	Union[Dict[str, np.ndarray], BatchFeature]							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	PaddingStrategy = PaddingStrategy.DO_NOT_PAD							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	Optional[bool] = None							,       ):



     """simple docstring"""
     _snake_case   	=      processed_features[self.model_input_names[0]]

     if padding_strategy == PaddingStrategy.LONGEST:
       _snake_case   	=      len(__lowerCamelCase				)

     if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
       _snake_case   	=      ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

     _snake_case   	=      padding_strategy != PaddingStrategy.DO_NOT_PAD and len(__lowerCamelCase				) < max_length

     if return_attention_mask and "attention_mask" not in processed_features:
       _snake_case   	=      np.ones(len(__lowerCamelCase				)							,       dtype=np.intaa				)

     if needs_to_be_padded:
       _snake_case   	=      max_length - len(__lowerCamelCase				)
       if self.padding_side == "right":
         if return_attention_mask:
           _snake_case   	=      np.pad(
               processed_features['''attention_mask''']							,       (0, difference)				)
         _snake_case   	=      ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
         _snake_case   	=      np.pad(
             __lowerCamelCase							,       __lowerCamelCase							,       '''constant'''							,       constant_values=self.padding_value				)
       elif self.padding_side == "left":
         if return_attention_mask:
           _snake_case   	=      np.pad(
               processed_features['''attention_mask''']							,       (difference, 0)				)
         _snake_case   	=      ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
         _snake_case   	=      np.pad(
             __lowerCamelCase							,       __lowerCamelCase							,       '''constant'''							,       constant_values=self.padding_value				)
       else:
         raise ValueError('''Invalid padding strategy:''' + str(self.padding_side				)				)

     return processed_features



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	Union[Dict[str, np.ndarray], BatchFeature]							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	Optional[bool] = None							,       ):



     """simple docstring"""
     if not truncation:
       return processed_features
     elif truncation and max_length is None:
       raise ValueError('''When setting ``truncation=True``, make sure that ``max_length`` is defined.'''				)

     _snake_case   	=      processed_features[self.model_input_names[0]]

     # find `max_length` that fits `pad_to_multiple_of`
     if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
       _snake_case   	=      ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

     _snake_case   	=      len(__lowerCamelCase				) > max_length

     if needs_to_be_truncated:
       _snake_case   	=      processed_features[self.model_input_names[0]][:max_length]
       if "attention_mask" in processed_features:
         _snake_case   	=      processed_features['''attention_mask'''][:max_length]

     return processed_features





   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	str=False							,       __lowerCamelCase    :	Any=None				):



     """simple docstring"""

     # Get padding strategy
     if padding is not False:
       if padding is True:
         _snake_case   	=      PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
       elif not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
         _snake_case   	=      PaddingStrategy(__lowerCamelCase				)
       elif isinstance(__lowerCamelCase							,       __lowerCamelCase				):
         _snake_case   	=      padding
     else:
       _snake_case   	=      PaddingStrategy.DO_NOT_PAD

     # Set max length if needed
     if max_length is None:
       if padding_strategy == PaddingStrategy.MAX_LENGTH:
         raise ValueError(
             f"""When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"""				)

        # Test if we have a padding value
     if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
       raise ValueError(
           '''Asking to pad but the feature_extractor does not have a padding value. Please select a value to use'''
           ''' as `padding_value`. For example: `feature_extractor.padding_value = 0.0`.'''				)

     return padding_strategy











'''simple docstring'''



import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							_a				(			SCREAMING_SNAKE_CASE ,      unittest.TestCase ):





       '''simple docstring'''






       A						:				str        =						LongformerTokenizer
       A						:				List[str]        =						True
       A						:				Optional[int]        =						LongformerTokenizerFast
       A						:				Tuple        =						True







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             SCREAMING_SNAKE_CASE   :      Any    					=						[
                 'l',
                 'o',
                 'w',
                 'e',
                 'r',
                 's',
                 't',
                 'i',
                 'd',
                 'n',
                 '\u0120',
                 '\u0120l',
                 '\u0120n',
                 '\u0120lo',
                 '\u0120low',
                 'er',
                 '\u0120lowest',
                 '\u0120newer',
                 '\u0120wider',
                 '<unk>',
             ]
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						dict(zip(A,							range(len(A  )  )  )  )
             SCREAMING_SNAKE_CASE   :      str    					=						['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
             SCREAMING_SNAKE_CASE   :      Tuple    					=						{'unk_token': '<unk>'}

             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						os.path.join(self.tmpdirname,							VOCAB_FILES_NAMES['vocab_file']  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						os.path.join(self.tmpdirname,							VOCAB_FILES_NAMES['merges_file']  )
             with open(self.vocab_file,							'w',							encoding='utf-8'  ) as fp:
                   fp.write(json.dumps(A  ) + '\n'  )
             with open(self.merges_file,							'w',							encoding='utf-8'  ) as fp:
                   fp.write('\n'.join(A  )  )







       def        UpperCamelCase_							(      self,							**A  ):







             '''simple docstring'''







             kwargs.update(self.special_tokens_map  )
             return self.tokenizer_class.from_pretrained(self.tmpdirname,							**A  )







       def        UpperCamelCase_							(      self,							**A  ):







             '''simple docstring'''







             kwargs.update(self.special_tokens_map  )
             return self.rust_tokenizer_class.from_pretrained(self.tmpdirname,							**A  )







       def        UpperCamelCase_							(      self,							A  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      int    					=						'lower newer'
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						'lower newer'
             return input_text, output_text







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[Any]    					=						self.tokenizer_class(self.vocab_file,							self.merges_file,							**self.special_tokens_map  )
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						'lower newer'
             SCREAMING_SNAKE_CASE   :      List[str]    					=						['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer.tokenize(A  )  # , add_prefix_space=True)
             self.assertListEqual(A,							A  )

             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokens + [tokenizer.unk_token]
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(A  ),							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[Any]    					=						self.get_tokenizer()

             self.assertListEqual(tokenizer.encode('Hello world!',							add_special_tokens=A  ),							[0, 31_414, 232, 328, 2]  )
             self.assertListEqual(
                 tokenizer.encode('Hello world! cécé herlolip 418',							add_special_tokens=A  ),							[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2],							)







       @slow
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      str    					=						self.tokenizer_class.from_pretrained('allenai/longformer-base-4096'  )

             SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer.encode('sequence builders',							add_special_tokens=A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.encode('multi-sequence build',							add_special_tokens=A  )

             SCREAMING_SNAKE_CASE   :      int    					=						tokenizer.encode(
                 'sequence builders',							add_special_tokens=A,							add_prefix_space=A  )
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer.encode(
                 'sequence builders',							'multi-sequence build',							add_special_tokens=A,							add_prefix_space=A  )

             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.build_inputs_with_special_tokens(A  )
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer.build_inputs_with_special_tokens(A,							A  )

             assert encoded_sentence == encoded_text_from_decode
             assert encoded_pair == encoded_pair_from_decode







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						self.get_tokenizer()

             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						'Encode this sequence.'
             SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.byte_encoder[' '.encode('utf-8'  )[0]]

             # Testing encoder arguments
             SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.encode(A,							add_special_tokens=A,							add_prefix_space=A  )
             SCREAMING_SNAKE_CASE   :      Dict    					=						tokenizer.convert_ids_to_tokens(encoded[0]  )[0]
             self.assertNotEqual(A,							A  )

             SCREAMING_SNAKE_CASE   :      str    					=						tokenizer.encode(A,							add_special_tokens=A,							add_prefix_space=A  )
             SCREAMING_SNAKE_CASE   :      str    					=						tokenizer.convert_ids_to_tokens(encoded[0]  )[0]
             self.assertEqual(A,							A  )

             tokenizer.add_special_tokens({'bos_token': '<s>'}  )
             SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.encode(A,							add_special_tokens=A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.convert_ids_to_tokens(encoded[1]  )[0]
             self.assertNotEqual(A,							A  )

             # Testing spaces after special tokens
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						'<mask>'
             tokenizer.add_special_tokens(
                 {'mask_token': AddedToken(A,							lstrip=A,							rstrip=A  )}  )  # mask token has a left space
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer.convert_tokens_to_ids(A  )

             SCREAMING_SNAKE_CASE   :      List[str]    					=						'Encode <mask> sequence'
             SCREAMING_SNAKE_CASE   :      List[str]    					=						'Encode <mask>sequence'

             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer.encode(A  )
             SCREAMING_SNAKE_CASE   :      Tuple    					=						encoded.index(A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]  )[0]
             self.assertEqual(A,							A  )

             SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer.encode(A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						encoded.index(A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]  )[0]
             self.assertNotEqual(A,							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             pass







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
                         SCREAMING_SNAKE_CASE   :      Optional[int]    					=						self.rust_tokenizer_class.from_pretrained(A,							**A  )
                         SCREAMING_SNAKE_CASE   :      Tuple    					=						self.tokenizer_class.from_pretrained(A,							**A  )
                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						'A, <mask> AllenNLP sentence.'
                         SCREAMING_SNAKE_CASE   :      Any    					=						tokenizer_r.encode_plus(A,							add_special_tokens=A,							return_token_type_ids=A  )
                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer_p.encode_plus(A,							add_special_tokens=A,							return_token_type_ids=A  )

                         # token_type_ids should put 0 everywhere
                         self.assertEqual(sum(tokens_r['token_type_ids']  ),							sum(tokens_p['token_type_ids']  )  )

                         # attention_mask should put 1 everywhere, so sum over length should be 1
                         self.assertEqual(
                             sum(tokens_r['attention_mask']  ) / len(tokens_r['attention_mask']  ),							sum(tokens_p['attention_mask']  ) / len(tokens_p['attention_mask']  ),							)

                         SCREAMING_SNAKE_CASE   :      Dict    					=						tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']  )
                         SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']  )

                         # Rust correctly handles the space before the mask while python doesnt
                         self.assertSequenceEqual(tokens_p['input_ids'],							[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]  )
                         self.assertSequenceEqual(tokens_r['input_ids'],							[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]  )

                         self.assertSequenceEqual(
                             A,							['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']  )
                         self.assertSequenceEqual(
                             A,							['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             for trim_offsets, add_prefix_space in itertools.product([True, False],							repeat=2  ):
                   SCREAMING_SNAKE_CASE   :      List[Any]    					=						self.rust_tokenizer_class.from_pretrained(
                       self.tmpdirname,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )

                   SCREAMING_SNAKE_CASE   :      Tuple    					=						json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()  )
                   SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()  )

                   self.assertEqual(pre_tokenizer_state['add_prefix_space'],							A  )

                   self.assertEqual(post_processor_state['add_prefix_space'],							A  )
                   self.assertEqual(post_processor_state['trim_offsets'],							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
                         SCREAMING_SNAKE_CASE   :      str    					=						'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
                         SCREAMING_SNAKE_CASE   :      Tuple    					=						F"{text_of_1_token} {text_of_1_token}"

                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(len(A  ) + 1, len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(len(A  ) + 1, len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      List[str]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(len(A  ), len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Any    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(len(A  ), len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Any    					=						F" {text}"

                         # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                         #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
                         # )
                         # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
                         # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
                         # self.assertEqual(
                         #     encoding.offset_mapping[1],
                         #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
                         # )

                         SCREAMING_SNAKE_CASE   :      str    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(1, 1 + len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(1 + len(A  ) + 1, 1 + len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      str    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, 1 + len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(1 + len(A  ), 1 + len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, 1 + len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(1 + len(A  ), 1 + len(A  ) + 1 + len(A  )),							)


