






import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

A_		:		str		     =			[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

A_		:		Optional[int]		     =			[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__  )	->    List[str]:
  UpperCamelCase_: Dict   				=    {
      'word_embeddings.weight': 'word_embeddings.weight',
      'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
      'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
      'weight': 'ln_f.weight',
      'bias': 'ln_f.bias',
  }

  if key in layer_rename_map:
    return layer_rename_map[key]

  # Handle transformer blocks
  UpperCamelCase_: Tuple   				=    int(re.match(R'.*layer_(\d*).*'     ,   UpperCAmelCase__  )[1]  )
  layer_number -= 3
  return F'''h.{layer_number}.''' + key

def   snake_case (UpperCAmelCase__  )	->    List[str]:
  if dtype == torch.bool:
    return 1 / 8
  UpperCamelCase_: Optional[Any]   				=    re.search(R'[^\d](\d+)$'     ,   str(UpperCAmelCase__  )  )
  if bit_search is None:
    raise ValueError(F'''`dtype` is not a valid dtype: {dtype}.'''  )
  UpperCamelCase_: List[Any]   				=    int(bit_search.groups()[0]  )
  return bit_size // 8

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__  )	->    Any:
  # Construct model
  if bloom_config_file == "":
    UpperCamelCase_: List[str]   				=    BloomConfig()
  else:
    UpperCamelCase_: List[str]   				=    BloomConfig.from_json_file(UpperCAmelCase__  )

  if shard_model:
    UpperCamelCase_: str   				=    os.listdir(UpperCAmelCase__  )
    UpperCamelCase_: List[str]   				=    sorted(filter(lambda UpperCAmelCase__  : s.startswith('layer'  ) and "model_00" in s     ,   UpperCAmelCase__  )  )

    UpperCamelCase_: Optional[int]   				=    {'weight_map': {}, 'metadata': {}}
    UpperCamelCase_: str   				=    0

    UpperCamelCase_: Optional[Any]   				=    None

    UpperCamelCase_: int   				=    BloomConfig()

    for j, file in enumerate(UpperCAmelCase__  ):
      print('Processing file: {}'.format(UpperCAmelCase__  )  )
      UpperCamelCase_: Tuple   				=    None

      for i in range(UpperCAmelCase__  ):
        # load all TP files
        UpperCamelCase_: List[Any]   				=    file.replace('model_00'     ,   F'''model_0{i}'''  )
        UpperCamelCase_: List[str]   				=    torch.load(os.path.join(UpperCAmelCase__     ,   UpperCAmelCase__  )     ,   map_location='cpu'  )

        # Rename keys in the transformers names
        UpperCamelCase_: Optional[int]   				=    list(temp.keys()  )
        for key in keys:
          UpperCamelCase_: List[Any]   				=    temp.pop(UpperCAmelCase__  )

        if tensors is None:
          UpperCamelCase_: Dict   				=    temp
        else:
          for key in tensors.keys():
            if any(key.endswith(UpperCAmelCase__  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
              # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
              tensors[key] += temp[key]
            else:
              # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
              UpperCamelCase_: List[Any]   				=    1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
              # We concatenate these weights accross TP ranks
              UpperCamelCase_: Dict   				=    torch.cat([tensors[key], temp[key]]     ,   dim=UpperCAmelCase__  )

            # Divide by the number of TP the weights we want to average
      for key in tensors.keys():
        if any(key.endswith(UpperCAmelCase__  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
          UpperCamelCase_: Optional[int]   				=    tensors[key] / pretraining_tp
      torch.save(
          UpperCAmelCase__     ,   os.path.join(
              UpperCAmelCase__     ,   'pytorch_model_{}-of-{}.bin'.format(str(j + 1  ).zfill(5  )     ,   str(len(UpperCAmelCase__  )  ).zfill(5  )  )     ,   )     ,   )

      for key in tensors.keys():
        UpperCamelCase_: int   				=    tensors[key]
        total_size += value.numel() * get_dtype_size(value.dtype  )
        if key not in index_dict["weight_map"]:
          UpperCamelCase_: Dict   				=    'pytorch_model_{}-of-{}.bin'.format(
              str(j + 1  ).zfill(5  )     ,   str(len(UpperCAmelCase__  )  ).zfill(5  )  )

    UpperCamelCase_: Union[str, Any]   				=    BloomConfig()
    UpperCamelCase_: Any   				=    pytorch_dump_folder_path + '/' + CONFIG_NAME
    UpperCamelCase_: Optional[int]   				=    total_size
    with open(UpperCAmelCase__     ,   'w'     ,   encoding='utf-8'  ) as f:
      f.write(config.to_json_string()  )
    with open(os.path.join(UpperCAmelCase__     ,   WEIGHTS_NAME + '.index.json'  )     ,   'w'     ,   encoding='utf-8'  ) as f:
      UpperCamelCase_: Tuple   				=    json.dumps(UpperCAmelCase__     ,   indent=2     ,   sort_keys=UpperCAmelCase__  ) + '\n'
      f.write(UpperCAmelCase__  )
  else:
    UpperCamelCase_: Optional[Any]   				=    BloomModel(UpperCAmelCase__  )

    UpperCamelCase_: Tuple   				=    os.listdir(UpperCAmelCase__  )
    UpperCamelCase_: Tuple   				=    sorted(filter(lambda UpperCAmelCase__  : s.startswith('layer'  ) and "model_00" in s     ,   UpperCAmelCase__  )  )

    UpperCamelCase_: Tuple   				=    None
    for i, file in enumerate(UpperCAmelCase__  ):
      UpperCamelCase_: Union[str, Any]   				=    None
      for i in range(UpperCAmelCase__  ):
        # load all TP files
        UpperCamelCase_: Any   				=    file.replace('model_00'     ,   F'''model_0{i}'''  )
        UpperCamelCase_: Union[str, Any]   				=    torch.load(os.path.join(UpperCAmelCase__     ,   UpperCAmelCase__  )     ,   map_location='cpu'  )

        # Rename keys in the transformers names
        UpperCamelCase_: Dict   				=    list(temp.keys()  )
        for key in keys:
          UpperCamelCase_: Any   				=    temp.pop(UpperCAmelCase__  )

        if tensors is None:
          UpperCamelCase_: Any   				=    temp
        else:
          for key in tensors.keys():
            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
            if any(key.endswith(UpperCAmelCase__  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
              tensors[key] += temp[key]
            else:
              # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
              UpperCamelCase_: int   				=    1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
              # We concatenate these weights accross TP ranks
              UpperCamelCase_: Optional[int]   				=    torch.cat([tensors[key], temp[key]]     ,   dim=UpperCAmelCase__  )

            # Divide by the number of TP the weights we want to average
      for key in tensors.keys():
        if any(key.endswith(UpperCAmelCase__  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
          UpperCamelCase_: Tuple   				=    tensors[key] / pretraining_tp

      UpperCamelCase_: Any   				=    model.load_state_dict(UpperCAmelCase__     ,   strict=UpperCAmelCase__  )
      assert not other_keys.unexpected_keys, F'''The keys {other_keys.unexpected_keys} are unexpected'''
      if missing_keys is None:
        UpperCamelCase_: Any   				=    set(other_keys.missing_keys  )
      else:
        UpperCamelCase_: int   				=    missing_keys.intersection(set(other_keys.missing_keys  )  )

    assert not missing_keys, F'''The keys {missing_keys} are missing'''

    # Save pytorch-model
    os.makedirs(UpperCAmelCase__     ,   exist_ok=UpperCAmelCase__  )
    UpperCamelCase_: Optional[Any]   				=    pytorch_dump_folder_path + '/' + WEIGHTS_NAME
    UpperCamelCase_: str   				=    pytorch_dump_folder_path + '/' + CONFIG_NAME
    print(F'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''  )
    if config.torch_dtype is not None:
      UpperCamelCase_: Tuple   				=    model.to(config.torch_dtype  )
    torch.save(model.state_dict()     ,   UpperCAmelCase__  )
    print(F'''Save configuration file to {pytorch_config_dump_path}'''  )
    with open(UpperCAmelCase__     ,   'w'     ,   encoding='utf-8'  ) as f:
      f.write(config.to_json_string()  )


if __name__ == "__main__":
   A_		:		Dict		     =			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--bloom_checkpoint_path',
       default=None,
       type=str,
       required=True,
       help='Path to the Megatron-LM checkpoint path.',
   )
   parser.add_argument(
       '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
   )
   parser.add_argument(
       '--bloom_config_file',
       default='',
       type=str,
       help=(
           'An optional config json file corresponding to the pre-trained model. \n'
           'This specifies the model architecture.'
       ),
   )
   parser.add_argument(
       '--shard_model',
       action='store_true',
       help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
   )
   parser.add_argument(
       '--pretraining_tp',
       default=4,
       type=int,
       help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
   )
   A_		:		Any		     =			parser.parse_args()
   convert_bloom_checkpoint_to_pytorch(
       args.bloom_checkpoint_path,
       args.bloom_config_file,
       args.pytorch_dump_folder_path,
       args.shard_model,
       args.pretraining_tp,
   )







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class     _lowerCAmelCase(	UpperCAmelCase_							,     unittest.TestCase ):





       """simple docstring"""







       a    :		Optional[int]									=LongformerTokenizer
       a    :		Optional[int]									=True
       a    :		Tuple									=LongformerTokenizerFast
       a    :		Dict									=True





       def       _a   ( self							):
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         UpperCamelCase_: int   				=    [
             'l',
             'o',
             'w',
             'e',
             'r',
             's',
             't',
             'i',
             'd',
             'n',
             '\u0120',
             '\u0120l',
             '\u0120n',
             '\u0120lo',
             '\u0120low',
             'er',
             '\u0120lowest',
             '\u0120newer',
             '\u0120wider',
             '<unk>',
         ]
         UpperCamelCase_: Optional[Any]   				=    dict(zip(_lowerCamelCase  ,      range(len(_lowerCamelCase							)							)							)							)
         UpperCamelCase_: Any   				=    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
         UpperCamelCase_: Tuple   				=    {'unk_token': '<unk>'}

         UpperCamelCase_: str   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['vocab_file']							)
         UpperCamelCase_: Tuple   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['merges_file']							)
         with open(self.vocab_file  ,      'w'  ,      encoding='utf-8'							) as fp:
           fp.write(json.dumps(_lowerCamelCase							) + '\n'							)
         with open(self.merges_file  ,      'w'  ,      encoding='utf-8'							) as fp:
           fp.write('\n'.join(_lowerCamelCase							)							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         kwargs.update(self.special_tokens_map							)
         return self.tokenizer_class.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         kwargs.update(self.special_tokens_map							)
         return self.rust_tokenizer_class.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: Union[str, Any]   				=    'lower newer'
         UpperCamelCase_: Optional[Any]   				=    'lower newer'
         return input_text, output_text





       def       _a   ( self							):
         UpperCamelCase_: int   				=    self.tokenizer_class(self.vocab_file  ,      self.merges_file  ,      **self.special_tokens_map							)
         UpperCamelCase_: Any   				=    'lower newer'
         UpperCamelCase_: Any   				=    ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
         UpperCamelCase_: str   				=    tokenizer.tokenize(_lowerCamelCase							)  # , add_prefix_space=True)
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: Dict   				=    tokens + [tokenizer.unk_token]
         UpperCamelCase_: int   				=    [0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase							)  ,      _lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Tuple   				=    self.get_tokenizer()

         self.assertListEqual(tokenizer.encode('Hello world!'  ,      add_special_tokens=_lowerCamelCase							)  ,      [0, 3_1_4_1_4, 2_3_2, 3_2_8, 2]							)
         self.assertListEqual(
             tokenizer.encode('Hello world! cécé herlolip 418'  ,      add_special_tokens=_lowerCamelCase							)  ,      [0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]  ,      )





       @slow
       def       _a   ( self							):
         UpperCamelCase_: int   				=    self.tokenizer_class.from_pretrained('allenai/longformer-base-4096'							)

         UpperCamelCase_: Dict   				=    tokenizer.encode('sequence builders'  ,      add_special_tokens=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.encode('multi-sequence build'  ,      add_special_tokens=_lowerCamelCase							)

         UpperCamelCase_: Union[str, Any]   				=    tokenizer.encode(
             'sequence builders'  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.encode(
             'sequence builders'  ,      'multi-sequence build'  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)

         UpperCamelCase_: int   				=    tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							)
         UpperCamelCase_: Tuple   				=    tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  ,      _lowerCamelCase							)

         assert encoded_sentence == encoded_text_from_decode
         assert encoded_pair == encoded_pair_from_decode





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    self.get_tokenizer()

         UpperCamelCase_: Optional[int]   				=    'Encode this sequence.'
         UpperCamelCase_: List[Any]   				=    tokenizer.byte_encoder[' '.encode('utf-8'							)[0]]

         # Testing encoder arguments
         UpperCamelCase_: Dict   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: List[str]   				=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: List[Any]   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: int   				=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         tokenizer.add_special_tokens({'bos_token': '<s>'}							)
         UpperCamelCase_: Optional[Any]   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         # Testing spaces after special tokens
         UpperCamelCase_: List[Any]   				=    '<mask>'
         tokenizer.add_special_tokens(
             {'mask_token': AddedToken(_lowerCamelCase  ,      lstrip=_lowerCamelCase  ,      rstrip=_lowerCamelCase							)}							)  # mask token has a left space
         UpperCamelCase_: List[Any]   				=    tokenizer.convert_tokens_to_ids(_lowerCamelCase							)

         UpperCamelCase_: Dict   				=    'Encode <mask> sequence'
         UpperCamelCase_: Dict   				=    'Encode <mask>sequence'

         UpperCamelCase_: Any   				=    tokenizer.encode(_lowerCamelCase							)
         UpperCamelCase_: Optional[Any]   				=    encoded.index(_lowerCamelCase							)
         UpperCamelCase_: Optional[Any]   				=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: List[str]   				=    tokenizer.encode(_lowerCamelCase							)
         UpperCamelCase_: List[Any]   				=    encoded.index(_lowerCamelCase							)
         UpperCamelCase_: Optional[int]   				=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)





       def       _a   ( self							):
         pass





       def       _a   ( self							):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Any   				=    self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: Tuple   				=    self.tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: List[Any]   				=    'A, <mask> AllenNLP sentence.'
             UpperCamelCase_: int   				=    tokenizer_r.encode_plus(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      return_token_type_ids=_lowerCamelCase							)
             UpperCamelCase_: Any   				=    tokenizer_p.encode_plus(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      return_token_type_ids=_lowerCamelCase							)

             # token_type_ids should put 0 everywhere
             self.assertEqual(sum(tokens_r['token_type_ids']							)  ,      sum(tokens_p['token_type_ids']							)							)

             # attention_mask should put 1 everywhere, so sum over length should be 1
             self.assertEqual(
                 sum(tokens_r['attention_mask']							) / len(tokens_r['attention_mask']							)  ,      sum(tokens_p['attention_mask']							) / len(tokens_p['attention_mask']							)  ,      )

             UpperCamelCase_: List[str]   				=    tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']							)
             UpperCamelCase_: str   				=    tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']							)

             # Rust correctly handles the space before the mask while python doesnt
             self.assertSequenceEqual(tokens_p['input_ids']  ,      [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]							)
             self.assertSequenceEqual(tokens_r['input_ids']  ,      [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]							)

             self.assertSequenceEqual(
                 _lowerCamelCase  ,      ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']							)
             self.assertSequenceEqual(
                 _lowerCamelCase  ,      ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']							)





       def       _a   ( self							):
         for trim_offsets, add_prefix_space in itertools.product([True, False]  ,      repeat=2							):
           UpperCamelCase_: Optional[Any]   				=    self.rust_tokenizer_class.from_pretrained(
               self.tmpdirname  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)

           UpperCamelCase_: Union[str, Any]   				=    json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
           UpperCamelCase_: Optional[int]   				=    json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

           self.assertEqual(pre_tokenizer_state['add_prefix_space']  ,      _lowerCamelCase							)

           self.assertEqual(post_processor_state['add_prefix_space']  ,      _lowerCamelCase							)
           self.assertEqual(post_processor_state['trim_offsets']  ,      _lowerCamelCase							)





       def       _a   ( self							):
         # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
         # `trim_offsets`
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Optional[int]   				=    'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
             UpperCamelCase_: Union[str, Any]   				=    f'''{text_of_1_token} {text_of_1_token}'''

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: str   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							) + 1, len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: str   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							) + 1, len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[Any]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Dict   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							), len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: List[str]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Optional[Any]   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							), len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[int]   				=    f''' {text}'''

             # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
             #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
             # )
             # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
             # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
             # self.assertEqual(
             #     encoding.offset_mapping[1],
             #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
             # )

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Tuple   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (1, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							) + 1, 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Dict   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Dict   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							), 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Union[str, Any]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Union[str, Any]   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							), 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )