






import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length

def   snake_case (UpperCAmelCase__  )	->    Optional[Any]:
  UpperCamelCase_: List[Any]   				=    int(__lowerCAmelCase  )
  UpperCamelCase_				,UpperCamelCase_				,UpperCamelCase_: Optional[Any]   				=    t // 3_6_0_0, (t // 6_0) % 6_0, t % 6_0
  return F'''{h}:{m:02d}:{s:02d}''' if h != 0 else F'''{m:02d}:{s:02d}'''

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__=3_0_0  )	->    str:
  # docstyle-ignore
  return F'''\n    <div>\n      {prefix}\n      <progress value=\'{value}\' max=\'{total}\' style=\'width:{width}px; height:20px; vertical-align: middle;\'></progress>\n      {label}\n    </div>\n    '''

def   snake_case (UpperCAmelCase__  )	->    Tuple:
  UpperCamelCase_: Optional[int]   				=    '<table border="1" class="dataframe">\n'
  html_code += """  <thead>\n <tr style="text-align: left;">\n"""
  for i in items[0]:
    html_code += F'''      <th>{i}</th>\n'''
  html_code += "    </tr>\n  </thead>\n  <tbody>\n"
  for line in items[1:]:
    html_code += "    <tr>\n"
    for elt in line:
      UpperCamelCase_: List[str]   				=    F'''{elt:.6f}''' if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase  ) else str(__lowerCAmelCase  )
      html_code += F'''      <td>{elt}</td>\n'''
    html_code += "    </tr>\n"
  html_code += "  </tbody>\n</table><p>"
  return html_code




class     _lowerCAmelCase:





       """simple docstring"""







       a    :		Optional[int]									=5
       a    :		str									=0.2





       def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase = None  ,      _lowerCamelCase = True  ,      _lowerCamelCase = None  ,      _lowerCamelCase = 3_0_0  ,      ):
         UpperCamelCase_: int   				=    total
         UpperCamelCase_: Any   				=    '' if prefix is None else prefix
         UpperCamelCase_: Optional[Any]   				=    leave
         UpperCamelCase_: int   				=    parent
         UpperCamelCase_: List[Any]   				=    width
         UpperCamelCase_: str   				=    None
         UpperCamelCase_: Union[str, Any]   				=    None
         UpperCamelCase_: Optional[Any]   				=    None





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase = False  ,      _lowerCamelCase = None							):
         UpperCamelCase_: Optional[Any]   				=    value
         if comment is not None:
           UpperCamelCase_: List[Any]   				=    comment
         if self.last_value is None:
           UpperCamelCase_: Union[str, Any]   				=    time.time()
           UpperCamelCase_: Dict   				=    value
           UpperCamelCase_: int   				=    None
           UpperCamelCase_: int   				=    self.warmup
           UpperCamelCase_: Optional[int]   				=    1
           self.update_bar(__snake_case							)
         elif value <= self.last_value and not force_update:
           return
         elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for  ,      self.total							):
           if self.first_calls > 0:
             self.first_calls -= 1
           UpperCamelCase_: int   				=    time.time()
           UpperCamelCase_: Any   				=    current_time - self.start_time
           # We could have value = self.start_value if the update is called twixe with the same start value.
           if value > self.start_value:
             UpperCamelCase_: Optional[int]   				=    self.elapsed_time / (value - self.start_value)
           else:
             UpperCamelCase_: int   				=    None
           if value >= self.total:
             UpperCamelCase_: List[str]   				=    self.total
             UpperCamelCase_: Optional[int]   				=    None
             if not self.leave:
               self.close()
           elif self.average_time_per_item is not None:
             UpperCamelCase_: Union[str, Any]   				=    self.average_time_per_item * (self.total - value)
           self.update_bar(__snake_case							)
           UpperCamelCase_: Union[str, Any]   				=    value
           UpperCamelCase_: int   				=    current_time
           if self.average_time_per_item is None:
             UpperCamelCase_: List[Any]   				=    1
           else:
             UpperCamelCase_: List[str]   				=    max(int(self.update_every / self.average_time_per_item							)  ,      1							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase=None							):
         UpperCamelCase_: Dict   				=    ' ' * (len(str(self.total							)							) - len(str(__snake_case							)							)) + str(__snake_case							)
         if self.elapsed_time is None:
           UpperCamelCase_: Tuple   				=    f'''[{spaced_value}/{self.total} : < :'''
         elif self.predicted_remaining is None:
           UpperCamelCase_: Union[str, Any]   				=    f'''[{spaced_value}/{self.total} {format_time(self.elapsed_time							)}'''
         else:
           UpperCamelCase_: int   				=    (
               f'''[{spaced_value}/{self.total} {format_time(self.elapsed_time							)} <'''
               f''' {format_time(self.predicted_remaining							)}'''
           )
           self.label += f''', {1/self.average_time_per_item:.2f} it/s'''
         self.label += "]" if self.comment is None or len(self.comment							) == 0 else f''', {self.comment}]'''
         self.display()





       def       _a   ( self							):
         UpperCamelCase_: Optional[Any]   				=    html_progress_bar(self.value  ,      self.total  ,      self.prefix  ,      self.label  ,      self.width							)
         if self.parent is not None:
           # If this is a child bar, the parent will take care of the display.
           self.parent.display()
           return
         if self.output is None:
           UpperCamelCase_: Dict   				=    disp.display(disp.HTML(self.html_code							)  ,      display_id=__snake_case							)
         else:
           self.output.update(disp.HTML(self.html_code							)							)





       def       _a   ( self							):
         if self.parent is None and self.output is not None:
           self.output.update(disp.HTML(''							)							)







class     _lowerCAmelCase(	lowerCamelCase_ ):





       """simple docstring"""







       def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase=None							):
         super().__init__(__snake_case							)
         UpperCamelCase_: List[Any]   				=    None if column_names is None else [column_names]
         UpperCamelCase_: Optional[Any]   				=    None





       def       _a   ( self							):
         UpperCamelCase_: str   				=    html_progress_bar(self.value  ,      self.total  ,      self.prefix  ,      self.label  ,      self.width							)
         if self.inner_table is not None:
           self.html_code += text_to_html_table(self.inner_table							)
         if self.child_bar is not None:
           self.html_code += self.child_bar.html_code
         if self.output is None:
           UpperCamelCase_: List[str]   				=    disp.display(disp.HTML(self.html_code							)  ,      display_id=__snake_case							)
         else:
           self.output.update(disp.HTML(self.html_code							)							)





       def       _a   ( self  ,      _lowerCamelCase							):
         if self.inner_table is None:
           UpperCamelCase_: List[str]   				=    [list(values.keys()							), list(values.values()							)]
         else:
           UpperCamelCase_: List[Any]   				=    self.inner_table[0]
           if len(self.inner_table							) == 1:
             # We give a chance to update the column names at the first iteration
             for key in values.keys():
               if key not in columns:
                 columns.append(__snake_case							)
             UpperCamelCase_: Tuple   				=    columns
           self.inner_table.append([values[c] for c in columns]							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase=None  ,      _lowerCamelCase=3_0_0							):
         UpperCamelCase_: str   				=    NotebookProgressBar(__snake_case  ,      prefix=__snake_case  ,      parent=self  ,      width=__snake_case							)
         return self.child_bar





       def       _a   ( self							):
         UpperCamelCase_: int   				=    None
         self.display()







class     _lowerCAmelCase(	lowerCamelCase_ ):





       """simple docstring"""







       def __init__( self							):
         UpperCamelCase_: Dict   				=    None
         UpperCamelCase_: str   				=    None
         UpperCamelCase_: Union[str, Any]   				=    False





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      **_lowerCamelCase							):
         UpperCamelCase_: Optional[int]   				=    'Epoch' if args.evaluation_strategy == IntervalStrategy.EPOCH else 'Step'
         UpperCamelCase_: Optional[Any]   				=    0
         UpperCamelCase_: Optional[int]   				=    0
         UpperCamelCase_: Dict   				=    [self.first_column] + ['Training Loss']
         if args.evaluation_strategy != IntervalStrategy.NO:
           column_names.append('Validation Loss'							)
         UpperCamelCase_: str   				=    NotebookTrainingTracker(state.max_steps  ,      __snake_case							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      **_lowerCamelCase							):
         UpperCamelCase_: Tuple   				=    int(state.epoch							) if int(state.epoch							) == state.epoch else f'''{state.epoch:.2f}'''
         self.training_tracker.update(
             state.global_step + 1  ,      comment=f'''Epoch {epoch}/{state.num_train_epochs}'''  ,      force_update=self._force_next_update  ,      )
         UpperCamelCase_: List[Any]   				=    False





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase=None  ,      **_lowerCamelCase							):
         if not has_length(__snake_case							):
           return
         if self.prediction_bar is None:
           if self.training_tracker is not None:
             UpperCamelCase_: Any   				=    self.training_tracker.add_child(len(__snake_case							)							)
           else:
             UpperCamelCase_: int   				=    NotebookProgressBar(len(__snake_case							)							)
           self.prediction_bar.update(1							)
         else:
           self.prediction_bar.update(self.prediction_bar.value + 1							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      **_lowerCamelCase							):
         if self.prediction_bar is not None:
           self.prediction_bar.close()
         UpperCamelCase_: int   				=    None





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase=None  ,      **_lowerCamelCase							):
         # Only for when there is no evaluation
         if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
           UpperCamelCase_: Tuple   				=    {'Training Loss': logs['loss']}
           # First column is necessarily Step sine we're not in epoch eval strategy
           UpperCamelCase_: Optional[int]   				=    state.global_step
           self.training_tracker.write_line(__snake_case							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase=None  ,      **_lowerCamelCase							):
         if self.training_tracker is not None:
           UpperCamelCase_: Dict   				=    {'Training Loss': 'No log', 'Validation Loss': 'No log'}
           for log in reversed(state.log_history							):
             if "loss" in log:
               UpperCamelCase_: List[str]   				=    log['loss']
               break

           if self.first_column == "Epoch":
             UpperCamelCase_: Optional[int]   				=    int(state.epoch							)
           else:
             UpperCamelCase_: List[Any]   				=    state.global_step
           UpperCamelCase_: Any   				=    'eval'
           for k in metrics:
             if k.endswith('_loss'							):
               UpperCamelCase_: List[Any]   				=    re.sub(r'\_loss$'  ,      ''  ,      __snake_case							)
           UpperCamelCase_: Any   				=    metrics.pop('total_flos'  ,      __snake_case							)
           UpperCamelCase_: List[Any]   				=    metrics.pop('epoch'  ,      __snake_case							)
           UpperCamelCase_: str   				=    metrics.pop(f'''{metric_key_prefix}_runtime'''  ,      __snake_case							)
           UpperCamelCase_: int   				=    metrics.pop(f'''{metric_key_prefix}_samples_per_second'''  ,      __snake_case							)
           UpperCamelCase_: Optional[int]   				=    metrics.pop(f'''{metric_key_prefix}_steps_per_second'''  ,      __snake_case							)
           UpperCamelCase_: str   				=    metrics.pop(f'''{metric_key_prefix}_jit_compilation_time'''  ,      __snake_case							)
           for k, v in metrics.items():
             if k == f'''{metric_key_prefix}_loss''':
               UpperCamelCase_: Tuple   				=    v
             else:
               UpperCamelCase_: Tuple   				=    k.split('_'							)
               UpperCamelCase_: Union[str, Any]   				=    ' '.join([part.capitalize() for part in splits[1:]]							)
               UpperCamelCase_: Union[str, Any]   				=    v
           self.training_tracker.write_line(__snake_case							)
           self.training_tracker.remove_child()
           UpperCamelCase_: int   				=    None
           # Evaluation takes a long time so we should force the next update.
           UpperCamelCase_: List[str]   				=    True





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      **_lowerCamelCase							):
         self.training_tracker.update(
             state.global_step  ,      comment=f'''Epoch {int(state.epoch							)}/{state.num_train_epochs}'''  ,      force_update=__snake_case							)
         UpperCamelCase_: Dict   				=    None







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class     _lowerCAmelCase(	UpperCAmelCase_							,     unittest.TestCase ):





       """simple docstring"""







       a    :		Optional[int]									=LongformerTokenizer
       a    :		Optional[int]									=True
       a    :		Tuple									=LongformerTokenizerFast
       a    :		Dict									=True





       def       _a   ( self							):
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         UpperCamelCase_: int   				=    [
             'l',
             'o',
             'w',
             'e',
             'r',
             's',
             't',
             'i',
             'd',
             'n',
             '\u0120',
             '\u0120l',
             '\u0120n',
             '\u0120lo',
             '\u0120low',
             'er',
             '\u0120lowest',
             '\u0120newer',
             '\u0120wider',
             '<unk>',
         ]
         UpperCamelCase_: Optional[Any]   				=    dict(zip(_lowerCamelCase  ,      range(len(_lowerCamelCase							)							)							)							)
         UpperCamelCase_: Any   				=    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
         UpperCamelCase_: Tuple   				=    {'unk_token': '<unk>'}

         UpperCamelCase_: str   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['vocab_file']							)
         UpperCamelCase_: Tuple   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['merges_file']							)
         with open(self.vocab_file  ,      'w'  ,      encoding='utf-8'							) as fp:
           fp.write(json.dumps(_lowerCamelCase							) + '\n'							)
         with open(self.merges_file  ,      'w'  ,      encoding='utf-8'							) as fp:
           fp.write('\n'.join(_lowerCamelCase							)							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         kwargs.update(self.special_tokens_map							)
         return self.tokenizer_class.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         kwargs.update(self.special_tokens_map							)
         return self.rust_tokenizer_class.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: Union[str, Any]   				=    'lower newer'
         UpperCamelCase_: Optional[Any]   				=    'lower newer'
         return input_text, output_text





       def       _a   ( self							):
         UpperCamelCase_: int   				=    self.tokenizer_class(self.vocab_file  ,      self.merges_file  ,      **self.special_tokens_map							)
         UpperCamelCase_: Any   				=    'lower newer'
         UpperCamelCase_: Any   				=    ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
         UpperCamelCase_: str   				=    tokenizer.tokenize(_lowerCamelCase							)  # , add_prefix_space=True)
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: Dict   				=    tokens + [tokenizer.unk_token]
         UpperCamelCase_: int   				=    [0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase							)  ,      _lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Tuple   				=    self.get_tokenizer()

         self.assertListEqual(tokenizer.encode('Hello world!'  ,      add_special_tokens=_lowerCamelCase							)  ,      [0, 3_1_4_1_4, 2_3_2, 3_2_8, 2]							)
         self.assertListEqual(
             tokenizer.encode('Hello world! cécé herlolip 418'  ,      add_special_tokens=_lowerCamelCase							)  ,      [0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]  ,      )





       @slow
       def       _a   ( self							):
         UpperCamelCase_: int   				=    self.tokenizer_class.from_pretrained('allenai/longformer-base-4096'							)

         UpperCamelCase_: Dict   				=    tokenizer.encode('sequence builders'  ,      add_special_tokens=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.encode('multi-sequence build'  ,      add_special_tokens=_lowerCamelCase							)

         UpperCamelCase_: Union[str, Any]   				=    tokenizer.encode(
             'sequence builders'  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.encode(
             'sequence builders'  ,      'multi-sequence build'  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)

         UpperCamelCase_: int   				=    tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							)
         UpperCamelCase_: Tuple   				=    tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  ,      _lowerCamelCase							)

         assert encoded_sentence == encoded_text_from_decode
         assert encoded_pair == encoded_pair_from_decode





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    self.get_tokenizer()

         UpperCamelCase_: Optional[int]   				=    'Encode this sequence.'
         UpperCamelCase_: List[Any]   				=    tokenizer.byte_encoder[' '.encode('utf-8'							)[0]]

         # Testing encoder arguments
         UpperCamelCase_: Dict   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: List[str]   				=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: List[Any]   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: int   				=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         tokenizer.add_special_tokens({'bos_token': '<s>'}							)
         UpperCamelCase_: Optional[Any]   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         # Testing spaces after special tokens
         UpperCamelCase_: List[Any]   				=    '<mask>'
         tokenizer.add_special_tokens(
             {'mask_token': AddedToken(_lowerCamelCase  ,      lstrip=_lowerCamelCase  ,      rstrip=_lowerCamelCase							)}							)  # mask token has a left space
         UpperCamelCase_: List[Any]   				=    tokenizer.convert_tokens_to_ids(_lowerCamelCase							)

         UpperCamelCase_: Dict   				=    'Encode <mask> sequence'
         UpperCamelCase_: Dict   				=    'Encode <mask>sequence'

         UpperCamelCase_: Any   				=    tokenizer.encode(_lowerCamelCase							)
         UpperCamelCase_: Optional[Any]   				=    encoded.index(_lowerCamelCase							)
         UpperCamelCase_: Optional[Any]   				=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: List[str]   				=    tokenizer.encode(_lowerCamelCase							)
         UpperCamelCase_: List[Any]   				=    encoded.index(_lowerCamelCase							)
         UpperCamelCase_: Optional[int]   				=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)





       def       _a   ( self							):
         pass





       def       _a   ( self							):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Any   				=    self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: Tuple   				=    self.tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: List[Any]   				=    'A, <mask> AllenNLP sentence.'
             UpperCamelCase_: int   				=    tokenizer_r.encode_plus(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      return_token_type_ids=_lowerCamelCase							)
             UpperCamelCase_: Any   				=    tokenizer_p.encode_plus(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      return_token_type_ids=_lowerCamelCase							)

             # token_type_ids should put 0 everywhere
             self.assertEqual(sum(tokens_r['token_type_ids']							)  ,      sum(tokens_p['token_type_ids']							)							)

             # attention_mask should put 1 everywhere, so sum over length should be 1
             self.assertEqual(
                 sum(tokens_r['attention_mask']							) / len(tokens_r['attention_mask']							)  ,      sum(tokens_p['attention_mask']							) / len(tokens_p['attention_mask']							)  ,      )

             UpperCamelCase_: List[str]   				=    tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']							)
             UpperCamelCase_: str   				=    tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']							)

             # Rust correctly handles the space before the mask while python doesnt
             self.assertSequenceEqual(tokens_p['input_ids']  ,      [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]							)
             self.assertSequenceEqual(tokens_r['input_ids']  ,      [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]							)

             self.assertSequenceEqual(
                 _lowerCamelCase  ,      ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']							)
             self.assertSequenceEqual(
                 _lowerCamelCase  ,      ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']							)





       def       _a   ( self							):
         for trim_offsets, add_prefix_space in itertools.product([True, False]  ,      repeat=2							):
           UpperCamelCase_: Optional[Any]   				=    self.rust_tokenizer_class.from_pretrained(
               self.tmpdirname  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)

           UpperCamelCase_: Union[str, Any]   				=    json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
           UpperCamelCase_: Optional[int]   				=    json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

           self.assertEqual(pre_tokenizer_state['add_prefix_space']  ,      _lowerCamelCase							)

           self.assertEqual(post_processor_state['add_prefix_space']  ,      _lowerCamelCase							)
           self.assertEqual(post_processor_state['trim_offsets']  ,      _lowerCamelCase							)





       def       _a   ( self							):
         # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
         # `trim_offsets`
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Optional[int]   				=    'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
             UpperCamelCase_: Union[str, Any]   				=    f'''{text_of_1_token} {text_of_1_token}'''

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: str   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							) + 1, len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: str   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							) + 1, len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[Any]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Dict   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							), len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: List[str]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Optional[Any]   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							), len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[int]   				=    f''' {text}'''

             # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
             #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
             # )
             # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
             # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
             # self.assertEqual(
             #     encoding.offset_mapping[1],
             #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
             # )

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Tuple   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (1, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							) + 1, 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Dict   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Dict   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							), 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Union[str, Any]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Union[str, Any]   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							), 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )