
import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
   from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
   lowercase_ 				=			True
   from torch.cuda.amp import autocast


lowercase_ 				=			logging.getLogger(__name__)





@dataclass
class     __UpperCamelCase :






     """simple docstring"""







     lowerCAmelCase_							   =			field(
         metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}       )
     lowerCAmelCase_							   =			field(
         default=lowerCAmelCase__ ,						metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''} ,						)
     lowerCAmelCase_							   =			field(
         default=lowerCAmelCase__ ,						metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}       )
     lowerCAmelCase_							   =			field(
         default=lowerCAmelCase__ ,						metadata={'''help''': '''Whether to log verbose messages or not.'''} ,						)
     lowerCAmelCase_							   =			field(
         default=2.0 ,						metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}       )
     lowerCAmelCase_							   =			field(
         default=0.5 ,						metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}       )
     lowerCAmelCase_							   =			field(
         default=0.999995 ,						metadata={'''help''': '''Decay of gumbel temperature during training.'''}       )






def 							a__					(							snake_case       ,			snake_case							):





  """simple docstring"""
  logging.basicConfig(
      format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''       ,			datefmt='''%m/%d/%Y %H:%M:%S'''       ,			handlers=[logging.StreamHandler(sys.stdout							)]       ,			)
  __SCREAMING_SNAKE_CASE      :	int					=		logging.WARNING
  if model_args.verbose_logging:
    __SCREAMING_SNAKE_CASE      :	int					=		logging.DEBUG
  elif trainer_utils.is_main_process(training_args.local_rank							):
    __SCREAMING_SNAKE_CASE      :	Dict					=		logging.INFO
  logger.setLevel(snake_case							)





@dataclass
class     __UpperCamelCase :






     """simple docstring"""







     lowerCAmelCase_							   =			field(
         default=lowerCAmelCase__ ,						metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}       )
     lowerCAmelCase_							   =			field(
         default=lowerCAmelCase__ ,						metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}       )
     lowerCAmelCase_							   =			field(
         default='''train''' ,						metadata={
             '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
         } ,						)
     lowerCAmelCase_							   =			field(
         default='''validation''' ,						metadata={
             '''help''': (
                 '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
             )
         } ,						)
     lowerCAmelCase_							   =			field(
         default='''file''' ,						metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''} ,						)
     lowerCAmelCase_							   =			field(
         default=lowerCAmelCase__ ,						metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}       )
     lowerCAmelCase_							   =			field(
         default=1 ,						metadata={
             '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
         } ,						)
     lowerCAmelCase_							   =			field(
         default=lowerCAmelCase__ ,						metadata={'''help''': '''The number of processes to use for the preprocessing.'''} ,						)
     lowerCAmelCase_							   =			field(
         default=20.0 ,						metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}       )






@dataclass
class     __UpperCamelCase :






     """simple docstring"""







     lowerCAmelCase_							   =			42
     lowerCAmelCase_							   =			42
     lowerCAmelCase_							   =			"longest"
     lowerCAmelCase_							   =			None
     lowerCAmelCase_							   =			None



     def __call__(				self							:							List[Any]     ,						_A							:							List[Dict[str, Union[List[int], torch.Tensor]]] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Tuple					=		self.feature_extractor.pad(
           _A     ,						max_length=self.max_length     ,						padding=self.padding     ,						pad_to_multiple_of=self.pad_to_multiple_of     ,						return_tensors='''pt'''     ,						)
       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1] )

       __SCREAMING_SNAKE_CASE      :	Any					=		batch['''input_values'''].shape[0]

       # make sure that no loss is computed on padded inputs
       if batch["attention_mask"] is not None:
         # compute real output lengths according to convolution formula
         __SCREAMING_SNAKE_CASE      :	str					=		self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1 ) ).to(
             torch.long )

         __SCREAMING_SNAKE_CASE      :	Tuple					=		torch.zeros(
             (batch_size, mask_indices_seq_length)     ,						dtype=torch.long     ,						device=batch['''input_values'''].device )

         # these two operations makes sure that all values
         # before the output lengths indices are attended to
         __SCREAMING_SNAKE_CASE      :	int					=		1
         __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		attention_mask.flip([-1] ).cumsum(-1 ).flip([-1] ).bool()

       # sample randomly masked indices
       __SCREAMING_SNAKE_CASE      :	Dict					=		_compute_mask_indices(
           (batch_size, mask_indices_seq_length)     ,						self.model.config.mask_time_prob     ,						self.model.config.mask_time_length     ,						attention_mask=_A     ,						min_masks=2     ,						)

       return batch






class     __UpperCamelCase (							lowerCAmelCase__       ):






     """simple docstring"""







     def __init__(				self							:							str     ,						*_A							:							Dict     ,						_A							:							str=1     ,						_A							:							int=0     ,						_A							:							List[Any]=1.0     ,						**_A							:							str ):







       """simple docstring"""




       super().__init__(*_A     ,						**_A )
       __SCREAMING_SNAKE_CASE      :	List[str]					=		0
       __SCREAMING_SNAKE_CASE      :	List[Any]					=		max_gumbel_temp
       __SCREAMING_SNAKE_CASE      :	List[Any]					=		min_gumbel_temp
       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		gumbel_temp_decay




     def  UpperCAmelCase__							(				self							:							Optional[Any]     ,						_A							:							nn.Module     ,						_A							:							Dict[str, Union[torch.Tensor, Any]] ):







       """simple docstring"""




       model.train()
       __SCREAMING_SNAKE_CASE      :	Any					=		self._prepare_inputs(_A )

       if self.use_amp:
         with autocast():
           __SCREAMING_SNAKE_CASE      :	Dict					=		self.compute_loss(_A     ,						_A )
       else:
         __SCREAMING_SNAKE_CASE      :	Dict					=		self.compute_loss(_A     ,						_A )

       if self.args.n_gpu > 1 or self.deepspeed:
         if model.module.config.ctc_loss_reduction == "mean":
           __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		loss.mean()
         elif model.module.config.ctc_loss_reduction == "sum":
           __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		loss.sum() / (inputs['''mask_time_indices''']).sum()
         else:
           raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']''' )

       if self.args.gradient_accumulation_steps > 1:
         __SCREAMING_SNAKE_CASE      :	Dict					=		loss / self.args.gradient_accumulation_steps

       if self.use_amp:
         self.scaler.scale(_A ).backward()
       elif self.use_apex:
         with amp.scale_loss(_A     ,						self.optimizer ) as scaled_loss:
           scaled_loss.backward()
       elif self.deepspeed:
         self.deepspeed.backward(_A )
       else:
         loss.backward()

       self.num_update_step += 1
       # make sure gumbel softmax temperature is decayed
       if self.args.n_gpu > 1 or self.deepspeed:
         model.module.set_gumbel_temperature(
             max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,						self.min_gumbel_temp ) )
       else:
         model.set_gumbel_temperature(
             max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,						self.min_gumbel_temp ) )

       return loss.detach()






def 							a__					(							):





  """simple docstring"""
  # See all possible arguments in src/transformers/training_args.py
  # or by passing the --help flag to this script.
  # We now keep distinct sets of args, for a cleaner separation of concerns.

  __SCREAMING_SNAKE_CASE      :	List[str]					=		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)

  __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE      :	Any					=		parser.parse_args_into_dataclasses()
  configure_logger(snake_case       ,			snake_case							)

  # Downloading and loading a dataset from the hub.
  __SCREAMING_SNAKE_CASE      :	str					=		load_dataset(data_args.dataset_name       ,			data_args.dataset_config_name       ,			cache_dir=model_args.cache_dir							)

  if "validation" not in datasets.keys():
    # make sure only "validation" and "train" keys remain"
    __SCREAMING_SNAKE_CASE      :	str					=		DatasetDict()
    __SCREAMING_SNAKE_CASE      :	str					=		load_dataset(
        data_args.dataset_name       ,			data_args.dataset_config_name       ,			split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''       ,			cache_dir=model_args.cache_dir       ,			)
    __SCREAMING_SNAKE_CASE      :	str					=		load_dataset(
        data_args.dataset_name       ,			data_args.dataset_config_name       ,			split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''       ,			cache_dir=model_args.cache_dir       ,			)
  else:
    # make sure only "validation" and "train" keys remain"
    __SCREAMING_SNAKE_CASE      :	Tuple					=		DatasetDict()
    __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		load_dataset(
        data_args.dataset_name       ,			data_args.dataset_config_name       ,			split='''validation'''       ,			cache_dir=model_args.cache_dir       ,			)
    __SCREAMING_SNAKE_CASE      :	Optional[int]					=		load_dataset(
        data_args.dataset_name       ,			data_args.dataset_config_name       ,			split=F'''{data_args.train_split_name}'''       ,			cache_dir=model_args.cache_dir       ,			)

  # only normalized-inputs-training is supported
  __SCREAMING_SNAKE_CASE      :	Optional[int]					=		WavaVecaFeatureExtractor.from_pretrained(
      model_args.model_name_or_path       ,			cache_dir=model_args.cache_dir       ,			do_normalize=snake_case							)

  def prepare_dataset(snake_case							):
    # check that all files have the correct sampling rate
    __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE      :	Optional[Any]					=		librosa.load(batch[data_args.speech_file_column]       ,			sr=feature_extractor.sampling_rate							)
    return batch

  # load audio files into numpy arrays
  __SCREAMING_SNAKE_CASE      :	List[Any]					=		datasets.map(
      snake_case       ,			num_proc=data_args.preprocessing_num_workers       ,			remove_columns=datasets['''train'''].column_names							)

  # filter audio files that are too long
  __SCREAMING_SNAKE_CASE      :	Dict					=		vectorized_datasets.filter(
      lambda snake_case							: len(data['''speech''']							) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate							)							)

  def normalize(snake_case							):
    return feature_extractor(batch['''speech''']       ,			sampling_rate=feature_extractor.sampling_rate							)

  # normalize and transform to `BatchFeatures`
  __SCREAMING_SNAKE_CASE      :	int					=		vectorized_datasets.map(
      snake_case       ,			batched=snake_case       ,			num_proc=data_args.preprocessing_num_workers       ,			load_from_cache_file=not data_args.overwrite_cache       ,			remove_columns=vectorized_datasets['''train'''].column_names       ,			)

  # pretraining is only supported for "newer" stable layer norm architecture
  # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
  __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		WavaVecaConfig.from_pretrained(
      model_args.model_name_or_path       ,			cache_dir=model_args.cache_dir       ,			gradient_checkpointing=training_args.gradient_checkpointing       ,			)

  if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
    raise ValueError(
        '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
        ''' ``config.feat_extract_norm=\'layer\''''							)

  __SCREAMING_SNAKE_CASE      :	List[str]					=		WavaVecaForPreTraining(snake_case							)

  __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		DataCollatorForWavaVecaPretraining(model=snake_case       ,			feature_extractor=snake_case							)

  __SCREAMING_SNAKE_CASE      :	Dict					=		WavaVecaPreTrainer(
      model=snake_case       ,			data_collator=snake_case       ,			args=snake_case       ,			train_dataset=vectorized_datasets['''train''']       ,			eval_dataset=vectorized_datasets['''validation''']       ,			tokenizer=snake_case       ,			max_gumbel_temp=model_args.max_gumbel_temperature       ,			min_gumbel_temp=model_args.min_gumbel_temperature       ,			gumbel_temp_decay=model_args.gumbel_temperature_decay       ,			)
  trainer.train()


if __name__ == "__main__":
   main()








import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase					,      unittest.TestCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  LongformerTokenizer
  _lowerCamelCase    =  True
  _lowerCamelCase    =  LongformerTokenizerFast
  _lowerCamelCase    =  True


  def  lowerCamelCase       (      self		:			int							)						->	Optional[int]:


         """simple docstring"""
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         _UpperCAmelCase							=    [
             """l""",
             """o""",
             """w""",
             """e""",
             """r""",
             """s""",
             """t""",
             """i""",
             """d""",
             """n""",
             """\u0120""",
             """\u0120l""",
             """\u0120n""",
             """\u0120lo""",
             """\u0120low""",
             """er""",
             """\u0120lowest""",
             """\u0120newer""",
             """\u0120wider""",
             """<unk>""",
         ]
         _UpperCAmelCase							=    dict(zip(lowerCamelCase       ,				range(len(lowerCamelCase							)							)							)							)
         _UpperCAmelCase							=    ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
         _UpperCAmelCase							=    {"""unk_token""": """<unk>"""}

         _UpperCAmelCase							=    os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""vocab_file"""]							)
         _UpperCAmelCase							=    os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""merges_file"""]							)
         with open(self.vocab_file       ,				"""w"""       ,				encoding="""utf-8"""							) as fp:
                fp.write(json.dumps(lowerCamelCase							) + """\n"""							)
         with open(self.merges_file       ,				"""w"""       ,				encoding="""utf-8"""							) as fp:
                fp.write("""\n""".join(lowerCamelCase							)							)


  def  lowerCamelCase       (      self		:			List[Any]       ,				**lowerCamelCase		:			str							)						->	Tuple:


         """simple docstring"""
         kwargs.update(self.special_tokens_map							)
         return self.tokenizer_class.from_pretrained(self.tmpdirname       ,				**lowerCamelCase							)


  def  lowerCamelCase       (      self		:			int       ,				**lowerCamelCase		:			str							)						->	int:


         """simple docstring"""
         kwargs.update(self.special_tokens_map							)
         return self.rust_tokenizer_class.from_pretrained(self.tmpdirname       ,				**lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Union[str, Any]       ,				lowerCamelCase		:			Tuple							)						->	Optional[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    """lower newer"""
         _UpperCAmelCase							=    """lower newer"""
         return input_text, output_text


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    self.tokenizer_class(self.vocab_file       ,				self.merges_file       ,				**self.special_tokens_map							)
         _UpperCAmelCase							=    """lower newer"""
         _UpperCAmelCase							=    ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
         _UpperCAmelCase							=    tokenizer.tokenize(lowerCamelCase							)  # , add_prefix_space=True)
         self.assertListEqual(lowerCamelCase       ,				lowerCamelCase							)

         _UpperCAmelCase							=    tokens + [tokenizer.unk_token]
         _UpperCAmelCase							=    [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase							)       ,				lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Dict							)						->	Tuple:


         """simple docstring"""
         _UpperCAmelCase							=    self.get_tokenizer()

         self.assertListEqual(tokenizer.encode("""Hello world!"""       ,				add_special_tokens=lowerCamelCase							)       ,				[0, 3_1414, 232, 328, 2]							)
         self.assertListEqual(
             tokenizer.encode("""Hello world! cécé herlolip 418"""       ,				add_special_tokens=lowerCamelCase							)       ,				[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]       ,				)


  @slow
  def  lowerCamelCase       (      self		:			Dict							)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""							)

         _UpperCAmelCase							=    tokenizer.encode("""sequence builders"""       ,				add_special_tokens=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.encode("""multi-sequence build"""       ,				add_special_tokens=lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.encode(
             """sequence builders"""       ,				add_special_tokens=lowerCamelCase       ,				add_prefix_space=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.encode(
             """sequence builders"""       ,				"""multi-sequence build"""       ,				add_special_tokens=lowerCamelCase       ,				add_prefix_space=lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase       ,				lowerCamelCase							)

         assert encoded_sentence == encoded_text_from_decode
         assert encoded_pair == encoded_pair_from_decode


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    self.get_tokenizer()

         _UpperCAmelCase							=    """Encode this sequence."""
         _UpperCAmelCase							=    tokenizer.byte_encoder[""" """.encode("""utf-8"""							)[0]]

         # Testing encoder arguments
         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase       ,				add_special_tokens=lowerCamelCase       ,				add_prefix_space=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertNotEqual(lowerCamelCase       ,				lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase       ,				add_special_tokens=lowerCamelCase       ,				add_prefix_space=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertEqual(lowerCamelCase       ,				lowerCamelCase							)

         tokenizer.add_special_tokens({"""bos_token""": """<s>"""}							)
         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
         self.assertNotEqual(lowerCamelCase       ,				lowerCamelCase							)

         # Testing spaces after special tokens
         _UpperCAmelCase							=    """<mask>"""
         tokenizer.add_special_tokens(
             {"""mask_token""": AddedToken(lowerCamelCase       ,				lstrip=lowerCamelCase       ,				rstrip=lowerCamelCase							)}							)  # mask token has a left space
         _UpperCAmelCase							=    tokenizer.convert_tokens_to_ids(lowerCamelCase							)

         _UpperCAmelCase							=    """Encode <mask> sequence"""
         _UpperCAmelCase							=    """Encode <mask>sequence"""

         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase							)
         _UpperCAmelCase							=    encoded.index(lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertEqual(lowerCamelCase       ,				lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase							)
         _UpperCAmelCase							=    encoded.index(lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertNotEqual(lowerCamelCase       ,				lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	Union[str, Any]:


         """simple docstring"""
         pass


  def  lowerCamelCase       (      self		:			Any							)						->	Union[str, Any]:


         """simple docstring"""
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(lowerCamelCase       ,				**lowerCamelCase							)
                       _UpperCAmelCase							=    self.tokenizer_class.from_pretrained(lowerCamelCase       ,				**lowerCamelCase							)
                       _UpperCAmelCase							=    """A, <mask> AllenNLP sentence."""
                       _UpperCAmelCase							=    tokenizer_r.encode_plus(lowerCamelCase       ,				add_special_tokens=lowerCamelCase       ,				return_token_type_ids=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_p.encode_plus(lowerCamelCase       ,				add_special_tokens=lowerCamelCase       ,				return_token_type_ids=lowerCamelCase							)

                       # token_type_ids should put 0 everywhere
                       self.assertEqual(sum(tokens_r["""token_type_ids"""]							)       ,				sum(tokens_p["""token_type_ids"""]							)							)

                       # attention_mask should put 1 everywhere, so sum over length should be 1
                       self.assertEqual(
                           sum(tokens_r["""attention_mask"""]							) / len(tokens_r["""attention_mask"""]							)       ,				sum(tokens_p["""attention_mask"""]							) / len(tokens_p["""attention_mask"""]							)       ,				)

                       _UpperCAmelCase							=    tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]							)
                       _UpperCAmelCase							=    tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]							)

                       # Rust correctly handles the space before the mask while python doesnt
                       self.assertSequenceEqual(tokens_p["""input_ids"""]       ,				[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]							)
                       self.assertSequenceEqual(tokens_r["""input_ids"""]       ,				[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]							)

                       self.assertSequenceEqual(
                           lowerCamelCase       ,				["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)
                       self.assertSequenceEqual(
                           lowerCamelCase       ,				["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)


  def  lowerCamelCase       (      self		:			int							)						->	str:


         """simple docstring"""
         for trim_offsets, add_prefix_space in itertools.product([True, False]       ,				repeat=2							):
                _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                    self.tmpdirname       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)

                _UpperCAmelCase							=    json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
                _UpperCAmelCase							=    json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

                self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]       ,				lowerCamelCase							)

                self.assertEqual(post_processor_state["""add_prefix_space"""]       ,				lowerCamelCase							)
                self.assertEqual(post_processor_state["""trim_offsets"""]       ,				lowerCamelCase							)





  def  lowerCamelCase       (      self		:			Dict							)						->	Tuple:


         """simple docstring"""
         # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
         # `trim_offsets`
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
                       _UpperCAmelCase							=    """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                       _UpperCAmelCase							=    f"""{text_of_1_token} {text_of_1_token}"""

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(len(lowerCamelCase							) + 1, len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(len(lowerCamelCase							) + 1, len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(len(lowerCamelCase							), len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(len(lowerCamelCase							), len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    f""" {text}"""

                       # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                       #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
                       # )
                       # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
                       # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
                       # self.assertEqual(
                       #     encoding.offset_mapping[1],
                       #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
                       # )

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(1, 1 + len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(1 + len(lowerCamelCase							) + 1, 1 + len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, 1 + len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(1 + len(lowerCamelCase							), 1 + len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, 1 + len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(1 + len(lowerCamelCase							), 1 + len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)