






import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       FlaubertForMultipleChoice,
       FlaubertForQuestionAnswering,
       FlaubertForQuestionAnsweringSimple,
       FlaubertForSequenceClassification,
       FlaubertForTokenClassification,
       FlaubertModel,
       FlaubertWithLMHeadModel,
   )
   from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class     _lowerCAmelCase(	UpperCAmelCase_ ):





       """simple docstring"""







       def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase=1_3  ,      _lowerCamelCase=7  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=False  ,      _lowerCamelCase=False  ,      _lowerCamelCase=False  ,      _lowerCamelCase=2  ,      _lowerCamelCase=9_9  ,      _lowerCamelCase=0  ,      _lowerCamelCase=3_2  ,      _lowerCamelCase=5  ,      _lowerCamelCase=4  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=5_1_2  ,      _lowerCamelCase=1_2  ,      _lowerCamelCase=2  ,      _lowerCamelCase=0.0_2  ,      _lowerCamelCase=3  ,      _lowerCamelCase=4  ,      _lowerCamelCase="last"  ,      _lowerCamelCase=None  ,      _lowerCamelCase=None  ,      ):
         UpperCamelCase_: Dict   				=    parent
         UpperCamelCase_: List[str]   				=    batch_size
         UpperCamelCase_: List[Any]   				=    seq_length
         UpperCamelCase_: Dict   				=    is_training
         UpperCamelCase_: List[Any]   				=    use_input_lengths
         UpperCamelCase_: Union[str, Any]   				=    use_token_type_ids
         UpperCamelCase_: str   				=    use_labels
         UpperCamelCase_: Dict   				=    gelu_activation
         UpperCamelCase_: Optional[int]   				=    sinusoidal_embeddings
         UpperCamelCase_: Dict   				=    causal
         UpperCamelCase_: List[Any]   				=    asm
         UpperCamelCase_: int   				=    n_langs
         UpperCamelCase_: str   				=    vocab_size
         UpperCamelCase_: Optional[Any]   				=    n_special
         UpperCamelCase_: List[str]   				=    hidden_size
         UpperCamelCase_: Union[str, Any]   				=    num_hidden_layers
         UpperCamelCase_: List[str]   				=    num_attention_heads
         UpperCamelCase_: Dict   				=    hidden_dropout_prob
         UpperCamelCase_: List[str]   				=    attention_probs_dropout_prob
         UpperCamelCase_: str   				=    max_position_embeddings
         UpperCamelCase_: Any   				=    type_vocab_size
         UpperCamelCase_: Any   				=    type_sequence_label_size
         UpperCamelCase_: Tuple   				=    initializer_range
         UpperCamelCase_: List[str]   				=    num_labels
         UpperCamelCase_: Optional[int]   				=    num_choices
         UpperCamelCase_: str   				=    summary_type
         UpperCamelCase_: str   				=    use_proj
         UpperCamelCase_: Union[str, Any]   				=    scope





       def       _a   ( self							):
         UpperCamelCase_: Tuple   				=    ids_tensor([self.batch_size, self.seq_length]  ,      self.vocab_size							)
         UpperCamelCase_: Optional[Any]   				=    random_attention_mask([self.batch_size, self.seq_length]							)

         UpperCamelCase_: Tuple   				=    None
         if self.use_input_lengths:
           UpperCamelCase_: Optional[Any]   				=    (
               ids_tensor([self.batch_size]  ,      vocab_size=2							) + self.seq_length - 2
           )  # small variation of seq_length

         UpperCamelCase_: str   				=    None
         if self.use_token_type_ids:
           UpperCamelCase_: Optional[Any]   				=    ids_tensor([self.batch_size, self.seq_length]  ,      self.n_langs							)

         UpperCamelCase_: Optional[int]   				=    None
         UpperCamelCase_: Optional[int]   				=    None
         UpperCamelCase_: List[Any]   				=    None
         if self.use_labels:
           UpperCamelCase_: List[str]   				=    ids_tensor([self.batch_size]  ,      self.type_sequence_label_size							)
           UpperCamelCase_: List[str]   				=    ids_tensor([self.batch_size, self.seq_length]  ,      self.num_labels							)
           UpperCamelCase_: Tuple   				=    ids_tensor([self.batch_size]  ,      2							).float()
           UpperCamelCase_: str   				=    ids_tensor([self.batch_size]  ,      self.num_choices							)

         UpperCamelCase_: Union[str, Any]   				=    self.get_config()

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )





       def       _a   ( self							):
         return FlaubertConfig(
             vocab_size=self.vocab_size  ,      n_special=self.n_special  ,      emb_dim=self.hidden_size  ,      n_layers=self.num_hidden_layers  ,      n_heads=self.num_attention_heads  ,      dropout=self.hidden_dropout_prob  ,      attention_dropout=self.attention_probs_dropout_prob  ,      gelu_activation=self.gelu_activation  ,      sinusoidal_embeddings=self.sinusoidal_embeddings  ,      asm=self.asm  ,      causal=self.causal  ,      n_langs=self.n_langs  ,      max_position_embeddings=self.max_position_embeddings  ,      initializer_range=self.initializer_range  ,      summary_type=self.summary_type  ,      use_proj=self.use_proj  ,      )





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         UpperCamelCase_: str   				=    FlaubertModel(config=_lowerCamelCase							)
         model.to(_lowerCamelCase							)
         model.eval()
         UpperCamelCase_: int   				=    model(_lowerCamelCase  ,      lengths=_lowerCamelCase  ,      langs=_lowerCamelCase							)
         UpperCamelCase_: List[Any]   				=    model(_lowerCamelCase  ,      langs=_lowerCamelCase							)
         UpperCamelCase_: Optional[Any]   				=    model(_lowerCamelCase							)
         self.parent.assertEqual(result.last_hidden_state.shape  ,      (self.batch_size, self.seq_length, self.hidden_size)							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         UpperCamelCase_: Tuple   				=    FlaubertWithLMHeadModel(_lowerCamelCase							)
         model.to(_lowerCamelCase							)
         model.eval()

         UpperCamelCase_: List[str]   				=    model(_lowerCamelCase  ,      token_type_ids=_lowerCamelCase  ,      labels=_lowerCamelCase							)
         self.parent.assertEqual(result.loss.shape  ,      ()							)
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.vocab_size)							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         UpperCamelCase_: str   				=    FlaubertForQuestionAnsweringSimple(_lowerCamelCase							)
         model.to(_lowerCamelCase							)
         model.eval()

         UpperCamelCase_: str   				=    model(_lowerCamelCase							)

         UpperCamelCase_: Union[str, Any]   				=    model(_lowerCamelCase  ,      start_positions=_lowerCamelCase  ,      end_positions=_lowerCamelCase							)
         self.parent.assertEqual(result.start_logits.shape  ,      (self.batch_size, self.seq_length)							)
         self.parent.assertEqual(result.end_logits.shape  ,      (self.batch_size, self.seq_length)							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         UpperCamelCase_: str   				=    FlaubertForQuestionAnswering(_lowerCamelCase							)
         model.to(_lowerCamelCase							)
         model.eval()

         UpperCamelCase_: Tuple   				=    model(_lowerCamelCase							)

         UpperCamelCase_: Optional[Any]   				=    model(
             _lowerCamelCase  ,      start_positions=_lowerCamelCase  ,      end_positions=_lowerCamelCase  ,      cls_index=_lowerCamelCase  ,      is_impossible=_lowerCamelCase  ,      p_mask=_lowerCamelCase  ,      )

         UpperCamelCase_: Any   				=    model(
             _lowerCamelCase  ,      start_positions=_lowerCamelCase  ,      end_positions=_lowerCamelCase  ,      cls_index=_lowerCamelCase  ,      is_impossible=_lowerCamelCase  ,      )

         ((UpperCamelCase_)				,): int   				=    result_with_labels.to_tuple()

         UpperCamelCase_: str   				=    model(_lowerCamelCase  ,      start_positions=_lowerCamelCase  ,      end_positions=_lowerCamelCase							)

         ((UpperCamelCase_)				,): int   				=    result_with_labels.to_tuple()

         self.parent.assertEqual(result_with_labels.loss.shape  ,      ()							)
         self.parent.assertEqual(result.start_top_log_probs.shape  ,      (self.batch_size, model.config.start_n_top)							)
         self.parent.assertEqual(result.start_top_index.shape  ,      (self.batch_size, model.config.start_n_top)							)
         self.parent.assertEqual(
             result.end_top_log_probs.shape  ,      (self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
         self.parent.assertEqual(
             result.end_top_index.shape  ,      (self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
         self.parent.assertEqual(result.cls_logits.shape  ,      (self.batch_size,)							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         UpperCamelCase_: int   				=    FlaubertForSequenceClassification(_lowerCamelCase							)
         model.to(_lowerCamelCase							)
         model.eval()

         UpperCamelCase_: Any   				=    model(_lowerCamelCase							)
         UpperCamelCase_: str   				=    model(_lowerCamelCase  ,      labels=_lowerCamelCase							)

         self.parent.assertEqual(result.loss.shape  ,      ()							)
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.type_sequence_label_size)							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         UpperCamelCase_: List[Any]   				=    self.num_labels
         UpperCamelCase_: List[Any]   				=    FlaubertForTokenClassification(_lowerCamelCase							)
         model.to(_lowerCamelCase							)
         model.eval()

         UpperCamelCase_: Union[str, Any]   				=    model(_lowerCamelCase  ,      attention_mask=_lowerCamelCase  ,      labels=_lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.num_labels)							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         UpperCamelCase_: int   				=    self.num_choices
         UpperCamelCase_: int   				=    FlaubertForMultipleChoice(config=_lowerCamelCase							)
         model.to(_lowerCamelCase							)
         model.eval()
         UpperCamelCase_: str   				=    input_ids.unsqueeze(1							).expand(-1  ,      self.num_choices  ,      -1							).contiguous()
         UpperCamelCase_: List[str]   				=    token_type_ids.unsqueeze(1							).expand(-1  ,      self.num_choices  ,      -1							).contiguous()
         UpperCamelCase_: int   				=    input_mask.unsqueeze(1							).expand(-1  ,      self.num_choices  ,      -1							).contiguous()
         UpperCamelCase_: str   				=    model(
             _lowerCamelCase  ,      attention_mask=_lowerCamelCase  ,      token_type_ids=_lowerCamelCase  ,      labels=_lowerCamelCase  ,      )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.num_choices)							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    self.prepare_config_and_inputs()
         (
             (
             UpperCamelCase_
         )				,(
             UpperCamelCase_
         )				,(
             UpperCamelCase_
         )				,(
             UpperCamelCase_
         )				,(
             UpperCamelCase_
         )				,(
             UpperCamelCase_
         )				,(
             UpperCamelCase_
         )				,(
             UpperCamelCase_
         )				,(
             UpperCamelCase_
         )				,
         ): List[Any]   				=    config_and_inputs
         UpperCamelCase_: Any   				=    {
             'input_ids': input_ids,
             'token_type_ids': token_type_ids,
             'lengths': input_lengths,
             'attention_mask': input_mask,
         }
         return config, inputs_dict







@require_torch
class     _lowerCAmelCase(	UpperCAmelCase_							,     UpperCAmelCase_							,     unittest.TestCase ):





       """simple docstring"""







       a    :		Optional[int]									=(
           (
               FlaubertModel,
               FlaubertWithLMHeadModel,
               FlaubertForQuestionAnswering,
               FlaubertForQuestionAnsweringSimple,
               FlaubertForSequenceClassification,
               FlaubertForTokenClassification,
               FlaubertForMultipleChoice,
           )
           if is_torch_available()
           else ()
       )
       a    :		List[str]									=(
           {
               '''feature-extraction''': FlaubertModel,
               '''fill-mask''': FlaubertWithLMHeadModel,
               '''question-answering''': FlaubertForQuestionAnsweringSimple,
               '''text-classification''': FlaubertForSequenceClassification,
               '''token-classification''': FlaubertForTokenClassification,
               '''zero-shot''': FlaubertForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase							):
         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith('Fast'							)
         ):
           # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
           # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
           # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
           return True

         return False





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase=False							):
         UpperCamelCase_: Tuple   				=    super()._prepare_for_class(_lowerCamelCase  ,      _lowerCamelCase  ,      return_labels=_lowerCamelCase							)

         if return_labels:
           if model_class.__name__ == "FlaubertForQuestionAnswering":
             UpperCamelCase_: Dict   				=    torch.zeros(
                 self.model_tester.batch_size  ,      dtype=torch.long  ,      device=_lowerCamelCase							)
             UpperCamelCase_: List[Any]   				=    torch.zeros(
                 self.model_tester.batch_size  ,      dtype=torch.long  ,      device=_lowerCamelCase							)

         return inputs_dict





       def       _a   ( self							):
         UpperCamelCase_: str   				=    FlaubertModelTester(self							)
         UpperCamelCase_: List[str]   				=    ConfigTester(self  ,      config_class=_lowerCamelCase  ,      emb_dim=3_7							)





       def       _a   ( self							):
         self.config_tester.run_common_tests()





       def       _a   ( self							):
         UpperCamelCase_: Dict   				=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_model(*_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Tuple   				=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_lm_head(*_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Tuple   				=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_simple_qa(*_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_qa(*_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[Any]   				=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_sequence_classif(*_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_token_classif(*_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: int   				=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_multiple_choice(*_lowerCamelCase							)





       @slow
       def       _a   ( self							):
         for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           UpperCamelCase_: Optional[int]   				=    FlaubertModel.from_pretrained(_lowerCamelCase							)
           self.assertIsNotNone(_lowerCamelCase							)





       @slow
       @require_torch_gpu
       def       _a   ( self							):
         UpperCamelCase_				,UpperCamelCase_: str   				=    self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
           # FlauBertForMultipleChoice behaves incorrectly in JIT environments.
           if model_class == FlaubertForMultipleChoice:
             return

           UpperCamelCase_: str   				=    True
           UpperCamelCase_: Union[str, Any]   				=    model_class(config=_lowerCamelCase							)

           UpperCamelCase_: Optional[Any]   				=    self._prepare_for_class(_lowerCamelCase  ,      _lowerCamelCase							)
           UpperCamelCase_: int   				=    torch.jit.trace(
               _lowerCamelCase  ,      (inputs_dict['input_ids'].to('cpu'							), inputs_dict['attention_mask'].to('cpu'							))							)

           with tempfile.TemporaryDirectory() as tmp:
             torch.jit.save(_lowerCamelCase  ,      os.path.join(_lowerCamelCase  ,      'traced_model.pt'							)							)
             UpperCamelCase_: Optional[int]   				=    torch.jit.load(os.path.join(_lowerCamelCase  ,      'traced_model.pt'							)  ,      map_location=_lowerCamelCase							)
             loaded(inputs_dict['input_ids'].to(_lowerCamelCase							)  ,      inputs_dict['attention_mask'].to(_lowerCamelCase							)							)







@require_torch
class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       @slow
       def       _a   ( self							):
         UpperCamelCase_: str   				=    FlaubertModel.from_pretrained('flaubert/flaubert_base_cased'							)
         UpperCamelCase_: str   				=    torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]							)
         with torch.no_grad():
           UpperCamelCase_: List[str]   				=    model(_lowerCamelCase							)[0]
         UpperCamelCase_: List[Any]   				=    torch.Size((1, 1_1, 7_6_8)							)
         self.assertEqual(output.shape  ,      _lowerCamelCase							)
         UpperCamelCase_: List[str]   				=    torch.tensor(
             [[[-2.6_2_5_1, -1.4_2_9_8, -0.0_2_2_7], [-2.8_5_1_0, -1.6_3_8_7, 0.2_2_5_8], [-2.8_1_1_4, -1.1_8_3_2, -0.3_0_6_6]]]							)

         self.assertTrue(torch.allclose(output[:, :3, :3]  ,      _lowerCamelCase  ,      atol=1e-4							)							)







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class     _lowerCAmelCase(	UpperCAmelCase_							,     unittest.TestCase ):





       """simple docstring"""







       a    :		Optional[int]									=LongformerTokenizer
       a    :		Optional[int]									=True
       a    :		Tuple									=LongformerTokenizerFast
       a    :		Dict									=True





       def       _a   ( self							):
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         UpperCamelCase_: int   				=    [
             'l',
             'o',
             'w',
             'e',
             'r',
             's',
             't',
             'i',
             'd',
             'n',
             '\u0120',
             '\u0120l',
             '\u0120n',
             '\u0120lo',
             '\u0120low',
             'er',
             '\u0120lowest',
             '\u0120newer',
             '\u0120wider',
             '<unk>',
         ]
         UpperCamelCase_: Optional[Any]   				=    dict(zip(_lowerCamelCase  ,      range(len(_lowerCamelCase							)							)							)							)
         UpperCamelCase_: Any   				=    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
         UpperCamelCase_: Tuple   				=    {'unk_token': '<unk>'}

         UpperCamelCase_: str   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['vocab_file']							)
         UpperCamelCase_: Tuple   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['merges_file']							)
         with open(self.vocab_file  ,      'w'  ,      encoding='utf-8'							) as fp:
           fp.write(json.dumps(_lowerCamelCase							) + '\n'							)
         with open(self.merges_file  ,      'w'  ,      encoding='utf-8'							) as fp:
           fp.write('\n'.join(_lowerCamelCase							)							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         kwargs.update(self.special_tokens_map							)
         return self.tokenizer_class.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         kwargs.update(self.special_tokens_map							)
         return self.rust_tokenizer_class.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: Union[str, Any]   				=    'lower newer'
         UpperCamelCase_: Optional[Any]   				=    'lower newer'
         return input_text, output_text





       def       _a   ( self							):
         UpperCamelCase_: int   				=    self.tokenizer_class(self.vocab_file  ,      self.merges_file  ,      **self.special_tokens_map							)
         UpperCamelCase_: Any   				=    'lower newer'
         UpperCamelCase_: Any   				=    ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
         UpperCamelCase_: str   				=    tokenizer.tokenize(_lowerCamelCase							)  # , add_prefix_space=True)
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: Dict   				=    tokens + [tokenizer.unk_token]
         UpperCamelCase_: int   				=    [0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase							)  ,      _lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Tuple   				=    self.get_tokenizer()

         self.assertListEqual(tokenizer.encode('Hello world!'  ,      add_special_tokens=_lowerCamelCase							)  ,      [0, 3_1_4_1_4, 2_3_2, 3_2_8, 2]							)
         self.assertListEqual(
             tokenizer.encode('Hello world! cécé herlolip 418'  ,      add_special_tokens=_lowerCamelCase							)  ,      [0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]  ,      )





       @slow
       def       _a   ( self							):
         UpperCamelCase_: int   				=    self.tokenizer_class.from_pretrained('allenai/longformer-base-4096'							)

         UpperCamelCase_: Dict   				=    tokenizer.encode('sequence builders'  ,      add_special_tokens=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.encode('multi-sequence build'  ,      add_special_tokens=_lowerCamelCase							)

         UpperCamelCase_: Union[str, Any]   				=    tokenizer.encode(
             'sequence builders'  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.encode(
             'sequence builders'  ,      'multi-sequence build'  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)

         UpperCamelCase_: int   				=    tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							)
         UpperCamelCase_: Tuple   				=    tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  ,      _lowerCamelCase							)

         assert encoded_sentence == encoded_text_from_decode
         assert encoded_pair == encoded_pair_from_decode





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    self.get_tokenizer()

         UpperCamelCase_: Optional[int]   				=    'Encode this sequence.'
         UpperCamelCase_: List[Any]   				=    tokenizer.byte_encoder[' '.encode('utf-8'							)[0]]

         # Testing encoder arguments
         UpperCamelCase_: Dict   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: List[str]   				=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: List[Any]   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase							)
         UpperCamelCase_: int   				=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         tokenizer.add_special_tokens({'bos_token': '<s>'}							)
         UpperCamelCase_: Optional[Any]   				=    tokenizer.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
         UpperCamelCase_: Any   				=    tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         # Testing spaces after special tokens
         UpperCamelCase_: List[Any]   				=    '<mask>'
         tokenizer.add_special_tokens(
             {'mask_token': AddedToken(_lowerCamelCase  ,      lstrip=_lowerCamelCase  ,      rstrip=_lowerCamelCase							)}							)  # mask token has a left space
         UpperCamelCase_: List[Any]   				=    tokenizer.convert_tokens_to_ids(_lowerCamelCase							)

         UpperCamelCase_: Dict   				=    'Encode <mask> sequence'
         UpperCamelCase_: Dict   				=    'Encode <mask>sequence'

         UpperCamelCase_: Any   				=    tokenizer.encode(_lowerCamelCase							)
         UpperCamelCase_: Optional[Any]   				=    encoded.index(_lowerCamelCase							)
         UpperCamelCase_: Optional[Any]   				=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: List[str]   				=    tokenizer.encode(_lowerCamelCase							)
         UpperCamelCase_: List[Any]   				=    encoded.index(_lowerCamelCase							)
         UpperCamelCase_: Optional[int]   				=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertNotEqual(_lowerCamelCase  ,      _lowerCamelCase							)





       def       _a   ( self							):
         pass





       def       _a   ( self							):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Any   				=    self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: Tuple   				=    self.tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: List[Any]   				=    'A, <mask> AllenNLP sentence.'
             UpperCamelCase_: int   				=    tokenizer_r.encode_plus(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      return_token_type_ids=_lowerCamelCase							)
             UpperCamelCase_: Any   				=    tokenizer_p.encode_plus(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      return_token_type_ids=_lowerCamelCase							)

             # token_type_ids should put 0 everywhere
             self.assertEqual(sum(tokens_r['token_type_ids']							)  ,      sum(tokens_p['token_type_ids']							)							)

             # attention_mask should put 1 everywhere, so sum over length should be 1
             self.assertEqual(
                 sum(tokens_r['attention_mask']							) / len(tokens_r['attention_mask']							)  ,      sum(tokens_p['attention_mask']							) / len(tokens_p['attention_mask']							)  ,      )

             UpperCamelCase_: List[str]   				=    tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']							)
             UpperCamelCase_: str   				=    tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']							)

             # Rust correctly handles the space before the mask while python doesnt
             self.assertSequenceEqual(tokens_p['input_ids']  ,      [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]							)
             self.assertSequenceEqual(tokens_r['input_ids']  ,      [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]							)

             self.assertSequenceEqual(
                 _lowerCamelCase  ,      ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']							)
             self.assertSequenceEqual(
                 _lowerCamelCase  ,      ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']							)





       def       _a   ( self							):
         for trim_offsets, add_prefix_space in itertools.product([True, False]  ,      repeat=2							):
           UpperCamelCase_: Optional[Any]   				=    self.rust_tokenizer_class.from_pretrained(
               self.tmpdirname  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)

           UpperCamelCase_: Union[str, Any]   				=    json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
           UpperCamelCase_: Optional[int]   				=    json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

           self.assertEqual(pre_tokenizer_state['add_prefix_space']  ,      _lowerCamelCase							)

           self.assertEqual(post_processor_state['add_prefix_space']  ,      _lowerCamelCase							)
           self.assertEqual(post_processor_state['trim_offsets']  ,      _lowerCamelCase							)





       def       _a   ( self							):
         # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
         # `trim_offsets`
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Optional[int]   				=    'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
             UpperCamelCase_: Union[str, Any]   				=    f'''{text_of_1_token} {text_of_1_token}'''

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: str   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							) + 1, len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: str   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							) + 1, len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[Any]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Dict   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							), len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: List[str]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Optional[Any]   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							), len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Optional[int]   				=    f''' {text}'''

             # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
             #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
             # )
             # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
             # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
             # self.assertEqual(
             #     encoding.offset_mapping[1],
             #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
             # )

             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Tuple   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (1, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							) + 1, 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Dict   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Dict   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							), 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: Union[str, Any]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      add_prefix_space=_lowerCamelCase  ,      trim_offsets=_lowerCamelCase							)
             UpperCamelCase_: Union[str, Any]   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							), 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )