





import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
          MobileBertModel,
      )
class     SCREAMING_SNAKE_CASE       :


   def __init__(   self      :    List[str]      ,					a      :    List[Any]      ,					a      :    List[str]=13      ,					a      :    List[Any]=7      ,					a      :    List[Any]=True      ,					a      :    Optional[int]=True      ,					a      :    str=True      ,					a      :    Tuple=True      ,					a      :    Optional[int]=99      ,					a      :    Tuple=64      ,					a      :    List[Any]=32      ,					a      :    Optional[Any]=5      ,					a      :    int=4      ,					a      :    Any=37      ,					a      :    Dict="gelu"      ,					a      :    Optional[int]=0.1      ,					a      :    List[str]=0.1      ,					a      :    Union[str, Any]=512      ,					a      :    Union[str, Any]=16      ,					a      :    Optional[int]=2      ,					a      :    Dict=0.02      ,					a      :    str=3      ,					a      :    List[Any]=4      ,					a      :    Tuple=None      ,					)->       Any:





     """simple docstring"""







     lowercase__           =  parent
     lowercase__           =  batch_size
     lowercase__           =  seq_length
     lowercase__           =  is_training
     lowercase__           =  use_input_mask
     lowercase__           =  use_token_type_ids
     lowercase__           =  use_labels
     lowercase__           =  vocab_size
     lowercase__           =  hidden_size
     lowercase__           =  embedding_size
     lowercase__           =  num_hidden_layers
     lowercase__           =  num_attention_heads
     lowercase__           =  intermediate_size
     lowercase__           =  hidden_act
     lowercase__           =  hidden_dropout_prob
     lowercase__           =  attention_probs_dropout_prob
     lowercase__           =  max_position_embeddings
     lowercase__           =  type_vocab_size
     lowercase__           =  type_sequence_label_size
     lowercase__           =  initializer_range
     lowercase__           =  num_labels
     lowercase__           =  num_choices
     lowercase__           =  scope


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       int:





     """simple docstring"""







     lowercase__           =  ids_tensor([self.batch_size, self.seq_length]      ,					self.vocab_size	)

     lowercase__           =  None
     if self.use_input_mask:
       lowercase__           =  random_attention_mask([self.batch_size, self.seq_length]	)

     lowercase__           =  None
     if self.use_token_type_ids:
       lowercase__           =  ids_tensor([self.batch_size, self.seq_length]      ,					self.type_vocab_size	)

     lowercase__           =  None
     lowercase__           =  None
     lowercase__           =  None
     if self.use_labels:
       lowercase__           =  ids_tensor([self.batch_size]      ,					self.type_sequence_label_size	)
       lowercase__           =  ids_tensor([self.batch_size, self.seq_length]      ,					self.num_labels	)
       lowercase__           =  ids_tensor([self.batch_size]      ,					self.num_choices	)

     lowercase__           =  self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]	)->       Union[str, Any]:





     """simple docstring"""







     return MobileBertConfig(
         vocab_size=self.vocab_size      ,					hidden_size=self.hidden_size      ,					num_hidden_layers=self.num_hidden_layers      ,					num_attention_heads=self.num_attention_heads      ,					intermediate_size=self.intermediate_size      ,					embedding_size=self.embedding_size      ,					hidden_act=self.hidden_act      ,					hidden_dropout_prob=self.hidden_dropout_prob      ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,					max_position_embeddings=self.max_position_embeddings      ,					type_vocab_size=self.type_vocab_size      ,					is_decoder=a      ,					initializer_range=self.initializer_range      ,					)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    str      ,					a      :    Union[str, Any]      ,					a      :    List[str]      ,					a      :    Dict      ,					a      :    Any      ,					a      :    Union[str, Any]      ,					a      :    List[Any]      ,					a      :    Tuple	)->       List[Any]:





     """simple docstring"""







     lowercase__           =  MobileBertModel(config=a	)
     model.to(a	)
     model.eval()
     lowercase__           =  model(a      ,					attention_mask=a      ,					token_type_ids=a	)
     lowercase__           =  model(a      ,					token_type_ids=a	)
     lowercase__           =  model(a	)

     self.parent.assertEqual(result.last_hidden_state.shape      ,					(self.batch_size, self.seq_length, self.hidden_size)	)
     self.parent.assertEqual(result.pooler_output.shape      ,					(self.batch_size, self.hidden_size)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int      ,					a      :    Dict      ,					a      :    Dict      ,					a      :    str      ,					a      :    List[Any]      ,					a      :    Tuple      ,					a      :    Tuple      ,					a      :    List[Any]	)->       Dict:





     """simple docstring"""







     lowercase__           =  MobileBertForMaskedLM(config=a	)
     model.to(a	)
     model.eval()
     lowercase__           =  model(a      ,					attention_mask=a      ,					token_type_ids=a      ,					labels=a	)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.seq_length, self.vocab_size)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]      ,					a      :    str      ,					a      :    int      ,					a      :    Tuple      ,					a      :    List[str]      ,					a      :    Tuple      ,					a      :    str      ,					a      :    Optional[Any]	)->       Tuple:





     """simple docstring"""







     lowercase__           =  MobileBertForNextSentencePrediction(config=a	)
     model.to(a	)
     model.eval()
     lowercase__           =  model(
         a      ,					attention_mask=a      ,					token_type_ids=a      ,					labels=a      ,					)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, 2)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    List[Any]      ,					a      :    List[Any]      ,					a      :    List[str]      ,					a      :    List[Any]      ,					a      :    Optional[Any]      ,					a      :    Optional[int]      ,					a      :    Union[str, Any]	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  MobileBertForPreTraining(config=a	)
     model.to(a	)
     model.eval()
     lowercase__           =  model(
         a      ,					attention_mask=a      ,					token_type_ids=a      ,					labels=a      ,					next_sentence_label=a      ,					)
     self.parent.assertEqual(result.prediction_logits.shape      ,					(self.batch_size, self.seq_length, self.vocab_size)	)
     self.parent.assertEqual(result.seq_relationship_logits.shape      ,					(self.batch_size, 2)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    List[str]      ,					a      :    str      ,					a      :    List[str]      ,					a      :    Optional[Any]      ,					a      :    Optional[Any]      ,					a      :    str      ,					a      :    List[Any]	)->       str:





     """simple docstring"""







     lowercase__           =  MobileBertForQuestionAnswering(config=a	)
     model.to(a	)
     model.eval()
     lowercase__           =  model(
         a      ,					attention_mask=a      ,					token_type_ids=a      ,					start_positions=a      ,					end_positions=a      ,					)
     self.parent.assertEqual(result.start_logits.shape      ,					(self.batch_size, self.seq_length)	)
     self.parent.assertEqual(result.end_logits.shape      ,					(self.batch_size, self.seq_length)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    str      ,					a      :    Tuple      ,					a      :    Dict      ,					a      :    Tuple      ,					a      :    Optional[Any]      ,					a      :    Union[str, Any]      ,					a      :    int      ,					a      :    Optional[Any]	)->       Dict:





     """simple docstring"""







     lowercase__           =  self.num_labels
     lowercase__           =  MobileBertForSequenceClassification(a	)
     model.to(a	)
     model.eval()
     lowercase__           =  model(a      ,					attention_mask=a      ,					token_type_ids=a      ,					labels=a	)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.num_labels)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict      ,					a      :    str      ,					a      :    Optional[int]      ,					a      :    Optional[Any]      ,					a      :    Optional[Any]      ,					a      :    Optional[Any]      ,					a      :    List[str]      ,					a      :    Tuple	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  self.num_labels
     lowercase__           =  MobileBertForTokenClassification(config=a	)
     model.to(a	)
     model.eval()
     lowercase__           =  model(a      ,					attention_mask=a      ,					token_type_ids=a      ,					labels=a	)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.seq_length, self.num_labels)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    int      ,					a      :    Optional[Any]      ,					a      :    str      ,					a      :    str      ,					a      :    Dict      ,					a      :    Optional[Any]      ,					a      :    List[str]	)->       str:





     """simple docstring"""







     lowercase__           =  self.num_choices
     lowercase__           =  MobileBertForMultipleChoice(config=a	)
     model.to(a	)
     model.eval()
     lowercase__           =  input_ids.unsqueeze(1	).expand(-1      ,					self.num_choices      ,					-1	).contiguous()
     lowercase__           =  token_type_ids.unsqueeze(1	).expand(-1      ,					self.num_choices      ,					-1	).contiguous()
     lowercase__           =  input_mask.unsqueeze(1	).expand(-1      ,					self.num_choices      ,					-1	).contiguous()
     lowercase__           =  model(
         a      ,					attention_mask=a      ,					token_type_ids=a      ,					labels=a      ,					)
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.num_choices)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       str:





     """simple docstring"""







     lowercase__           =  self.prepare_config_and_inputs()
     (
         (
         lowercase__
     )      ,   (
         lowercase__
     )      ,   (
         lowercase__
     )      ,   (
         lowercase__
     )      ,   (
         lowercase__
     )      ,   (
         lowercase__
     )      ,   (
         lowercase__
     )      ,   
     )           =  config_and_inputs
     lowercase__           =  {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
     return config, inputs_dict






@require_torch
class     SCREAMING_SNAKE_CASE       (UpperCAmelCase      ,     UpperCAmelCase      ,     unittest.TestCase				):
   _UpperCamelCase   :		Dict			 =	(
       (
           MobileBertModel,
           MobileBertForMaskedLM,
           MobileBertForMultipleChoice,
           MobileBertForNextSentencePrediction,
           MobileBertForPreTraining,
           MobileBertForQuestionAnswering,
           MobileBertForSequenceClassification,
           MobileBertForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   _UpperCamelCase   :		Optional[Any]			 =	(
       {
           'feature-extraction': MobileBertModel,
           'fill-mask': MobileBertForMaskedLM,
           'question-answering': MobileBertForQuestionAnswering,
           'text-classification': MobileBertForSequenceClassification,
           'token-classification': MobileBertForTokenClassification,
           'zero-shot': MobileBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   _UpperCamelCase   :		List[Any]			 =	True


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    List[Any]      ,					a      :    Optional[Any]      ,					a      :    str=False	)->       int:





     """simple docstring"""







     lowercase__           =  super()._prepare_for_class(a      ,					a      ,					return_labels=a	)

     if return_labels:
       if model_class in get_values(a	):
         lowercase__           =  torch.zeros(
             (self.model_tester.batch_size, self.model_tester.seq_length)      ,					dtype=torch.long      ,					device=a	)
         lowercase__           =  torch.zeros(
             self.model_tester.batch_size      ,					dtype=torch.long      ,					device=a	)
     return inputs_dict


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       str:





     """simple docstring"""







     lowercase__           =  MobileBertModelTester(self	)
     lowercase__           =  ConfigTester(self      ,					config_class=a      ,					hidden_size=37	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any	)->       Tuple:





     """simple docstring"""







     self.config_tester.run_common_tests()


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Union[str, Any]	)->       List[str]:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_model(*a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Union[str, Any]	)->       str:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_masked_lm(*a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       Any:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_multiple_choice(*a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       str:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Union[str, Any]	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_pretraining(*a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       Union[str, Any]:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_question_answering(*a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]	)->       List[str]:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_sequence_classification(*a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]	)->       Dict:





     """simple docstring"""







     lowercase__           =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_token_classification(*a	)






def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Optional[int]:
  return torch.tensor(
      _SCREAMING_SNAKE_CASE		,						dtype=torch.long		,						device=_SCREAMING_SNAKE_CASE		,						)


lowercase_          =		1E-3


@require_torch
@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE       (unittest.TestCase				):


   @slow
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any	)->       Union[str, Any]:





     """simple docstring"""







     lowercase__           =  MobileBertModel.from_pretrained('google/mobilebert-uncased'	).to(a	)
     lowercase__           =  _long_tensor([[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]	)
     with torch.no_grad():
       lowercase__           =  model(a	)[0]
     lowercase__           =  torch.Size((1, 9, 512)	)
     self.assertEqual(output.shape      ,					a	)
     lowercase__           =  torch.tensor(
         [
             [
                 [-2.4_7_3_6_5_2_6E0_7, 8.2_6_9_1_6_5_6E0_4, 1.6_5_2_1_8_3_8E0_5],
                 [-5.7_5_4_1_7_0_4E-0_1, 3.9_0_5_6_0_2_2E0_0, 4.4_0_1_1_5_0_7E0_0],
                 [2.6_0_4_7_3_5_9E0_0, 1.5_6_7_7_6_5_2E0_0, -1.7_3_2_4_1_8_8E-0_1],
             ]
         ]      ,					device=a      ,					)

     # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
     # ~1 difference, it's therefore not a good idea to measure using addition.
     # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
     # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
     lowercase__           =  torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE	)
     lowercase__           =  torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE	)

     self.assertTrue(lower_bound and upper_bound	)









import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase_          =		logging.get_logger(__name__)


lowercase_          =		{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

lowercase_          =		{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

lowercase_          =		{
    """allenai/longformer-base-4096""": 4_096,
    """allenai/longformer-large-4096""": 4_096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4_096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4_096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4_096,
}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def       __UpperCamelCase  ()    ->       Union[str, Any]:
  lowercase__           =  (
      list(range(ord('!'		)		,						ord('~'		) + 1		)		) + list(range(ord('¡'		)		,						ord('¬'		) + 1		)		) + list(range(ord('®'		)		,						ord('ÿ'		) + 1		)		)
  )
  lowercase__           =  bs[:]
  lowercase__           =  0
  for b in range(2**8		):
    if b not in bs:
      bs.append(_SCREAMING_SNAKE_CASE		)
      cs.append(2**8 + n		)
      n += 1
  lowercase__           =  [chr(_SCREAMING_SNAKE_CASE		) for n in cs]
  return dict(zip(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		)		)




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Union[str, Any]:
  lowercase__           =  set()
  lowercase__           =  word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)		)
    lowercase__           =  char
  return pairs


class     SCREAMING_SNAKE_CASE       (UpperCAmelCase				):
   _UpperCamelCase   :		str			 =	VOCAB_FILES_NAMES
   _UpperCamelCase   :		Optional[int]			 =	PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase   :		Union[str, Any]			 =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase   :		Any			 =	['input_ids', 'attention_mask']


   def __init__(   self      :    Dict      ,					a      :    Union[str, Any]      ,					a      :    Optional[Any]      ,					a      :    List[str]="replace"      ,					a      :    Optional[int]="<s>"      ,					a      :    List[str]="</s>"      ,					a      :    List[Any]="</s>"      ,					a      :    Union[str, Any]="<s>"      ,					a      :    Any="<unk>"      ,					a      :    Optional[int]="<pad>"      ,					a      :    Optional[Any]="<mask>"      ,					a      :    Tuple=False      ,					**a      :    List[Any]      ,					)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else bos_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else eos_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else sep_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else cls_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else unk_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else pad_token

     # Mask token behave like a normal word, i.e. include the space before it
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else mask_token

     super().__init__(
         errors=a      ,					bos_token=a      ,					eos_token=a      ,					unk_token=a      ,					sep_token=a      ,					cls_token=a      ,					pad_token=a      ,					mask_token=a      ,					add_prefix_space=a      ,					**a      ,					)

     with open(a      ,					encoding='utf-8'	) as vocab_handle:
       lowercase__           =  json.load(a	)
     lowercase__           =  {v: k for k, v in self.encoder.items()}
     lowercase__           =  errors  # how to handle errors in decoding
     lowercase__           =  bytes_to_unicode()
     lowercase__           =  {v: k for k, v in self.byte_encoder.items()}
     with open(a      ,					encoding='utf-8'	) as merges_handle:
       lowercase__           =  merges_handle.read().split('\n'	)[1:-1]
     lowercase__           =  [tuple(merge.split()	) for merge in bpe_merges]
     lowercase__           =  dict(zip(a      ,					range(len(a	)	)	)	)
     lowercase__           =  {}
     lowercase__           =  add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     lowercase__           =  re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       Any:





     """simple docstring"""







     return len(self.encoder	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       Optional[int]:





     """simple docstring"""







     return dict(self.encoder      ,					**self.added_tokens_encoder	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]      ,					a      :    List[Any]	)->       Dict:





     """simple docstring"""







     if token in self.cache:
       return self.cache[token]
     lowercase__           =  tuple(a	)
     lowercase__           =  get_pairs(a	)

     if not pairs:
       return token

     while True:
       lowercase__           =  min(a      ,					key=lambda a	: self.bpe_ranks.get(a      ,					float('inf'	)	)	)
       if bigram not in self.bpe_ranks:
         break
       lowercase__      ,   lowercase__           =  bigram
       lowercase__           =  []
       lowercase__           =  0
       while i < len(a	):
         try:
           lowercase__           =  word.index(a      ,					a	)
         except ValueError:
           new_word.extend(word[i:]	)
           break
         else:
           new_word.extend(word[i:j]	)
           lowercase__           =  j

         if word[i] == first and i < len(a	) - 1 and word[i + 1] == second:
           new_word.append(first + second	)
           i += 2
         else:
           new_word.append(word[i]	)
           i += 1
       lowercase__           =  tuple(a	)
       lowercase__           =  new_word
       if len(a	) == 1:
         break
       else:
         lowercase__           =  get_pairs(a	)
     lowercase__           =  ' '.join(a	)
     lowercase__           =  word
     return word


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]      ,					a      :    str	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  []
     for token in re.findall(self.pat      ,					a	):
       lowercase__           =  ''.join(
           self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a	).split(' '	)	)
     return bpe_tokens


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    List[Any]	)->       Optional[int]:





     """simple docstring"""







     return self.encoder.get(a      ,					self.encoder.get(self.unk_token	)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]      ,					a      :    Optional[Any]	)->       Union[str, Any]:





     """simple docstring"""







     return self.decoder.get(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    Optional[int]	)->       Dict:





     """simple docstring"""







     lowercase__           =  ''.join(a	)
     lowercase__           =  bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8'      ,					errors=self.errors	)
     return text


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    str      ,					a      :    Optional[str] = None	)->       Tuple[str]:





     """simple docstring"""







     if not os.path.isdir(a	):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
       return
     lowercase__           =  os.path.join(
         a      ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
     lowercase__           =  os.path.join(
         a      ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

     with open(a      ,					'w'      ,					encoding='utf-8'	) as f:
       f.write(json.dumps(self.encoder      ,					indent=2      ,					sort_keys=a      ,					ensure_ascii=a	) + '\n'	)

     lowercase__           =  0
     with open(a      ,					'w'      ,					encoding='utf-8'	) as writer:
       writer.write('#version: 0.2\n'	)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,					key=lambda a	: kv[1]	):
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
               ' Please check that the tokenizer is not corrupted!'	)
           lowercase__           =  token_index
         writer.write(' '.join(a	) + '\n'	)
         index += 1

     return vocab_file, merge_file


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]      ,					a      :    List[int]      ,					a      :    Optional[List[int]] = None	)->       List[int]:





     """simple docstring"""







     if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     lowercase__           =  [self.cls_token_id]
     lowercase__           =  [self.sep_token_id]
     return cls + token_ids_a + sep + sep + token_ids_a + sep


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    List[int]      ,					a      :    Optional[List[int]] = None      ,					a      :    bool = False	)->       List[int]:





     """simple docstring"""







     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=a      ,					token_ids_a=a      ,					already_has_special_tokens=a	)

     if token_ids_a is None:
       return [1] + ([0] * len(a	)) + [1]
     return [1] + ([0] * len(a	)) + [1, 1] + ([0] * len(a	)) + [1]


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    List[int]      ,					a      :    Optional[List[int]] = None	)->       List[int]:





     """simple docstring"""







     lowercase__           =  [self.sep_token_id]
     lowercase__           =  [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    Dict      ,					a      :    Dict=False      ,					**a      :    Union[str, Any]	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  kwargs.pop('add_prefix_space'      ,					self.add_prefix_space	)
     if (is_split_into_words or add_prefix_space) and (len(a	) > 0 and not text[0].isspace()):
       lowercase__           =  ' ' + text
     return (text, kwargs)


