



'''simple docstring'''





import tempfile
import unittest

from transformers import TaConfig, is_torch_available
from transformers.testing_utils import (
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
    torch_device,
)

from ...generation.test_utils import GenerationTesterMixin
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import AutoTokenizer, UMTaForConditionalGeneration, UMTaForQuestionAnswering, UMTaModel






class    lowerCamelCase_       :
  """simple docstring"""




  def __init__(					self :      Tuple     ,      _a :      Union[str, Any]     ,      _a :      Any=99     ,      _a :      int=13     ,      _a :      Tuple=7     ,      _a :      Optional[Any]=9     ,      _a :      Optional[int]=True     ,      _a :      List[str]=True     ,      _a :      List[str]=False     ,      _a :      Any=32     ,      _a :      List[Any]=5     ,      _a :      Optional[int]=4     ,      _a :      int=37     ,      _a :      Union[str, Any]=8     ,      _a :      Tuple=0.1     ,      _a :      Dict=0.002     ,      _a :      Tuple=1     ,      _a :      List[str]=0     ,      _a :      int=0     ,      _a :      Optional[int]=None     ,      _a :      Optional[int]=None     ,      )   ->   List[str]:
    __lowerCamelCase			:			str       			=					parent
    __lowerCamelCase			:			Dict       			=					batch_size
    __lowerCamelCase			:			List[Any]       			=					encoder_seq_length
    __lowerCamelCase			:			int       			=					decoder_seq_length
    # For common tests
    __lowerCamelCase			:			List[str]       			=					self.decoder_seq_length
    __lowerCamelCase			:			Optional[int]       			=					is_training
    __lowerCamelCase			:			Optional[int]       			=					use_attention_mask
    __lowerCamelCase			:			Dict       			=					use_labels
    __lowerCamelCase			:			int       			=					vocab_size
    __lowerCamelCase			:			Optional[int]       			=					hidden_size
    __lowerCamelCase			:			List[str]       			=					num_hidden_layers
    __lowerCamelCase			:			Any       			=					num_attention_heads
    __lowerCamelCase			:			Optional[Any]       			=					d_ff
    __lowerCamelCase			:			List[str]       			=					relative_attention_num_buckets
    __lowerCamelCase			:			Optional[int]       			=					dropout_rate
    __lowerCamelCase			:			List[Any]       			=					initializer_factor
    __lowerCamelCase			:			Tuple       			=					eos_token_id
    __lowerCamelCase			:			Optional[int]       			=					pad_token_id
    __lowerCamelCase			:			Any       			=					decoder_start_token_id
    __lowerCamelCase			:			Tuple       			=					None
    __lowerCamelCase			:			str       			=					decoder_layers




  def 		_lowercase	(					self :      int  )   ->   int:
    return TaConfig.from_pretrained('google/umt5-base'  )




  def 		_lowercase	(					self :      Union[str, Any]     ,      _a :      Tuple     ,      _a :      Optional[int]     ,      _a :      Optional[int]     ,      _a :      int=None     ,      _a :      List[str]=None     ,      _a :      Tuple=None     ,      _a :      List[str]=None     ,      _a :      Dict=None     ,      )   ->   str:
    if attention_mask is None:
      __lowerCamelCase			:			List[str]       			=					input_ids.ne(config.pad_token_id  )
    if decoder_attention_mask is None:
      __lowerCamelCase			:			int       			=					decoder_input_ids.ne(config.pad_token_id  )
    if head_mask is None:
      __lowerCamelCase			:			List[Any]       			=					torch.ones(config.num_hidden_layers     ,      config.num_attention_heads     ,      device=_a  )
    if decoder_head_mask is None:
      __lowerCamelCase			:			Any       			=					torch.ones(config.num_decoder_layers     ,      config.num_attention_heads     ,      device=_a  )
    if cross_attn_head_mask is None:
      __lowerCamelCase			:			Optional[Any]       			=					torch.ones(
          config.num_decoder_layers     ,      config.num_attention_heads     ,      device=_a  )
    return {
        "input_ids": input_ids,
        "decoder_input_ids": decoder_input_ids,
        "attention_mask": attention_mask,
        "decoder_attention_mask": decoder_attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
        "cross_attn_head_mask": cross_attn_head_mask,
    }




  def 		_lowercase	(					self :      Any  )   ->   Union[str, Any]:
    __lowerCamelCase			:			Tuple       			=					ids_tensor([self.batch_size, self.encoder_seq_length]     ,      self.vocab_size  )
    __lowerCamelCase			:			Any       			=					ids_tensor([self.batch_size, self.decoder_seq_length]     ,      self.vocab_size  )

    # we need to clamp the input ids here to avoid having pad token in between
    # this is because for NllbMoe the position_ids are prepared such that
    # all pad tokens have pos id = 2 and rest are between 2..seq_length
    # and the seq_length here is seq_length - num_pad_tokens
    # but when using past, there is no way of knowing if the past input ids had
    # pad tokens in them, which results in incorrect seq_lenth and which in turn results in
    # position_ids being off by num_pad_tokens in past input
    __lowerCamelCase			:			Tuple       			=					input_ids.clamp(self.pad_token_id + 1  )
    __lowerCamelCase			:			Tuple       			=					decoder_input_ids.clamp(self.pad_token_id + 1  )

    __lowerCamelCase			:			Tuple       			=					self.get_config()
    __lowerCamelCase			:			Dict       			=					config.num_attention_heads
    __lowerCamelCase			:			Union[str, Any]       			=					self.prepare_inputs_dict(_a     ,      _a     ,      _a  )
    return config, input_dict




  def 		_lowercase	(					self :      List[Any]  )   ->   str:
    __lowerCamelCase			,__lowerCamelCase			:			Tuple       			=					self.prepare_config_and_inputs()
    return config, inputs_dict




  def 		_lowercase	(					self :      Tuple  )   ->   Optional[int]:
    return TaConfig(
        vocab_size=166     ,      d_model=self.hidden_size     ,      d_ff=self.d_ff     ,      d_kv=self.hidden_size // self.num_attention_heads     ,      num_layers=self.num_hidden_layers     ,      num_decoder_layers=self.decoder_layers     ,      num_heads=self.num_attention_heads     ,      relative_attention_num_buckets=self.relative_attention_num_buckets     ,      dropout_rate=self.dropout_rate     ,      initializer_factor=self.initializer_factor     ,      eos_token_id=self.eos_token_id     ,      bos_token_id=self.pad_token_id     ,      pad_token_id=self.pad_token_id     ,      decoder_start_token_id=self.decoder_start_token_id     ,      )




  def 		_lowercase	(					self :      Optional[int]  )   ->   str:
    return TaConfig(
        vocab_size=self.vocab_size     ,      d_model=self.hidden_size     ,      d_ff=self.d_ff     ,      d_kv=self.hidden_size // self.num_attention_heads     ,      num_layers=self.num_hidden_layers     ,      num_decoder_layers=self.decoder_layers     ,      num_heads=self.num_attention_heads     ,      relative_attention_num_buckets=self.relative_attention_num_buckets     ,      dropout_rate=self.dropout_rate     ,      initializer_factor=self.initializer_factor     ,      eos_token_id=self.eos_token_id     ,      bos_token_id=self.pad_token_id     ,      pad_token_id=self.pad_token_id     ,      decoder_start_token_id=self.decoder_start_token_id     ,      )




  def 		_lowercase	(					self :      List[str]     ,      _a :      Union[str, Any]     ,      _a :      Optional[int]     ,      _a :      Tuple     ,      _a :      Tuple     ,      _a :      str     ,      _a :      str     ,      )   ->   Dict:
    __lowerCamelCase			:			Any       			=					UMTaModel(config=_a  )
    model.to(_a  )
    model.eval()
    __lowerCamelCase			:			Any       			=					model(
        input_ids=_a     ,      decoder_input_ids=_a     ,      attention_mask=_a     ,      decoder_attention_mask=_a     ,      )
    __lowerCamelCase			:			Dict       			=					model(input_ids=_a     ,      decoder_input_ids=_a  )
    __lowerCamelCase			:			Optional[Any]       			=					result.last_hidden_state
    __lowerCamelCase			:			Optional[Any]       			=					result.past_key_values
    __lowerCamelCase			:			Any       			=					result.encoder_last_hidden_state

    self.parent.assertEqual(encoder_output.size()     ,      (self.batch_size, self.encoder_seq_length, self.hidden_size)  )
    self.parent.assertEqual(decoder_output.size()     ,      (self.batch_size, self.decoder_seq_length, self.hidden_size)  )
    # There should be `num_layers` key value embeddings stored in decoder_past
    self.parent.assertEqual(len(_a  )     ,      config.num_layers  )
    # There should be a self attn key, a self attn value, a cross attn key and a cross attn value stored in each decoder_past tuple
    self.parent.assertEqual(len(decoder_past[0]  )     ,      4  )




  def 		_lowercase	(					self :      List[str]     ,      _a :      str     ,      _a :      List[Any]     ,      _a :      Tuple     ,      _a :      Tuple     ,      _a :      List[Any]     ,      _a :      List[str]     ,      )   ->   Dict:
    __lowerCamelCase			:			Optional[Any]       			=					UMTaModel(config=_a  ).get_decoder().to(_a  ).eval()
    # first forward pass
    __lowerCamelCase			:			str       			=					model(_a     ,      use_cache=_a  )
    __lowerCamelCase			:			Any       			=					model(_a  )
    __lowerCamelCase			:			List[str]       			=					model(_a     ,      use_cache=_a  )

    self.parent.assertTrue(len(_a  ) == len(_a  )  )
    self.parent.assertTrue(len(_a  ) == len(_a  ) + 1  )

    __lowerCamelCase			,__lowerCamelCase			:			Dict       			=					outputs.to_tuple()

    # create hypothetical next token and extent to next_input_ids
    __lowerCamelCase			:			Union[str, Any]       			=					ids_tensor((self.batch_size, 1)     ,      config.vocab_size  )

    # append to next input_ids and
    __lowerCamelCase			:			List[str]       			=					torch.cat([input_ids, next_tokens]     ,      dim=-1  )

    __lowerCamelCase			:			int       			=					model(_a  )['last_hidden_state']
    __lowerCamelCase			:			List[str]       			=					model(_a     ,      past_key_values=_a  )['last_hidden_state']

    # select random slice
    __lowerCamelCase			:			Tuple       			=					ids_tensor((1,)     ,      output_from_past.shape[-1]  ).item()
    __lowerCamelCase			:			Any       			=					output_from_no_past[:, -1, random_slice_idx].detach()
    __lowerCamelCase			:			str       			=					output_from_past[:, 0, random_slice_idx].detach()

    # test that outputs are equal for slice
    self.parent.assertTrue(torch.allclose(_a     ,      _a     ,      atol=1e-3  )  )




  def 		_lowercase	(					self :      Union[str, Any]     ,      _a :      List[Any]     ,      _a :      Optional[int]     ,      )   ->   Optional[Any]:
    __lowerCamelCase			:			List[str]       			=					UMTaModel(config=_a  ).to(_a  ).half().eval()
    __lowerCamelCase			:			Tuple       			=					model(**_a  )['last_hidden_state']
    self.parent.assertFalse(torch.isnan(_a  ).any().item()  )






@require_torch
class    lowerCamelCase_       (							SCREAMING_SNAKE_CASE_  ,  SCREAMING_SNAKE_CASE_  ,  SCREAMING_SNAKE_CASE_  ,  unittest.TestCase	):
  """simple docstring"""




  a_				   =(
      (UMTaModel, UMTaForConditionalGeneration, UMTaForQuestionAnswering) if is_torch_available() else ()
  )
  a_				   =(UMTaForConditionalGeneration,) if is_torch_available() else ()
  a_				   =(
      {
          """conversational""": UMTaForConditionalGeneration,
          """feature-extraction""": UMTaModel,
          """summarization""": UMTaForConditionalGeneration,
          """text2text-generation""": UMTaForConditionalGeneration,
          """translation""": UMTaForConditionalGeneration,
          """question-answering""": UMTaForQuestionAnswering,
      }
      if is_torch_available()
      else {}
  )
  a_				   =True
  a_				   =False
  a_				   =False
  a_				   =True
  a_				   =True
  # The small UMT5 model needs higher percentages for CPU/MP tests
  a_				   =[0.8, 0.9]




  def 		_lowercase	(					self :      str  )   ->   List[str]:
    __lowerCamelCase			:			int       			=					UMTaModelTester(self  )




  @unittest.skip('Test has a segmentation fault on torch 1.8.0'  )
  def 		_lowercase	(					self :      Tuple  )   ->   Union[str, Any]:
    __lowerCamelCase			:			List[str]       			=					self.model_tester.prepare_config_and_inputs()
    __lowerCamelCase			:			List[str]       			=					UMTaModel(config_and_inputs[0]  ).to(_a  )
    with tempfile.TemporaryDirectory() as tmpdirname:
      torch.onnx.export(
          _a     ,      (config_and_inputs[1], config_and_inputs[3], config_and_inputs[2])     ,      f'{tmpdirname}/t5_test.onnx'     ,      export_params=_a     ,      opset_version=9     ,      input_names=['input_ids', 'decoder_input_ids']     ,      )




  @unittest.skipIf(torch_device == 'cpu'     ,      'Cant do half precision'  )
  def 		_lowercase	(					self :      List[str]  )   ->   Optional[Any]:
    __lowerCamelCase			:			List[str]       			=					self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model_fpaa_forward(*_a  )




  def 		_lowercase	(					self :      Optional[Any]  )   ->   List[Any]:
    __lowerCamelCase			:			int       			=					['encoder_attentions', 'decoder_attentions', 'cross_attentions']
    __lowerCamelCase			:			Optional[Any]       			=					self.model_tester.prepare_config_and_inputs()
    __lowerCamelCase			:			str       			=					config_and_inputs[0]
    __lowerCamelCase			:			Any       			=					UMTaForConditionalGeneration(_a  ).eval()
    model.to(_a  )

    __lowerCamelCase			:			Dict       			=					{
        'head_mask': torch.zeros(config.num_layers     ,      config.num_heads     ,      device=_a  ),
        'decoder_head_mask': torch.zeros(config.num_decoder_layers     ,      config.num_heads     ,      device=_a  ),
        'cross_attn_head_mask': torch.zeros(config.num_decoder_layers     ,      config.num_heads     ,      device=_a  ),
    }

    for attn_name, (name, mask) in zip(_a     ,      head_masking.items()  ):
      __lowerCamelCase			:			List[Any]       			=					{name: mask}
      # Explicitly pass decoder_head_mask as it is required from T5 model when head_mask specified
      if name == "head_mask":
        __lowerCamelCase			:			Dict       			=					torch.ones(
            config.num_decoder_layers     ,      config.num_heads     ,      device=_a  )

      __lowerCamelCase			:			int       			=					model.generate(
          config_and_inputs[1]['input_ids']     ,      num_beams=1     ,      max_length=3     ,      output_attentions=_a     ,      return_dict_in_generate=_a     ,      **_a     ,      )
      # We check the state of decoder_attentions and cross_attentions just from the last step
      __lowerCamelCase			:			Union[str, Any]       			=					out[attn_name] if attn_name == attention_names[0] else out[attn_name][-1]
      self.assertEqual(sum([w.sum().item() for w in attn_weights]  )     ,      0.0  )




  @unittest.skip('Does not work on the tiny model as we keep hitting edge cases.'  )
  def 		_lowercase	(					self :      str  )   ->   Union[str, Any]:
    pass






@require_torch
@require_sentencepiece
@require_tokenizers
class    lowerCamelCase_       (							unittest.TestCase	):
  """simple docstring"""




  @slow
  @unittest.skip(
      'Unless we stop stripping left and right by default for all special tokens, the expected ids obtained here will not match the original ones. Wait for https://github.com/huggingface/transformers/pull/23909 to be merged'  )
  def 		_lowercase	(					self :      Dict  )   ->   Tuple:
    __lowerCamelCase			:			Tuple       			=					UMTaForConditionalGeneration.from_pretrained('google/umt5-small'     ,      return_dict=_a  ).to(_a  )
    __lowerCamelCase			:			Tuple       			=					AutoTokenizer.from_pretrained('google/umt5-small'     ,      use_fast=_a     ,      legacy=_a  )
    __lowerCamelCase			:			Union[str, Any]       			=					[
        'Bonjour monsieur <extra_id_0> bien <extra_id_1>.',
        'No se como puedo <extra_id_0>.',
        'This is the reason why we <extra_id_0> them.',
        'The <extra_id_0> walks in <extra_id_1>, seats',
        'A <extra_id_0> walks into a bar and orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>.',
    ]
    __lowerCamelCase			:			Tuple       			=					tokenizer(_a     ,      return_tensors='pt'     ,      padding=_a  ).input_ids
    # fmt: off
    __lowerCamelCase			:			int       			=					torch.tensor(
        [
            [ 3_8530, 21_0703, 25_6299, 1410, 25_6298, 274, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
            [   826, 321, 671, 2_5922, 25_6299, 274, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
            [  1460, 339, 312, 1_9014, 1_0620, 758, 25_6299, 2355,274, 1, 0, 0, 0, 0, 0, 0,0, 0],
            [   517, 25_6299, 1_4869, 281, 301, 25_6298, 275, 11_9983,1, 0, 0, 0, 0, 0, 0, 0,0, 0],
            [   320, 25_6299, 1_4869, 281, 2234, 289, 2275, 333,6_1391, 289, 25_6298, 543, 25_6297, 16_8714, 329, 25_6296,274, 1],
        ]  )
    # fmt: on
    torch.testing.assert_allclose(_a     ,      _a  )

    __lowerCamelCase			:			Any       			=					model.generate(input_ids.to(_a  )  )
    __lowerCamelCase			:			Optional[int]       			=					[
        '<pad><extra_id_0> et<extra_id_1> [eod] <extra_id_2><extra_id_55>.. [eod] 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 <extra_id_56>ajšietosto<extra_id_56>lleux<extra_id_19><extra_id_6>ajšie</s>',
        '<pad><extra_id_0>.<extra_id_1>.,<0x0A>...spech <0x0A><extra_id_20> <extra_id_21></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>',
        '<pad><extra_id_0> are not going to be a part of the world. We are not going to be a part of<extra_id_1> and<extra_id_2><0x0A><extra_id_48>.<extra_id_48></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>',
        '<pad><extra_id_0> door<extra_id_1>, the door<extra_id_2> 피해[/</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>',
        '<pad><extra_id_0>nyone who<extra_id_1> drink<extra_id_2> a<extra_id_3> alcohol<extra_id_4> A<extra_id_5> A. This<extra_id_6> I<extra_id_7><extra_id_52><extra_id_53></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>',
    ]
    __lowerCamelCase			:			Union[str, Any]       			=					tokenizer.batch_decode(_a  )
    self.assertEqual(_a     ,      _a  )











'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_UpperCamelCase											=		logging.get_logger(__name__)

_UpperCamelCase											=		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase											=		{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

_UpperCamelCase											=		{
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


_UpperCamelCase											=		{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}






class    lowerCamelCase_       (							SCREAMING_SNAKE_CASE_	):
  """simple docstring"""




  a_				   =VOCAB_FILES_NAMES
  a_				   =PRETRAINED_VOCAB_FILES_MAP
  a_				   =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_				   =PRETRAINED_INIT_CONFIGURATION
  a_				   =RoFormerTokenizer




  def __init__(					self :      List[str]     ,      _a :      Optional[Any]=None     ,      _a :      Dict=None     ,      _a :      Dict=True     ,      _a :      Union[str, Any]="[UNK]"     ,      _a :      List[Any]="[SEP]"     ,      _a :      List[Any]="[PAD]"     ,      _a :      Any="[CLS]"     ,      _a :      Any="[MASK]"     ,      _a :      Dict=True     ,      _a :      Any=None     ,      **_a :      Tuple     ,      )   ->   Optional[int]:
    super().__init__(
        _a     ,      tokenizer_file=_a     ,      do_lower_case=_a     ,      unk_token=_a     ,      sep_token=_a     ,      pad_token=_a     ,      cls_token=_a     ,      mask_token=_a     ,      tokenize_chinese_chars=_a     ,      strip_accents=_a     ,      **_a     ,      )

    __lowerCamelCase			:			List[str]       			=					json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
    if (
        pre_tok_state.get('lowercase'     ,      _a  ) != do_lower_case
        or pre_tok_state.get('strip_accents'     ,      _a  ) != strip_accents
    ):
      __lowerCamelCase			:			Dict       			=					getattr(_a     ,      pre_tok_state.pop('type'  )  )
      __lowerCamelCase			:			Tuple       			=					do_lower_case
      __lowerCamelCase			:			int       			=					strip_accents
      __lowerCamelCase			:			Dict       			=					pre_tok_class(**_a  )

    __lowerCamelCase			:			Any       			=					do_lower_case




  def __getstate__(					self :      Union[str, Any]  )   ->   List[Any]:
    __lowerCamelCase			:			Optional[Any]       			=					self.__dict__.copy()
    __lowerCamelCase			:			Any       			=					BertPreTokenizer()
    return state




  def __setstate__(					self :      Optional[Any]     ,      _a :      Optional[int]  )   ->   Tuple:
    __lowerCamelCase			:			Tuple       			=					d
    __lowerCamelCase			:			Optional[Any]       			=					self.__dict__['_tokenizer'].get_vocab()
    __lowerCamelCase			:			Any       			=					PreTokenizer.custom(JiebaPreTokenizer(_a  )  )




  def 		_lowercase	(					self :      Any     ,      _a :      Union[str, Any]     ,      _a :      str=None  )   ->   Any:
    __lowerCamelCase			:			Optional[Any]       			=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

    return output




  def 		_lowercase	(					self :      int     ,      _a :      List[int]     ,      _a :      Optional[List[int]] = None  )   ->   List[int]:
    __lowerCamelCase			:			int       			=					[self.sep_token_id]
    __lowerCamelCase			:			int       			=					[self.cls_token_id]
    if token_ids_a is None:
      return len(cls + token_ids_a + sep  ) * [0]
    return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




  def 		_lowercase	(					self :      List[str]     ,      _a :      str     ,      _a :      Optional[str] = None  )   ->   Tuple[str]:
    __lowerCamelCase			:			Optional[int]       			=					self._tokenizer.model.save(_a     ,      name=_a  )
    return tuple(_a  )




  def 		_lowercase	(					self :      int     ,      _a :      Optional[Any]     ,      _a :      Tuple=None     ,      _a :      List[str]=None     ,      _a :      Optional[int]=False     ,      **_a :      Dict     ,      )   ->   Dict:
    __lowerCamelCase			:			Dict       			=					BertPreTokenizer()
    return super().save_pretrained(_a     ,      _a     ,      _a     ,      _a     ,      **_a  )






