






'''simple docstring'''



import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          DistilBertForMaskedLM,
          DistilBertForMultipleChoice,
          DistilBertForQuestionAnswering,
          DistilBertForSequenceClassification,
          DistilBertForTokenClassification,
          DistilBertModel,
      )
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      def __init__(			self      :		List[str]  ,   UpperCAmelCase      :		Any  ,   UpperCAmelCase      :		Tuple=13  ,   UpperCAmelCase      :		Dict=7  ,   UpperCAmelCase      :		int=True  ,   UpperCAmelCase      :		Union[str, Any]=True  ,   UpperCAmelCase      :		Dict=False  ,   UpperCAmelCase      :		List[Any]=True  ,   UpperCAmelCase      :		Dict=99  ,   UpperCAmelCase      :		Union[str, Any]=32  ,   UpperCAmelCase      :		Dict=5  ,   UpperCAmelCase      :		Tuple=4  ,   UpperCAmelCase      :		List[str]=37  ,   UpperCAmelCase      :		Tuple="gelu"  ,   UpperCAmelCase      :		List[str]=0.1  ,   UpperCAmelCase      :		Tuple=0.1  ,   UpperCAmelCase      :		int=512  ,   UpperCAmelCase      :		Optional[Any]=16  ,   UpperCAmelCase      :		Tuple=2  ,   UpperCAmelCase      :		Optional[int]=0.0_2  ,   UpperCAmelCase      :		Optional[int]=3  ,   UpperCAmelCase      :		Optional[int]=4  ,   UpperCAmelCase      :		List[Any]=None  ,   )	->	List[Any]:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=parent
       lowercase       : Tuple						=batch_size
       lowercase       : Dict						=seq_length
       lowercase       : List[Any]						=is_training
       lowercase       : int						=use_input_mask
       lowercase       : List[Any]						=use_token_type_ids
       lowercase       : List[Any]						=use_labels
       lowercase       : Any						=vocab_size
       lowercase       : List[Any]						=hidden_size
       lowercase       : str						=num_hidden_layers
       lowercase       : Dict						=num_attention_heads
       lowercase       : Union[str, Any]						=intermediate_size
       lowercase       : Union[str, Any]						=hidden_act
       lowercase       : List[Any]						=hidden_dropout_prob
       lowercase       : str						=attention_probs_dropout_prob
       lowercase       : Optional[Any]						=max_position_embeddings
       lowercase       : Dict						=type_vocab_size
       lowercase       : List[Any]						=type_sequence_label_size
       lowercase       : List[str]						=initializer_range
       lowercase       : Union[str, Any]						=num_labels
       lowercase       : List[str]						=num_choices
       lowercase       : Union[str, Any]						=scope






      def 					A__   (			self      :		Any		)	->	Dict:


       '''simple docstring'''

       lowercase       : Tuple						=ids_tensor([self.batch_size, self.seq_length]  ,   self.vocab_size		)

       lowercase       : Dict						=None
       if self.use_input_mask:
        lowercase       : Optional[Any]						=random_attention_mask([self.batch_size, self.seq_length]		)

       lowercase       : Union[str, Any]						=None
       lowercase       : int						=None
       lowercase       : int						=None
       if self.use_labels:
        lowercase       : Dict						=ids_tensor([self.batch_size]  ,   self.type_sequence_label_size		)
        lowercase       : Any						=ids_tensor([self.batch_size, self.seq_length]  ,   self.num_labels		)
        lowercase       : Union[str, Any]						=ids_tensor([self.batch_size]  ,   self.num_choices		)

       lowercase       : Tuple						=self.get_config()

       return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






      def 					A__   (			self      :		Any		)	->	Tuple:


       '''simple docstring'''

       return DistilBertConfig(
           vocab_size=self.vocab_size  ,   dim=self.hidden_size  ,   n_layers=self.num_hidden_layers  ,   n_heads=self.num_attention_heads  ,   hidden_dim=self.intermediate_size  ,   hidden_act=self.hidden_act  ,   dropout=self.hidden_dropout_prob  ,   attention_dropout=self.attention_probs_dropout_prob  ,   max_position_embeddings=self.max_position_embeddings  ,   initializer_range=self.initializer_range  ,   )






      def 					A__   (			self      :		List[Any]  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		Optional[Any]  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		int  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		Dict		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : int						=DistilBertModel(config=UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()
       lowercase       : List[str]						=model(UpperCAmelCase  ,   UpperCAmelCase		)
       lowercase       : Dict						=model(UpperCAmelCase		)
       self.parent.assertEqual(result.last_hidden_state.shape  ,   (self.batch_size, self.seq_length, self.hidden_size)		)






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		List[Any]  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		Optional[Any]  ,   UpperCAmelCase      :		Optional[int]		)	->	int:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=DistilBertForMaskedLM(config=UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()
       lowercase       : Optional[Any]						=model(UpperCAmelCase  ,   attention_mask=UpperCAmelCase  ,   labels=UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape  ,   (self.batch_size, self.seq_length, self.vocab_size)		)






      def 					A__   (			self      :		Union[str, Any]  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		Optional[int]  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		int  ,   UpperCAmelCase      :		List[str]		)	->	List[str]:


       '''simple docstring'''

       lowercase       : List[Any]						=DistilBertForQuestionAnswering(config=UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()
       lowercase       : Any						=model(
           UpperCAmelCase  ,   attention_mask=UpperCAmelCase  ,   start_positions=UpperCAmelCase  ,   end_positions=UpperCAmelCase		)
       self.parent.assertEqual(result.start_logits.shape  ,   (self.batch_size, self.seq_length)		)
       self.parent.assertEqual(result.end_logits.shape  ,   (self.batch_size, self.seq_length)		)






      def 					A__   (			self      :		Any  ,   UpperCAmelCase      :		Optional[int]  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		Optional[Any]  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		Dict  ,   UpperCAmelCase      :		Dict		)	->	str:


       '''simple docstring'''

       lowercase       : Tuple						=self.num_labels
       lowercase       : int						=DistilBertForSequenceClassification(UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()
       lowercase       : Any						=model(UpperCAmelCase  ,   attention_mask=UpperCAmelCase  ,   labels=UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape  ,   (self.batch_size, self.num_labels)		)






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		Dict  ,   UpperCAmelCase      :		Union[str, Any]		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Tuple						=self.num_labels
       lowercase       : List[str]						=DistilBertForTokenClassification(config=UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()

       lowercase       : Optional[int]						=model(UpperCAmelCase  ,   attention_mask=UpperCAmelCase  ,   labels=UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape  ,   (self.batch_size, self.seq_length, self.num_labels)		)






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		int  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		Any  ,   UpperCAmelCase      :		Optional[Any]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : Tuple						=self.num_choices
       lowercase       : Any						=DistilBertForMultipleChoice(config=UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()
       lowercase       : Union[str, Any]						=input_ids.unsqueeze(1		).expand(-1  ,   self.num_choices  ,   -1		).contiguous()
       lowercase       : str						=input_mask.unsqueeze(1		).expand(-1  ,   self.num_choices  ,   -1		).contiguous()
       lowercase       : str						=model(
           UpperCAmelCase  ,   attention_mask=UpperCAmelCase  ,   labels=UpperCAmelCase  ,   )
       self.parent.assertEqual(result.logits.shape  ,   (self.batch_size, self.num_choices)		)






      def 					A__   (			self      :		Optional[int]		)	->	str:


       '''simple docstring'''

       lowercase       : List[str]						=self.prepare_config_and_inputs()
       ((lowercase)	,		(lowercase)	,		(lowercase)	,		(lowercase)	,		(lowercase)	,		(lowercase))       : List[str]						=config_and_inputs
       lowercase       : Union[str, Any]						={'''input_ids''': input_ids, '''attention_mask''': input_mask}
       return config, inputs_dict





@require_torch
class UpperCAmelCase_       (     __A					,							__A					,							unittest.TestCase					):




      """simple docstring"""




      UpperCamelCase_             =      (
          (
              DistilBertModel,
              DistilBertForMaskedLM,
              DistilBertForMultipleChoice,
              DistilBertForQuestionAnswering,
              DistilBertForSequenceClassification,
              DistilBertForTokenClassification,
          )
          if is_torch_available()
          else None
      )
      UpperCamelCase_             =      (
          {
              '''feature-extraction''': DistilBertModel,
              '''fill-mask''': DistilBertForMaskedLM,
              '''question-answering''': DistilBertForQuestionAnswering,
              '''text-classification''': DistilBertForSequenceClassification,
              '''token-classification''': DistilBertForTokenClassification,
              '''zero-shot''': DistilBertForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      UpperCamelCase_             =      True
      UpperCamelCase_             =      True
      UpperCamelCase_             =      True
      UpperCamelCase_             =      True






      def 					A__   (			self      :		Any		)	->	int:


       '''simple docstring'''

       lowercase       : List[Any]						=DistilBertModelTester(self		)
       lowercase       : Union[str, Any]						=ConfigTester(self  ,   config_class=UpperCAmelCase  ,   dim=37		)






      def 					A__   (			self      :		Tuple		)	->	str:


       '''simple docstring'''

       self.config_tester.run_common_tests()






      def 					A__   (			self      :		Optional[int]		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : Any						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_model(*UpperCAmelCase		)






      def 					A__   (			self      :		Tuple		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : Optional[int]						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_masked_lm(*UpperCAmelCase		)






      def 					A__   (			self      :		Tuple		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : Any						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_question_answering(*UpperCAmelCase		)






      def 					A__   (			self      :		int		)	->	str:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_sequence_classification(*UpperCAmelCase		)






      def 					A__   (			self      :		List[Any]		)	->	str:


       '''simple docstring'''

       lowercase       : Optional[int]						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_token_classification(*UpperCAmelCase		)






      def 					A__   (			self      :		Dict		)	->	int:


       '''simple docstring'''

       lowercase       : int						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_multiple_choice(*UpperCAmelCase		)






      @slow
      def 					A__   (			self      :		List[str]		)	->	str:


       '''simple docstring'''

       for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        lowercase       : List[str]						=DistilBertModel.from_pretrained(UpperCAmelCase		)
        self.assertIsNotNone(UpperCAmelCase		)






      @slow
      @require_torch_gpu
      def 					A__   (			self      :		List[Any]		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase	,		lowercase       : Optional[int]						=self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
        # BertForMultipleChoice behaves incorrectly in JIT environments.
        if model_class == DistilBertForMultipleChoice:
         return

        lowercase       : Optional[Any]						=True
        lowercase       : Tuple						=model_class(config=UpperCAmelCase		)

        lowercase       : Tuple						=self._prepare_for_class(UpperCAmelCase  ,   UpperCAmelCase		)
        lowercase       : Tuple						=torch.jit.trace(
            UpperCAmelCase  ,   (inputs_dict['''input_ids'''].to('''cpu'''		), inputs_dict['''attention_mask'''].to('''cpu'''		))		)

        with tempfile.TemporaryDirectory() as tmp:
         torch.jit.save(UpperCAmelCase  ,   os.path.join(UpperCAmelCase  ,   '''traced_model.pt'''		)		)
         lowercase       : Any						=torch.jit.load(os.path.join(UpperCAmelCase  ,   '''traced_model.pt'''		)  ,   map_location=UpperCAmelCase		)
         loaded(inputs_dict['''input_ids'''].to(UpperCAmelCase		)  ,   inputs_dict['''attention_mask'''].to(UpperCAmelCase		)		)





@require_torch
class UpperCAmelCase_       (     unittest.TestCase					):




      """simple docstring"""




      @slow
      def 					A__   (			self      :		str		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : Dict						=DistilBertModel.from_pretrained('''distilbert-base-uncased'''		)
       lowercase       : List[str]						=torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]		)
       lowercase       : List[Any]						=torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]		)
       with torch.no_grad():
        lowercase       : Any						=model(UpperCAmelCase  ,   attention_mask=UpperCAmelCase		)[0]
       lowercase       : Union[str, Any]						=torch.Size((1, 11, 768)		)
       self.assertEqual(output.shape  ,   UpperCAmelCase		)
       lowercase       : Union[str, Any]						=torch.tensor(
           [[[-0.1_6_3_9, 0.3_2_9_9, 0.1_6_4_8], [-0.1_7_4_6, 0.3_2_8_9, 0.1_7_1_0], [-0.1_8_8_4, 0.3_3_5_7, 0.1_8_1_0]]]		)

       self.assertTrue(torch.allclose(output[:, 1:4, 1:4]  ,   UpperCAmelCase  ,   atol=1e-4		)		)








"""simple docstring"""






import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
     import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case   					=				logging.get_logger(__name__)

snake_case   					=				{'''vocab_file''': '''vocab.txt'''}

snake_case   					=				{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

snake_case   					=				{
    '''openbmb/cpm-ant-10b''': 1_0_2_4,
}




def 			snake_case		(		lowerCAmelCase_	)						->				int:
  _snake_case   	=      collections.OrderedDict()
  with open(lowerCAmelCase_  ,						'''r'''  ,						encoding='''utf-8'''	) as reader:
    _snake_case   	=      reader.readlines()
  for index, token in enumerate(lowerCAmelCase_	):
    _snake_case   	=      token.rstrip('''\n'''	)
    _snake_case   	=      index
  return vocab



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):



   def __init__(       self    :	Union[str, Any]							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	List[str]="<unk>"							,       __lowerCamelCase    :	Tuple=2_0_0				):



     """simple docstring"""
     _snake_case   	=      vocab
     _snake_case   	=      unk_token
     _snake_case   	=      max_input_chars_per_word





   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      list(__lowerCamelCase				)
     if len(__lowerCamelCase				) > self.max_input_chars_per_word:
       return [self.unk_token]

     _snake_case   	=      0
     _snake_case   	=      []
     while start < len(__lowerCamelCase				):
       _snake_case   	=      len(__lowerCamelCase				)
       _snake_case   	=      None
       while start < end:
         _snake_case   	=      ''''''.join(chars[start:end]				)
         if substr in self.vocab:
           _snake_case   	=      substr
           break
         end -= 1
       if cur_substr is None:
         sub_tokens.append(self.unk_token				)
         start += 1
       else:
         sub_tokens.append(__lowerCamelCase				)
         _snake_case   	=      end

     return sub_tokens



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   A__		:	List[str]           =  VOCAB_FILES_NAMES
   A__		:	str           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	str           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__		:	Union[str, Any]           =  ['''input_ids''', '''attention_mask''']
   A__		:	Optional[int]           =  False



   def __init__(       self    :	Dict							,       __lowerCamelCase    :	Optional[Any]							,       __lowerCamelCase    :	str="<d>"							,       __lowerCamelCase    :	Tuple="</d>"							,       __lowerCamelCase    :	Tuple="<s>"							,       __lowerCamelCase    :	int="</s>"							,       __lowerCamelCase    :	List[str]="<pad>"							,       __lowerCamelCase    :	int="<unk>"							,       __lowerCamelCase    :	int="</n>"							,       __lowerCamelCase    :	Tuple="</_>"							,       __lowerCamelCase    :	Optional[Any]="left"							,       **__lowerCamelCase    :	str							,       ):



     """simple docstring"""
     requires_backends(self							,       ['''jieba''']				)
     super().__init__(
         bod_token=__lowerCamelCase							,       eod_token=__lowerCamelCase							,       bos_token=__lowerCamelCase							,       eos_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       line_token=__lowerCamelCase							,       space_token=__lowerCamelCase							,       padding_side=__lowerCamelCase							,       **__lowerCamelCase							,       )
     _snake_case   	=      bod_token
     _snake_case   	=      eod_token
     _snake_case   	=      load_vocab(__lowerCamelCase				)
     _snake_case   	=      self.encoder[space_token]
     _snake_case   	=      self.encoder[line_token]

     del self.encoder[space_token]
     del self.encoder[line_token]

     _snake_case   	=      collections.OrderedDict(sorted(self.encoder.items()							,       key=lambda __lowerCamelCase				: x[1]				)				)
     _snake_case   	=      {v: k for k, v in self.encoder.items()}

     _snake_case   	=      WordpieceTokenizer(vocab=self.encoder							,       unk_token=self.unk_token				)



   @property
   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     return self.encoder[self.bod_token]



   @property
   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     return self.encoder[self.eod_token]



   @property
   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     return self.encoder["\n"]



   @property
   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     return len(self.encoder				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     return dict(self.encoder							,       **self.added_tokens_encoder				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      []
     for x in jieba.cut(__lowerCamelCase							,       cut_all=__lowerCamelCase				):
       output_tokens.extend(self.wordpiece_tokenizer.tokenize(__lowerCamelCase				)				)
     return output_tokens



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	Union[str, Any]							,       **__lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      [i for i in token_ids if i >= 0]
     _snake_case   	=      [
         x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
     ]
     return super()._decode(__lowerCamelCase							,       **__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	Union[str, Any]				):



     """simple docstring"""
     return token in self.encoder



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     return "".join(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.encoder.get(__lowerCamelCase							,       self.encoder.get(self.unk_token				)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.decoder.get(__lowerCamelCase							,       self.unk_token				)



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     if os.path.isdir(__lowerCamelCase				):
       _snake_case   	=      os.path.join(
           __lowerCamelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
     else:
       _snake_case   	=      (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
     _snake_case   	=      0
     if " " in self.encoder:
       _snake_case   	=      self.encoder[''' ''']
       del self.encoder[" "]
     if "\n" in self.encoder:
       _snake_case   	=      self.encoder['''\n''']
       del self.encoder["\n"]
     _snake_case   	=      collections.OrderedDict(sorted(self.encoder.items()							,       key=lambda __lowerCamelCase				: x[1]				)				)
     with open(__lowerCamelCase							,       '''w'''							,       encoding='''utf-8'''				) as writer:
       for token, token_index in self.encoder.items():
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
               ''' Please check that the vocabulary is not corrupted!'''				)
           _snake_case   	=      token_index
         writer.write(token + '''\n'''				)
         index += 1
     return (vocab_file,)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	List[int] = None				):



     """simple docstring"""
     if token_ids_a is None:
       return [self.bos_token_id] + token_ids_a
     return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a





   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None							,       __lowerCamelCase    :	bool = False				):



     """simple docstring"""

     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__lowerCamelCase							,       token_ids_a=__lowerCamelCase							,       already_has_special_tokens=__lowerCamelCase				)

     if token_ids_a is not None:
       return [1] + ([0] * len(__lowerCamelCase				)) + [1] + ([0] * len(__lowerCamelCase				))
     return [1] + ([0] * len(__lowerCamelCase				))





