


from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TFConvBertForMaskedLM,
     TFConvBertForMultipleChoice,
     TFConvBertForQuestionAnswering,
     TFConvBertForSequenceClassification,
     TFConvBertForTokenClassification,
     TFConvBertModel,
 )




class A    :






      def __init__(   self		:	Union[str, Any]     ,  lowercase_		:	Optional[Any]     ,  lowercase_		:	Optional[Any]=13     ,  lowercase_		:	Union[str, Any]=7     ,  lowercase_		:	Tuple=True     ,  lowercase_		:	str=True     ,  lowercase_		:	Union[str, Any]=True     ,  lowercase_		:	Any=True     ,  lowercase_		:	List[str]=99     ,  lowercase_		:	Optional[int]=32     ,  lowercase_		:	List[str]=2     ,  lowercase_		:	Union[str, Any]=4     ,  lowercase_		:	Optional[Any]=37     ,  lowercase_		:	Tuple="gelu"     ,  lowercase_		:	Tuple=0.1     ,  lowercase_		:	List[str]=0.1     ,  lowercase_		:	List[str]=512     ,  lowercase_		:	Any=16     ,  lowercase_		:	Union[str, Any]=2     ,  lowercase_		:	int=0.02     ,  lowercase_		:	List[Any]=3     ,  lowercase_		:	Any=4     ,  lowercase_		:	Dict=None     ,  )   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            =parent
             _lowerCamelCase			:						Tuple            =13
             _lowerCamelCase			:						Optional[Any]            =7
             _lowerCamelCase			:						Optional[Any]            =True
             _lowerCamelCase			:						Dict            =True
             _lowerCamelCase			:						List[str]            =True
             _lowerCamelCase			:						Any            =True
             _lowerCamelCase			:						List[Any]            =99
             _lowerCamelCase			:						Dict            =384
             _lowerCamelCase			:						Dict            =2
             _lowerCamelCase			:						List[Any]            =4
             _lowerCamelCase			:						Dict            =37
             _lowerCamelCase			:						Tuple            ='gelu'
             _lowerCamelCase			:						Optional[Any]            =0.1
             _lowerCamelCase			:						str            =0.1
             _lowerCamelCase			:						Tuple            =512
             _lowerCamelCase			:						int            =16
             _lowerCamelCase			:						List[str]            =2
             _lowerCamelCase			:						Optional[int]            =0.02
             _lowerCamelCase			:						Optional[Any]            =3
             _lowerCamelCase			:						List[Any]            =4
             _lowerCamelCase			:						Any            =128
             _lowerCamelCase			:						int            =2
             _lowerCamelCase			:						Dict            =9
             _lowerCamelCase			:						str            =1
             _lowerCamelCase			:						Union[str, Any]            =None






      def 			lowerCamelCase			(   self		:	Any					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						Optional[int]            =ids_tensor([self.batch_size, self.seq_length]     ,  self.vocab_size					)

             _lowerCamelCase			:						int            =None
             if self.use_input_mask:
                    _lowerCamelCase			:						Tuple            =random_attention_mask([self.batch_size, self.seq_length]					)

             _lowerCamelCase			:						List[Any]            =None
             if self.use_token_type_ids:
                    _lowerCamelCase			:						List[str]            =ids_tensor([self.batch_size, self.seq_length]     ,  self.type_vocab_size					)

             _lowerCamelCase			:						List[str]            =None
             _lowerCamelCase			:						List[str]            =None
             _lowerCamelCase			:						str            =None
             if self.use_labels:
                    _lowerCamelCase			:						List[Any]            =ids_tensor([self.batch_size]     ,  self.type_sequence_label_size					)
                    _lowerCamelCase			:						Tuple            =ids_tensor([self.batch_size, self.seq_length]     ,  self.num_labels					)
                    _lowerCamelCase			:						Any            =ids_tensor([self.batch_size]     ,  self.num_choices					)

             _lowerCamelCase			:						str            =ConvBertConfig(
                 vocab_size=self.vocab_size     ,  hidden_size=self.hidden_size     ,  num_hidden_layers=self.num_hidden_layers     ,  num_attention_heads=self.num_attention_heads     ,  intermediate_size=self.intermediate_size     ,  hidden_act=self.hidden_act     ,  hidden_dropout_prob=self.hidden_dropout_prob     ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,  max_position_embeddings=self.max_position_embeddings     ,  type_vocab_size=self.type_vocab_size     ,  initializer_range=self.initializer_range     ,  return_dict=lowercase_     ,  )

             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






      def 			lowerCamelCase			(   self		:	Optional[Any]     ,  lowercase_		:	Dict     ,  lowercase_		:	Dict     ,  lowercase_		:	Optional[Any]     ,  lowercase_		:	str     ,  lowercase_		:	List[Any]     ,  lowercase_		:	Union[str, Any]     ,  lowercase_		:	int					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						Any            =TFConvBertModel(config=lowercase_					)
             _lowerCamelCase			:						Optional[int]            ={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

             _lowerCamelCase			:						int            =[input_ids, input_mask]
             _lowerCamelCase			:						List[str]            =model(lowercase_					)

             _lowerCamelCase			:						Optional[int]            =model(lowercase_					)

             self.parent.assertEqual(result.last_hidden_state.shape     ,  (self.batch_size, self.seq_length, self.hidden_size)					)






      def 			lowerCamelCase			(   self		:	Dict     ,  lowercase_		:	Optional[int]     ,  lowercase_		:	int     ,  lowercase_		:	Any     ,  lowercase_		:	Optional[int]     ,  lowercase_		:	Any     ,  lowercase_		:	Optional[Any]     ,  lowercase_		:	Optional[Any]					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						str            =TFConvBertForMaskedLM(config=lowercase_					)
             _lowerCamelCase			:						str            ={
                 'input_ids': input_ids,
                 'attention_mask': input_mask,
                 'token_type_ids': token_type_ids,
             }
             _lowerCamelCase			:						List[Any]            =model(lowercase_					)
             self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)					)






      def 			lowerCamelCase			(   self		:	Any     ,  lowercase_		:	Tuple     ,  lowercase_		:	str     ,  lowercase_		:	Any     ,  lowercase_		:	Union[str, Any]     ,  lowercase_		:	Optional[Any]     ,  lowercase_		:	Optional[int]     ,  lowercase_		:	List[str]					)   ->		List[Any]:
             """simple docstring"""




             _lowerCamelCase			:						Tuple            =self.num_labels
             _lowerCamelCase			:						Union[str, Any]            =TFConvBertForSequenceClassification(config=lowercase_					)
             _lowerCamelCase			:						Dict            ={
                 'input_ids': input_ids,
                 'attention_mask': input_mask,
                 'token_type_ids': token_type_ids,
             }

             _lowerCamelCase			:						List[str]            =model(lowercase_					)
             self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.num_labels)					)






      def 			lowerCamelCase			(   self		:	Dict     ,  lowercase_		:	Tuple     ,  lowercase_		:	Any     ,  lowercase_		:	Tuple     ,  lowercase_		:	Tuple     ,  lowercase_		:	Tuple     ,  lowercase_		:	List[str]     ,  lowercase_		:	Union[str, Any]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =self.num_choices
             _lowerCamelCase			:						Tuple            =TFConvBertForMultipleChoice(config=lowercase_					)
             _lowerCamelCase			:						List[str]            =tf.tile(tf.expand_dims(lowercase_     ,  1					)     ,  (1, self.num_choices, 1)					)
             _lowerCamelCase			:						Tuple            =tf.tile(tf.expand_dims(lowercase_     ,  1					)     ,  (1, self.num_choices, 1)					)
             _lowerCamelCase			:						Optional[Any]            =tf.tile(tf.expand_dims(lowercase_     ,  1					)     ,  (1, self.num_choices, 1)					)
             _lowerCamelCase			:						List[str]            ={
                 'input_ids': multiple_choice_inputs_ids,
                 'attention_mask': multiple_choice_input_mask,
                 'token_type_ids': multiple_choice_token_type_ids,
             }
             _lowerCamelCase			:						Dict            =model(lowercase_					)
             self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.num_choices)					)






      def 			lowerCamelCase			(   self		:	str     ,  lowercase_		:	Tuple     ,  lowercase_		:	List[str]     ,  lowercase_		:	Any     ,  lowercase_		:	int     ,  lowercase_		:	List[Any]     ,  lowercase_		:	Tuple     ,  lowercase_		:	List[str]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            =self.num_labels
             _lowerCamelCase			:						Tuple            =TFConvBertForTokenClassification(config=lowercase_					)
             _lowerCamelCase			:						Any            ={
                 'input_ids': input_ids,
                 'attention_mask': input_mask,
                 'token_type_ids': token_type_ids,
             }
             _lowerCamelCase			:						List[Any]            =model(lowercase_					)
             self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.num_labels)					)






      def 			lowerCamelCase			(   self		:	Tuple     ,  lowercase_		:	List[str]     ,  lowercase_		:	Optional[Any]     ,  lowercase_		:	Any     ,  lowercase_		:	Union[str, Any]     ,  lowercase_		:	Optional[int]     ,  lowercase_		:	Optional[int]     ,  lowercase_		:	List[str]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Optional[Any]            =TFConvBertForQuestionAnswering(config=lowercase_					)
             _lowerCamelCase			:						Dict            ={
                 'input_ids': input_ids,
                 'attention_mask': input_mask,
                 'token_type_ids': token_type_ids,
             }

             _lowerCamelCase			:						Any            =model(lowercase_					)
             self.parent.assertEqual(result.start_logits.shape     ,  (self.batch_size, self.seq_length)					)
             self.parent.assertEqual(result.end_logits.shape     ,  (self.batch_size, self.seq_length)					)






      def 			lowerCamelCase			(   self		:	Dict					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						int            =self.prepare_config_and_inputs()
             (
                 (
                 _lowerCamelCase
             )      ,	(
                 _lowerCamelCase
             )      ,	(
                 _lowerCamelCase
             )      ,	(
                 _lowerCamelCase
             )      ,	(
                 _lowerCamelCase
             )      ,	(
                 _lowerCamelCase
             )      ,	(
                 _lowerCamelCase
             )      ,	
             )			:						List[str]            =config_and_inputs
             _lowerCamelCase			:						Optional[Any]            ={'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
             return config, inputs_dict




@require_tf
class A    (  UpperCamelCase_       ,				UpperCamelCase_       ,				unittest.TestCase	):
      UpperCamelCase__				:    Tuple =(
          (
              TFConvBertModel,
              TFConvBertForMaskedLM,
              TFConvBertForQuestionAnswering,
              TFConvBertForSequenceClassification,
              TFConvBertForTokenClassification,
              TFConvBertForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      UpperCamelCase__				:    int =(
          {
              'feature-extraction': TFConvBertModel,
              'fill-mask': TFConvBertForMaskedLM,
              'question-answering': TFConvBertForQuestionAnswering,
              'text-classification': TFConvBertForSequenceClassification,
              'token-classification': TFConvBertForTokenClassification,
              'zero-shot': TFConvBertForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      UpperCamelCase__				:    List[Any] =False
      UpperCamelCase__				:    List[str] =False
      UpperCamelCase__				:    Optional[Any] =False






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =TFConvBertModelTester(self					)
             _lowerCamelCase			:						str            =ConfigTester(self     ,  config_class=lowercase_     ,  hidden_size=37					)






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		str:
             """simple docstring"""




             self.config_tester.run_common_tests()






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						Optional[Any]            =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*lowercase_					)






      def 			lowerCamelCase			(   self		:	int					)   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_masked_lm(*lowercase_					)






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Optional[int]            =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_multiple_choice(*lowercase_					)






      def 			lowerCamelCase			(   self		:	Tuple					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Optional[int]            =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_question_answering(*lowercase_					)






      def 			lowerCamelCase			(   self		:	Any					)   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_sequence_classification(*lowercase_					)






      def 			lowerCamelCase			(   self		:	int					)   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						Any            =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_token_classification(*lowercase_					)






      @slow
      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		Tuple:
             """simple docstring"""




             _lowerCamelCase      ,	_lowerCamelCase			:						Union[str, Any]            =self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase			:						List[Any]            =True
             _lowerCamelCase			:						List[str]            =True

             if hasattr(lowercase_     ,  'use_cache'					):
                    _lowerCamelCase			:						Any            =True

             _lowerCamelCase			:						List[str]            =getattr(self.model_tester     ,  'encoder_seq_length'     ,  self.model_tester.seq_length					)
             _lowerCamelCase			:						Tuple            =getattr(self.model_tester     ,  'key_length'     ,  lowercase_					)

             for model_class in self.all_model_classes:
                    _lowerCamelCase			:						Tuple            =self._prepare_for_class(lowercase_     ,  lowercase_					)
                    _lowerCamelCase			:						List[Any]            =model_class(lowercase_					)
                    _lowerCamelCase			:						Any            =len(model(lowercase_					)					)

                    with tempfile.TemporaryDirectory() as tmpdirname:
                           model.save_pretrained(lowercase_     ,  saved_model=lowercase_					)
                           _lowerCamelCase			:						Any            =os.path.join(lowercase_     ,  'saved_model'     ,  '1'					)
                           _lowerCamelCase			:						List[Any]            =tf.keras.models.load_model(lowercase_					)
                           _lowerCamelCase			:						Tuple            =model(lowercase_					)

                           if self.is_encoder_decoder:
                                  _lowerCamelCase			:						Dict            =outputs['encoder_hidden_states']
                                  _lowerCamelCase			:						Optional[int]            =outputs['encoder_attentions']
                           else:
                                  _lowerCamelCase			:						List[str]            =outputs['hidden_states']
                                  _lowerCamelCase			:						Dict            =outputs['attentions']

                           self.assertEqual(len(lowercase_					)     ,  lowercase_					)

                           _lowerCamelCase			:						str            =getattr(
                               self.model_tester     ,  'expected_num_hidden_layers'     ,  self.model_tester.num_hidden_layers + 1					)

                           self.assertEqual(len(lowercase_					)     ,  lowercase_					)
                           self.assertListEqual(
                               list(output_hidden_states[0].shape[-2:]					)     ,  [self.model_tester.seq_length, self.model_tester.hidden_size]     ,  )

                           self.assertEqual(len(lowercase_					)     ,  self.model_tester.num_hidden_layers					)
                           self.assertListEqual(
                               list(output_attentions[0].shape[-3:]					)     ,  [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]     ,  )






      @slow
      def 			lowerCamelCase			(   self		:	Any					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            =TFConvBertModel.from_pretrained('YituTech/conv-bert-base'					)
             self.assertIsNotNone(lowercase_					)






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		int:
             """simple docstring"""




             _lowerCamelCase      ,	_lowerCamelCase			:						Optional[Any]            =self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase			:						Union[str, Any]            =True
             _lowerCamelCase			:						List[Any]            =getattr(self.model_tester     ,  'decoder_seq_length'     ,  self.model_tester.seq_length					)
             _lowerCamelCase			:						Optional[int]            =getattr(self.model_tester     ,  'encoder_seq_length'     ,  self.model_tester.seq_length					)
             _lowerCamelCase			:						List[str]            =getattr(self.model_tester     ,  'key_length'     ,  lowercase_					)
             _lowerCamelCase			:						int            =getattr(self.model_tester     ,  'key_length'     ,  lowercase_					)

             def check_decoder_attentions_output(lowercase_		:	Any					):
                    _lowerCamelCase			:						Tuple            =len(lowercase_					)
                    self.assertEqual(out_len % 2     ,  0					)
                    _lowerCamelCase			:						List[Any]            =outputs.decoder_attentions
                    self.assertEqual(len(lowercase_					)     ,  self.model_tester.num_hidden_layers					)
                    self.assertListEqual(
                        list(decoder_attentions[0].shape[-3:]					)     ,  [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]     ,  )

             def check_encoder_attentions_output(lowercase_		:	Any					):
                    _lowerCamelCase			:						List[str]            =[
                        t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
                    ]
                    self.assertEqual(len(lowercase_					)     ,  self.model_tester.num_hidden_layers					)
                    self.assertListEqual(
                        list(attentions[0].shape[-3:]					)     ,  [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]     ,  )

             for model_class in self.all_model_classes:
                    _lowerCamelCase			:						List[str]            =True
                    _lowerCamelCase			:						Optional[Any]            =False
                    _lowerCamelCase			:						Union[str, Any]            =model_class(lowercase_					)
                    _lowerCamelCase			:						Optional[int]            =model(self._prepare_for_class(lowercase_     ,  lowercase_					)					)
                    _lowerCamelCase			:						Union[str, Any]            =len(lowercase_					)
                    self.assertEqual(config.output_hidden_states     ,  lowercase_					)
                    check_encoder_attentions_output(lowercase_					)

                    if self.is_encoder_decoder:
                           _lowerCamelCase			:						Any            =model_class(lowercase_					)
                           _lowerCamelCase			:						Optional[Any]            =model(self._prepare_for_class(lowercase_     ,  lowercase_					)					)
                           self.assertEqual(config.output_hidden_states     ,  lowercase_					)
                           check_decoder_attentions_output(lowercase_					)

                    # Check that output attentions can also be changed via the config
                    del inputs_dict["output_attentions"]
                    _lowerCamelCase			:						str            =True
                    _lowerCamelCase			:						Any            =model_class(lowercase_					)
                    _lowerCamelCase			:						List[Any]            =model(self._prepare_for_class(lowercase_     ,  lowercase_					)					)
                    self.assertEqual(config.output_hidden_states     ,  lowercase_					)
                    check_encoder_attentions_output(lowercase_					)

                    # Check attention is always last and order is fine
                    _lowerCamelCase			:						Optional[int]            =True
                    _lowerCamelCase			:						Dict            =True
                    _lowerCamelCase			:						Union[str, Any]            =model_class(lowercase_					)
                    _lowerCamelCase			:						List[Any]            =model(self._prepare_for_class(lowercase_     ,  lowercase_					)					)

                    self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)     ,  len(lowercase_					)					)
                    self.assertEqual(model.config.output_hidden_states     ,  lowercase_					)
                    check_encoder_attentions_output(lowercase_					)




@require_tf
class A    (  unittest.TestCase	):






      @slow
      def 			lowerCamelCase			(   self		:	str					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						Tuple            =TFConvBertModel.from_pretrained('YituTech/conv-bert-base'					)
             _lowerCamelCase			:						Dict            =tf.constant([[0, 1, 2, 3, 4, 5]]					)
             _lowerCamelCase			:						str            =model(lowercase_					)[0]

             _lowerCamelCase			:						Optional[Any]            =[1, 6, 768]
             self.assertEqual(output.shape     ,  lowercase_					)

             _lowerCamelCase			:						Optional[Any]            =tf.constant(
                 [
                     [
                         [-0.03475493, -0.4686034, -0.30638832],
                         [0.22637248, -0.26988646, -0.7423424],
                         [0.10324868, -0.45013508, -0.58280784],
                     ]
                 ]					)
             tf.debugging.assert_near(output[:, :3, :3]     ,  lowercase_     ,  atol=1E-4					)






import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase		   =			logging.get_logger(__name__)

lowerCamelCase		   =			'▁'

lowerCamelCase		   =			{'vocab_file': 'prophetnet.tokenizer'}

lowerCamelCase		   =			{
    'vocab_file': {
        'microsoft/xprophetnet-large-wiki100-cased': (
            'https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'
        ),
    }
}

lowerCamelCase		   =			{
    'microsoft/xprophetnet-large-wiki100-cased': {'do_lower_case': False},
}

lowerCamelCase		   =			{
    'microsoft/xprophetnet-large-wiki100-cased': 5_12,
}




def       a_ (       SCREAMING_SNAKE_CASE__   : Any	):
       '''simple docstring'''





       _lowerCamelCase			:						List[str]            =collections.OrderedDict()
       with open(SCREAMING_SNAKE_CASE__						,     'r'						,     encoding='utf-8'	) as reader:
              _lowerCamelCase			:						Dict            =reader.readlines()
       for index, token in enumerate(SCREAMING_SNAKE_CASE__	):
              _lowerCamelCase			:						int            =token.rstrip('\n'	)
              _lowerCamelCase			:						Tuple            =index
       return vocab




class A    (  UpperCamelCase_	):
      UpperCamelCase__				:    str =VOCAB_FILES_NAMES
      UpperCamelCase__				:    int =PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__				:    List[Any] =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__				:    int =['input_ids', 'attention_mask']






      def __init__(   self		:	Tuple     ,  lowercase_		:	Union[str, Any]     ,  lowercase_		:	List[str]="[SEP]"     ,  lowercase_		:	Any="[SEP]"     ,  lowercase_		:	Optional[int]="[SEP]"     ,  lowercase_		:	Optional[Any]="[UNK]"     ,  lowercase_		:	int="[PAD]"     ,  lowercase_		:	Union[str, Any]="[CLS]"     ,  lowercase_		:	Optional[int]="[MASK]"     ,  lowercase_		:	Optional[Dict[str, Any]] = None     ,  **lowercase_		:	str     ,  )   ->		None:
             """simple docstring"""




             _lowerCamelCase			:						int            ={} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 bos_token=lowercase_     ,  eos_token=lowercase_     ,  sep_token=lowercase_     ,  unk_token=lowercase_     ,  pad_token=lowercase_     ,  cls_token=lowercase_     ,  mask_token=lowercase_     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **lowercase_     ,  )

             try:
                    import sentencepiece as spm
             except ImportError:
                    logger.warning(
                        'You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'
                        ' pip install sentencepiece'					)
                    raise

             _lowerCamelCase			:						Optional[int]            =spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(str(lowercase_					)					)
             _lowerCamelCase			:						Dict            =vocab_file

             # Original fairseq vocab and spm vocab must be "aligned":
             # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
             # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
             # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
             # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

             # put special tokens and [unused] tokens into the vocab
             _lowerCamelCase			:						int            ={'[PAD]': 0, '[CLS]': 1, '[SEP]': 2, '[UNK]': 3, '[MASK]': 4}

             for i in range(10					):
                    _lowerCamelCase			:						Dict            =F'''[unused{i}]'''
                    _lowerCamelCase			:						List[str]            =5 + i

             # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
             _lowerCamelCase			:						Any            =12
             _lowerCamelCase			:						Dict            ={v: k for k, v in self.fairseq_tokens_to_ids.items()}
             for k in self.fairseq_tokens_to_ids.keys():
                    self.unique_no_split_tokens.append(lowercase_					)






      def __getstate__(   self		:	int					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            =self.__dict__.copy()
             _lowerCamelCase			:						Dict            =None
             return state






      def __setstate__(   self		:	str     ,  lowercase_		:	Union[str, Any]					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =d
             try:
                    import sentencepiece as spm
             except ImportError:
                    logger.warning(
                        'You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'
                        ' pip install sentencepiece'					)
                    raise

             # for backward compatibility
             if not hasattr(self     ,  'sp_model_kwargs'					):
                    _lowerCamelCase			:						Any            ={}

             _lowerCamelCase			:						Optional[Any]            =spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(self.vocab_file					)






      def 			lowerCamelCase			(   self		:	List[str]     ,  lowercase_		:	List[int]     ,  lowercase_		:	Optional[List[int]] = None     ,  lowercase_		:	bool = False					)   ->		List[int]:
             """simple docstring"""




             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=lowercase_     ,  token_ids_a=lowercase_     ,  already_has_special_tokens=lowercase_					)

             if token_ids_a is None:
                    return ([0] * len(lowercase_					)) + [1]
             return ([0] * len(lowercase_					)) + [1] + ([0] * len(lowercase_					)) + [1]






      def 			lowerCamelCase			(   self		:	List[str]     ,  lowercase_		:	List[int]     ,  lowercase_		:	Optional[List[int]] = None					)   ->		List[int]:
             """simple docstring"""




             _lowerCamelCase			:						Any            =[self.sep_token_id]

             if token_ids_a is None:
                    return len(token_ids_a + sep					) * [0]
             return len(token_ids_a + sep + sep + token_ids_a + sep					) * [0]






      @property
      def 			lowerCamelCase			(   self		:	Any					)   ->		Optional[Any]:
             """simple docstring"""




             return len(self.sp_model					) + self.fairseq_offset






      def 			lowerCamelCase			(   self		:	Dict					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            ={self.convert_ids_to_tokens(lowercase_					): i for i in range(self.vocab_size					)}
             vocab.update(self.added_tokens_encoder					)
             return vocab






      def 			lowerCamelCase			(   self		:	List[Any]     ,  lowercase_		:	str					)   ->		str:
             """simple docstring"""




             return self.sp_model.encode(lowercase_     ,  out_type=lowercase_					)






      def 			lowerCamelCase			(   self		:	List[str]     ,  lowercase_		:	Dict					)   ->		str:
             """simple docstring"""




             if token in self.fairseq_tokens_to_ids:
                    return self.fairseq_tokens_to_ids[token]
             _lowerCamelCase			:						Optional[int]            =self.sp_model.PieceToId(lowercase_					)

             # Need to return unknown token if the SP model returned 0
             return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






      def 			lowerCamelCase			(   self		:	str     ,  lowercase_		:	Dict					)   ->		Any:
             """simple docstring"""




             if index in self.fairseq_ids_to_tokens:
                    return self.fairseq_ids_to_tokens[index]
             return self.sp_model.IdToPiece(index - self.fairseq_offset					)






      def 			lowerCamelCase			(   self		:	Any     ,  lowercase_		:	List[str]					)   ->		Tuple:
             """simple docstring"""




             _lowerCamelCase			:						int            =''.join(lowercase_					).replace(lowercase_     ,  ' '					).strip()
             return out_string






      def 			lowerCamelCase			(   self		:	int     ,  lowercase_		:	str     ,  lowercase_		:	Optional[str] = None					)   ->		Tuple[str]:
             """simple docstring"""




             if not os.path.isdir(lowercase_					):
                    logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
                    return
             _lowerCamelCase			:						Dict            =os.path.join(
                 lowercase_     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

             if os.path.abspath(self.vocab_file					) != os.path.abspath(lowercase_					) and os.path.isfile(self.vocab_file					):
                    copyfile(self.vocab_file     ,  lowercase_					)
             elif not os.path.isfile(self.vocab_file					):
                    with open(lowercase_     ,  'wb'					) as fi:
                           _lowerCamelCase			:						Dict            =self.sp_model.serialized_model_proto()
                           fi.write(lowercase_					)

             return (out_vocab_file,)






      def 			lowerCamelCase			(   self		:	List[str]     ,  lowercase_		:	List[int]     ,  lowercase_		:	Optional[List[int]] = None					)   ->		List[int]:
             """simple docstring"""




             if token_ids_a is None:
                    return token_ids_a + [self.sep_token_id]
             _lowerCamelCase			:						Any            =[self.sep_token_id]
             return token_ids_a + sep + token_ids_a + sep


