

import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_			      =      logging.get_logger(__name__)

lowerCAmelCase_			      =      {'''vocab_file''': '''sentencepiece.model'''}

lowerCAmelCase_			      =      {
    '''vocab_file''': {
        '''google/rembert''': '''https://huggingface.co/google/rembert/resolve/main/sentencepiece.model''',
    },
}

lowerCAmelCase_			      =      {
    '''google/rembert''': 2_56,
}





class       snake_case_  (						__A   ):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE	:  Tuple		     =							VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE	:  str		     =							PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE	:  int		     =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







			def __init__(	self					:							Tuple				,  _UpperCamelCase					:							Any				,  _UpperCamelCase					:							Tuple=False				,  _UpperCamelCase					:							Dict=True				,  _UpperCamelCase					:							Dict=True				,  _UpperCamelCase					:							Tuple="[CLS]"				,  _UpperCamelCase					:							Dict="[SEP]"				,  _UpperCamelCase					:							Tuple="[UNK]"				,  _UpperCamelCase					:							List[str]="[SEP]"				,  _UpperCamelCase					:							Tuple="[PAD]"				,  _UpperCamelCase					:							Optional[Any]="[CLS]"				,  _UpperCamelCase					:							List[str]="[MASK]"				,  **_UpperCamelCase					:							List[Any]				,  ) ->Tuple:
									super().__init__(
									    do_lower_case=_UpperCamelCase				,  remove_space=_UpperCamelCase				,  keep_accents=_UpperCamelCase				,  bos_token=_UpperCamelCase				,  eos_token=_UpperCamelCase				,  unk_token=_UpperCamelCase				,  sep_token=_UpperCamelCase				,  pad_token=_UpperCamelCase				,  cls_token=_UpperCamelCase				,  mask_token=_UpperCamelCase				,  **_UpperCamelCase				,  )

									snake_case_  =      do_lower_case
									snake_case_  =      remove_space
									snake_case_  =      keep_accents
									snake_case_  =      vocab_file

									snake_case_  =      spm.SentencePieceProcessor()
									self.sp_model.Load(_UpperCamelCase				)







			@property
			def 							snake_case__(	self					:							List[Any]				) ->Dict:
									return len(self.sp_model				)







			def 							snake_case__(	self					:							Tuple				) ->Tuple:
									snake_case_  =      {self.convert_ids_to_tokens(_UpperCamelCase				): i for i in range(self.vocab_size				)}
									vocab.update(self.added_tokens_encoder				)
									return vocab







			def __getstate__(	self					:							Dict				) ->Optional[Any]:
									snake_case_  =      self.__dict__.copy()
									snake_case_  =      None
									return state







			def __setstate__(	self					:							Any				,  _UpperCamelCase					:							List[Any]				) ->Optional[Any]:
									snake_case_  =      d
									snake_case_  =      spm.SentencePieceProcessor()
									self.sp_model.Load(self.vocab_file				)







			def 							snake_case__(	self					:							Tuple				,  _UpperCamelCase					:							Any				,  _UpperCamelCase					:							Any=False				) ->List[Any]:
									snake_case_  =      self.sp_model.EncodeAsPieces(_UpperCamelCase				)
									return pieces







			def 							snake_case__(	self					:							str				,  _UpperCamelCase					:							Optional[int]				) ->Tuple:
									return self.sp_model.PieceToId(_UpperCamelCase				)







			def 							snake_case__(	self					:							Tuple				,  _UpperCamelCase					:							int				) ->List[str]:
									return self.sp_model.IdToPiece(_UpperCamelCase				)







			def 							snake_case__(	self					:							Union[str, Any]				,  _UpperCamelCase					:							Dict				) ->Dict:
									snake_case_  =      self.sp_model.decode_pieces(_UpperCamelCase				)
									return out_string







			def 							snake_case__(	self					:							Any				,  _UpperCamelCase					:							List[int]				,  _UpperCamelCase					:							Optional[List[int]] = None				) ->List[int]:
									snake_case_  =      [self.sep_token_id]
									snake_case_  =      [self.cls_token_id]
									if token_ids_a is None:
															return cls + token_ids_a + sep
									return cls + token_ids_a + sep + token_ids_a + sep







			def 							snake_case__(	self					:							Tuple				,  _UpperCamelCase					:							List[int]				,  _UpperCamelCase					:							Optional[List[int]] = None				,  _UpperCamelCase					:							bool = False				) ->List[int]:

									if already_has_special_tokens:
															if token_ids_a is not None:
																					raise ValueError(
																					    '''You should not supply a second sequence if the provided sequence of '''
																					    '''ids is already formatted with special tokens for the model.'''				)
															return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

									if token_ids_a is not None:
															return [1] + ([0] * len(_UpperCamelCase				)) + [1] + ([0] * len(_UpperCamelCase				)) + [1]
									return [1] + ([0] * len(_UpperCamelCase				)) + [1]







			def 							snake_case__(	self					:							List[Any]				,  _UpperCamelCase					:							List[int]				,  _UpperCamelCase					:							Optional[List[int]] = None				) ->List[int]:
									snake_case_  =      [self.sep_token_id]
									snake_case_  =      [self.cls_token_id]

									if token_ids_a is None:
															return len(cls + token_ids_a + sep				) * [0]
									return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







			def 							snake_case__(	self					:							List[str]				,  _UpperCamelCase					:							str				,  _UpperCamelCase					:							Optional[str] = None				) ->Tuple[str]:
									if not os.path.isdir(_UpperCamelCase				):
															logger.error('''Vocabulary path ({}) should be a directory'''.format(_UpperCamelCase				)				)
															return
									snake_case_  =      os.path.join(
									    _UpperCamelCase				,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

									if os.path.abspath(self.vocab_file				) != os.path.abspath(_UpperCamelCase				):
															copyfile(self.vocab_file				,  _UpperCamelCase				)

									return (out_vocab_file,)







"""simple docstring"""
from __future__ import annotations

import unittest

from transformers import DistilBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers.models.distilbert.modeling_tf_distilbert import (
			    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
			    TFDistilBertForMaskedLM,
			    TFDistilBertForMultipleChoice,
			    TFDistilBertForQuestionAnswering,
			    TFDistilBertForSequenceClassification,
			    TFDistilBertForTokenClassification,
			    TFDistilBertModel,
			)







class      UpperCamelCase_			:





	def __init__( self   :					Optional[Any]      ,      lowerCAmelCase_   :					int      ,      )		->     Tuple:
				UpperCAmelCase_	:					Optional[int]     	=   parent
				UpperCAmelCase_	:					Tuple     	=   13
				UpperCAmelCase_	:					int     	=   7
				UpperCAmelCase_	:					Optional[Any]     	=   True
				UpperCAmelCase_	:					int     	=   True
				UpperCAmelCase_	:					Tuple     	=   False
				UpperCAmelCase_	:					Any     	=   True
				UpperCAmelCase_	:					List[Any]     	=   99
				UpperCAmelCase_	:					Any     	=   32
				UpperCAmelCase_	:					str     	=   2
				UpperCAmelCase_	:					Tuple     	=   4
				UpperCAmelCase_	:					Dict     	=   37
				UpperCAmelCase_	:					Optional[int]     	=   "gelu"
				UpperCAmelCase_	:					List[str]     	=   0.1
				UpperCAmelCase_	:					Tuple     	=   0.1
				UpperCAmelCase_	:					Any     	=   512
				UpperCAmelCase_	:					List[str]     	=   16
				UpperCAmelCase_	:					Optional[Any]     	=   2
				UpperCAmelCase_	:					Union[str, Any]     	=   0.0_2
				UpperCAmelCase_	:					str     	=   3
				UpperCAmelCase_	:					Optional[int]     	=   4
				UpperCAmelCase_	:					Dict     	=   None





	def 	_SCREAMING_SNAKE_CASE ( self   :					int		)		->     List[str]:
				UpperCAmelCase_	:					Tuple     	=   ids_tensor([self.batch_size, self.seq_length]      ,      self.vocab_size		)

				UpperCAmelCase_	:					Optional[Any]     	=   None
				if self.use_input_mask:
							UpperCAmelCase_	:					Optional[int]     	=   random_attention_mask([self.batch_size, self.seq_length]		)

				UpperCAmelCase_	:					Tuple     	=   None
				UpperCAmelCase_	:					List[str]     	=   None
				UpperCAmelCase_	:					Dict     	=   None
				if self.use_labels:
							UpperCAmelCase_	:					Optional[Any]     	=   ids_tensor([self.batch_size]      ,      self.type_sequence_label_size		)
							UpperCAmelCase_	:					Any     	=   ids_tensor([self.batch_size, self.seq_length]      ,      self.num_labels		)
							UpperCAmelCase_	:					Tuple     	=   ids_tensor([self.batch_size]      ,      self.num_choices		)

				UpperCAmelCase_	:					Dict     	=   DistilBertConfig(
				    vocab_size=self.vocab_size      ,      dim=self.hidden_size      ,      n_layers=self.num_hidden_layers      ,      n_heads=self.num_attention_heads      ,      hidden_dim=self.intermediate_size      ,      hidden_act=self.hidden_act      ,      dropout=self.hidden_dropout_prob      ,      attention_dropout=self.attention_probs_dropout_prob      ,      max_position_embeddings=self.max_position_embeddings      ,      initializer_range=self.initializer_range      ,      )

				return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]      ,      lowerCAmelCase_   :					Tuple      ,      lowerCAmelCase_   :					Tuple      ,      lowerCAmelCase_   :					List[Any]      ,      lowerCAmelCase_   :					List[Any]      ,      lowerCAmelCase_   :					Tuple      ,      lowerCAmelCase_   :					Any		)		->     str:
				UpperCAmelCase_	:					Optional[Any]     	=   TFDistilBertModel(config=lowerCAmelCase_		)
				UpperCAmelCase_	:					Any     	=   {"input_ids": input_ids, "attention_mask": input_mask}

				UpperCAmelCase_	:					str     	=   model(lowerCAmelCase_		)

				UpperCAmelCase_	:					List[Any]     	=   [input_ids, input_mask]

				UpperCAmelCase_	:					Optional[int]     	=   model(lowerCAmelCase_		)

				self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.hidden_size)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Tuple      ,      lowerCAmelCase_   :					Optional[Any]      ,      lowerCAmelCase_   :					Optional[int]      ,      lowerCAmelCase_   :					Dict      ,      lowerCAmelCase_   :					Optional[Any]      ,      lowerCAmelCase_   :					str      ,      lowerCAmelCase_   :					Union[str, Any]		)		->     List[str]:
				UpperCAmelCase_	:					Dict     	=   TFDistilBertForMaskedLM(config=lowerCAmelCase_		)
				UpperCAmelCase_	:					int     	=   {"input_ids": input_ids, "attention_mask": input_mask}
				UpperCAmelCase_	:					List[str]     	=   model(lowerCAmelCase_		)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.vocab_size)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]      ,      lowerCAmelCase_   :					int      ,      lowerCAmelCase_   :					List[str]      ,      lowerCAmelCase_   :					Dict      ,      lowerCAmelCase_   :					int      ,      lowerCAmelCase_   :					List[Any]      ,      lowerCAmelCase_   :					Dict		)		->     Dict:
				UpperCAmelCase_	:					Optional[Any]     	=   TFDistilBertForQuestionAnswering(config=lowerCAmelCase_		)
				UpperCAmelCase_	:					List[Any]     	=   {
				    "input_ids": input_ids,
				    "attention_mask": input_mask,
				}
				UpperCAmelCase_	:					Any     	=   model(lowerCAmelCase_		)
				self.parent.assertEqual(result.start_logits.shape      ,      (self.batch_size, self.seq_length)		)
				self.parent.assertEqual(result.end_logits.shape      ,      (self.batch_size, self.seq_length)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					int      ,      lowerCAmelCase_   :					Optional[int]      ,      lowerCAmelCase_   :					Optional[int]      ,      lowerCAmelCase_   :					str      ,      lowerCAmelCase_   :					List[Any]      ,      lowerCAmelCase_   :					Dict      ,      lowerCAmelCase_   :					Optional[Any]		)		->     Optional[int]:
				UpperCAmelCase_	:					List[Any]     	=   self.num_labels
				UpperCAmelCase_	:					Tuple     	=   TFDistilBertForSequenceClassification(lowerCAmelCase_		)
				UpperCAmelCase_	:					int     	=   {"input_ids": input_ids, "attention_mask": input_mask}
				UpperCAmelCase_	:					Optional[Any]     	=   model(lowerCAmelCase_		)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.num_labels)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[int]      ,      lowerCAmelCase_   :					str      ,      lowerCAmelCase_   :					Optional[int]      ,      lowerCAmelCase_   :					Optional[Any]      ,      lowerCAmelCase_   :					Tuple      ,      lowerCAmelCase_   :					Tuple      ,      lowerCAmelCase_   :					Union[str, Any]		)		->     Optional[Any]:
				UpperCAmelCase_	:					Union[str, Any]     	=   self.num_choices
				UpperCAmelCase_	:					List[Any]     	=   TFDistilBertForMultipleChoice(lowerCAmelCase_		)
				UpperCAmelCase_	:					List[Any]     	=   tf.tile(tf.expand_dims(lowerCAmelCase_      ,      1		)      ,      (1, self.num_choices, 1)		)
				UpperCAmelCase_	:					List[Any]     	=   tf.tile(tf.expand_dims(lowerCAmelCase_      ,      1		)      ,      (1, self.num_choices, 1)		)
				UpperCAmelCase_	:					Any     	=   {
				    "input_ids": multiple_choice_inputs_ids,
				    "attention_mask": multiple_choice_input_mask,
				}
				UpperCAmelCase_	:					List[Any]     	=   model(lowerCAmelCase_		)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.num_choices)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Tuple      ,      lowerCAmelCase_   :					Union[str, Any]      ,      lowerCAmelCase_   :					Tuple      ,      lowerCAmelCase_   :					List[Any]      ,      lowerCAmelCase_   :					Any      ,      lowerCAmelCase_   :					Union[str, Any]      ,      lowerCAmelCase_   :					Union[str, Any]		)		->     Tuple:
				UpperCAmelCase_	:					Union[str, Any]     	=   self.num_labels
				UpperCAmelCase_	:					List[str]     	=   TFDistilBertForTokenClassification(lowerCAmelCase_		)
				UpperCAmelCase_	:					int     	=   {"input_ids": input_ids, "attention_mask": input_mask}
				UpperCAmelCase_	:					Optional[Any]     	=   model(lowerCAmelCase_		)
				self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.num_labels)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any		)		->     Dict:
				UpperCAmelCase_	:					Optional[int]     	=   self.prepare_config_and_inputs()
				((UpperCAmelCase_)	,      (UpperCAmelCase_)	,      (UpperCAmelCase_)	,      (UpperCAmelCase_)	,      (UpperCAmelCase_)	,      (UpperCAmelCase_))	:					str     	=   config_and_inputs
				UpperCAmelCase_	:					List[str]     	=   {"input_ids": input_ids, "attention_mask": input_mask}
				return config, inputs_dict







@require_tf
class      UpperCamelCase_			(__A							,		__A							,		unittest.TestCase	):
	__magic_name__       =							(
	    (
	        TFDistilBertModel,
	        TFDistilBertForMaskedLM,
	        TFDistilBertForQuestionAnswering,
	        TFDistilBertForSequenceClassification,
	        TFDistilBertForTokenClassification,
	        TFDistilBertForMultipleChoice,
	    )
	    if is_tf_available()
	    else None
	)
	__magic_name__       =							(
	    {
	        '''feature-extraction''': TFDistilBertModel,
	        '''fill-mask''': TFDistilBertForMaskedLM,
	        '''question-answering''': TFDistilBertForQuestionAnswering,
	        '''text-classification''': TFDistilBertForSequenceClassification,
	        '''token-classification''': TFDistilBertForTokenClassification,
	        '''zero-shot''': TFDistilBertForSequenceClassification,
	    }
	    if is_tf_available()
	    else {}
	)
	__magic_name__       =							False
	__magic_name__       =							False





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Tuple:
				UpperCAmelCase_	:					Any     	=   TFDistilBertModelTester(self		)
				UpperCAmelCase_	:					List[str]     	=   ConfigTester(self      ,      config_class=lowerCAmelCase_      ,      dim=37		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     Optional[int]:
				self.config_tester.run_common_tests()





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     int:
				UpperCAmelCase_	:					str     	=   self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_distilbert_model(*lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					int		)		->     Union[str, Any]:
				UpperCAmelCase_	:					List[Any]     	=   self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_distilbert_for_masked_lm(*lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					int		)		->     Optional[Any]:
				UpperCAmelCase_	:					str     	=   self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_distilbert_for_question_answering(*lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any		)		->     Union[str, Any]:
				UpperCAmelCase_	:					Optional[Any]     	=   self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_distilbert_for_sequence_classification(*lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     int:
				UpperCAmelCase_	:					Optional[Any]     	=   self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_distilbert_for_multiple_choice(*lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Tuple:
				UpperCAmelCase_	:					str     	=   self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_distilbert_for_token_classification(*lowerCAmelCase_		)





	@slow
	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     Dict:
				for model_name in list(TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]		):
							UpperCAmelCase_	:					Tuple     	=   TFDistilBertModel.from_pretrained(lowerCAmelCase_		)
							self.assertIsNotNone(lowerCAmelCase_		)







@require_tf
class      UpperCamelCase_			(unittest.TestCase	):





	@slow
	def 	_SCREAMING_SNAKE_CASE ( self   :					Dict		)		->     Optional[int]:
				UpperCAmelCase_	:					Optional[int]     	=   TFDistilBertModel.from_pretrained("distilbert-base-uncased"		)
				UpperCAmelCase_	:					Tuple     	=   tf.constant([[0, 1, 2, 3, 4, 5]]		)
				UpperCAmelCase_	:					Dict     	=   model(lowerCAmelCase_		)[0]

				UpperCAmelCase_	:					Union[str, Any]     	=   [1, 6, 768]
				self.assertEqual(output.shape      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Dict     	=   tf.constant(
				    [
				        [
				            [0.1_9_2_6_1_8_8_5, -0.1_3_7_3_2_9_5_5, 0.4_1_1_9_7_9_9],
				            [0.2_2_1_5_0_1_5_6, -0.0_7_4_2_2_6_6_1, 0.3_9_0_3_7_2_0_4],
				            [0.2_2_7_5_6_0_1_8, -0.0_8_9_6_4_1_4, 0.3_7_0_1_4_6_7],
				        ]
				    ]		)
				tf.debugging.assert_near(output[:, :3, :3]      ,      lowerCAmelCase_      ,      atol=1e-4		)



