



from __future__ import annotations

import unittest

from transformers import DistilBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers.models.distilbert.modeling_tf_distilbert import (
					    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
					    TFDistilBertForMaskedLM,
					    TFDistilBertForMultipleChoice,
					    TFDistilBertForQuestionAnswering,
					    TFDistilBertForSequenceClassification,
					    TFDistilBertForTokenClassification,
					    TFDistilBertModel,
					)
class      UpperCamelCase :







		"""simple docstring"""



		def __init__(  self		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Any						,)    ->   Optional[Any]:


						'''simple docstring'''
						A	 =      parent
						A	 =      1_3
						A	 =      7
						A	 =      True
						A	 =      True
						A	 =      False
						A	 =      True
						A	 =      9_9
						A	 =      3_2
						A	 =      2
						A	 =      4
						A	 =      3_7
						A	 =      """gelu"""
						A	 =      0.1
						A	 =      0.1
						A	 =      5_1_2
						A	 =      1_6
						A	 =      2
						A	 =      0.02
						A	 =      3
						A	 =      4
						A	 =      None
		def       A(  self		:       Optional[int]		)    ->   List[str]:


						'''simple docstring'''
						A	 =      ids_tensor([self.batch_size, self.seq_length]						,self.vocab_size		)

						A	 =      None
						if self.use_input_mask:
										A	 =      random_attention_mask([self.batch_size, self.seq_length]		)

						A	 =      None
						A	 =      None
						A	 =      None
						if self.use_labels:
										A	 =      ids_tensor([self.batch_size]						,self.type_sequence_label_size		)
										A	 =      ids_tensor([self.batch_size, self.seq_length]						,self.num_labels		)
										A	 =      ids_tensor([self.batch_size]						,self.num_choices		)

						A	 =      DistilBertConfig(
						    vocab_size=self.vocab_size						,dim=self.hidden_size						,n_layers=self.num_hidden_layers						,n_heads=self.num_attention_heads						,hidden_dim=self.intermediate_size						,hidden_act=self.hidden_act						,dropout=self.hidden_dropout_prob						,attention_dropout=self.attention_probs_dropout_prob						,max_position_embeddings=self.max_position_embeddings						,initializer_range=self.initializer_range						,)

						return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
		def       A(  self		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       Optional[Any]		)    ->   List[str]:


						'''simple docstring'''
						A	 =      TFDistilBertModel(config=lowercase__		)
						A	 =      {"""input_ids""": input_ids, """attention_mask""": input_mask}

						A	 =      model(lowercase__		)

						A	 =      [input_ids, input_mask]

						A	 =      model(lowercase__		)

						self.parent.assertEqual(result.last_hidden_state.shape						,(self.batch_size, self.seq_length, self.hidden_size)		)
		def       A(  self		:       str						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]		)    ->   Any:


						'''simple docstring'''
						A	 =      TFDistilBertForMaskedLM(config=lowercase__		)
						A	 =      {"""input_ids""": input_ids, """attention_mask""": input_mask}
						A	 =      model(lowercase__		)
						self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.seq_length, self.vocab_size)		)
		def       A(  self		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Any						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Tuple		)    ->   Optional[Any]:


						'''simple docstring'''
						A	 =      TFDistilBertForQuestionAnswering(config=lowercase__		)
						A	 =      {
						    """input_ids""": input_ids,
						    """attention_mask""": input_mask,
						}
						A	 =      model(lowercase__		)
						self.parent.assertEqual(result.start_logits.shape						,(self.batch_size, self.seq_length)		)
						self.parent.assertEqual(result.end_logits.shape						,(self.batch_size, self.seq_length)		)
		def       A(  self		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Optional[int]		)    ->   Dict:


						'''simple docstring'''
						A	 =      self.num_labels
						A	 =      TFDistilBertForSequenceClassification(lowercase__		)
						A	 =      {"""input_ids""": input_ids, """attention_mask""": input_mask}
						A	 =      model(lowercase__		)
						self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.num_labels)		)
		def       A(  self		:       Tuple						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Any						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]		)    ->   Optional[int]:


						'''simple docstring'''
						A	 =      self.num_choices
						A	 =      TFDistilBertForMultipleChoice(lowercase__		)
						A	 =      tf.tile(tf.expand_dims(lowercase__						,1		)						,(1, self.num_choices, 1)		)
						A	 =      tf.tile(tf.expand_dims(lowercase__						,1		)						,(1, self.num_choices, 1)		)
						A	 =      {
						    """input_ids""": multiple_choice_inputs_ids,
						    """attention_mask""": multiple_choice_input_mask,
						}
						A	 =      model(lowercase__		)
						self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.num_choices)		)
		def       A(  self		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       Any						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       str		)    ->   int:


						'''simple docstring'''
						A	 =      self.num_labels
						A	 =      TFDistilBertForTokenClassification(lowercase__		)
						A	 =      {"""input_ids""": input_ids, """attention_mask""": input_mask}
						A	 =      model(lowercase__		)
						self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.seq_length, self.num_labels)		)







		def       A(  self		:       Optional[Any]		)    ->   Optional[Any]:


						'''simple docstring'''
						A	 =      self.prepare_config_and_inputs()
						(A)	 =      config_and_inputs
						A	 =      {"""input_ids""": input_ids, """attention_mask""": input_mask}
						return config, inputs_dict


@require_tf
class      UpperCamelCase (				_UpperCAmelCase			,       _UpperCAmelCase			,       unittest.TestCase ):







		"""simple docstring"""



		snake_case    =   (
		    (
		        TFDistilBertModel,
		        TFDistilBertForMaskedLM,
		        TFDistilBertForQuestionAnswering,
		        TFDistilBertForSequenceClassification,
		        TFDistilBertForTokenClassification,
		        TFDistilBertForMultipleChoice,
		    )
		    if is_tf_available()
		    else None
		)
		snake_case    =   (
		    {
		        """feature-extraction""": TFDistilBertModel,
		        """fill-mask""": TFDistilBertForMaskedLM,
		        """question-answering""": TFDistilBertForQuestionAnswering,
		        """text-classification""": TFDistilBertForSequenceClassification,
		        """token-classification""": TFDistilBertForTokenClassification,
		        """zero-shot""": TFDistilBertForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)
		snake_case    =   False
		snake_case    =   False
		def       A(  self		:       Optional[Any]		)    ->   Dict:


						'''simple docstring'''
						A	 =      TFDistilBertModelTester(self		)
						A	 =      ConfigTester(self						,config_class=lowercase__						,dim=3_7		)
		def       A(  self		:       Tuple		)    ->   Optional[Any]:


						'''simple docstring'''
						self.config_tester.run_common_tests()
		def       A(  self		:       Any		)    ->   List[str]:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_model(*lowercase__		)
		def       A(  self		:       int		)    ->   List[str]:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_masked_lm(*lowercase__		)
		def       A(  self		:       Union[str, Any]		)    ->   Tuple:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_question_answering(*lowercase__		)
		def       A(  self		:       Dict		)    ->   Union[str, Any]:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_sequence_classification(*lowercase__		)
		def       A(  self		:       Optional[int]		)    ->   List[Any]:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_multiple_choice(*lowercase__		)
		def       A(  self		:       int		)    ->   Dict:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_token_classification(*lowercase__		)







		@slow
		def       A(  self		:       List[str]		)    ->   Union[str, Any]:


						'''simple docstring'''
						for model_name in list(TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]		):
										A	 =      TFDistilBertModel.from_pretrained(lowercase__		)
										self.assertIsNotNone(lowercase__		)


@require_tf
class      UpperCamelCase (				unittest.TestCase ):







		"""simple docstring"""



		@slow
		def       A(  self		:       str		)    ->   Dict:


						'''simple docstring'''
						A	 =      TFDistilBertModel.from_pretrained('distilbert-base-uncased'		)
						A	 =      tf.constant([[0, 1, 2, 3, 4, 5]]		)
						A	 =      model(lowercase__		)[0]

						A	 =      [1, 6, 7_6_8]
						self.assertEqual(output.shape						,lowercase__		)

						A	 =      tf.constant(
						    [
						        [
						            [0.19261885, -0.13732955, 0.4119799],
						            [0.22150156, -0.07422661, 0.39037204],
						            [0.22756018, -0.0896414, 0.3701467],
						        ]
						    ]		)
						tf.debugging.assert_near(output[:, :3, :3]						,lowercase__						,atol=1E-4		)













import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


lowerCAmelCase_           = logging.get_logger(__name__)

lowerCAmelCase_           = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_           = {
    'tokenizer_file': {
        'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json',
    },
}

lowerCAmelCase_           = {
    'gpt-neox-20b': 2_048,
}
class      UpperCamelCase (				snake_case__ ):







		"""simple docstring"""



		snake_case    =   VOCAB_FILES_NAMES
		snake_case    =   PRETRAINED_VOCAB_FILES_MAP
		snake_case    =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		snake_case    =   ["input_ids", "attention_mask"]
		def __init__(  self		:       str						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]=None						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]=None						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]=None						,_SCREAMING_SNAKE_CASE		:       Dict="<|endoftext|>"						,_SCREAMING_SNAKE_CASE		:       int="<|endoftext|>"						,_SCREAMING_SNAKE_CASE		:       Optional[int]="<|endoftext|>"						,_SCREAMING_SNAKE_CASE		:       List[Any]=False						,**_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,)    ->   int:


						'''simple docstring'''
						super().__init__(
						    _SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE						,tokenizer_file=_SCREAMING_SNAKE_CASE						,unk_token=_SCREAMING_SNAKE_CASE						,bos_token=_SCREAMING_SNAKE_CASE						,eos_token=_SCREAMING_SNAKE_CASE						,add_prefix_space=_SCREAMING_SNAKE_CASE						,**_SCREAMING_SNAKE_CASE						,)

						A	 =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
						if pre_tok_state.get('add_prefix_space'						,_SCREAMING_SNAKE_CASE		) != add_prefix_space:
										A	 =      getattr(_SCREAMING_SNAKE_CASE						,pre_tok_state.pop('type'		)		)
										A	 =      add_prefix_space
										A	 =      pre_tok_class(**_SCREAMING_SNAKE_CASE		)

						A	 =      add_prefix_space
		def       A(  self		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Optional[str] = None		)    ->   Tuple[str]:


						'''simple docstring'''
						A	 =      self._tokenizer.model.save(_SCREAMING_SNAKE_CASE						,name=_SCREAMING_SNAKE_CASE		)
						return tuple(_SCREAMING_SNAKE_CASE		)







		def       A(  self		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       "Conversation"		)    ->   List[int]:


						'''simple docstring'''
						A	 =      []
						for is_user, text in conversation.iter_texts():
										input_ids.extend(self.encode(_SCREAMING_SNAKE_CASE						,add_special_tokens=_SCREAMING_SNAKE_CASE		) + [self.eos_token_id]		)

						if len(_SCREAMING_SNAKE_CASE		) > self.model_max_length:
										A	 =      input_ids[-self.model_max_length :]
						return input_ids





