"""simple docstring"""



from __future__ import annotations

import unittest

from transformers import BlenderbotSmallConfig, BlenderbotSmallTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel
@require_tf
class 						__UpperCAmelCase    :







			'''simple docstring'''
			_UpperCamelCase  :						int					     =				BlenderbotSmallConfig
			_UpperCamelCase  :						Union[str, Any]					     =				{}
			_UpperCamelCase  :						Tuple					     =				'gelu'





			def __init__(			self   ,      snake_case_   ,      snake_case_=13   ,      snake_case_=7   ,      snake_case_=True   ,      snake_case_=False   ,      snake_case_=99   ,      snake_case_=32   ,      snake_case_=2   ,      snake_case_=4   ,      snake_case_=37   ,      snake_case_=0.1   ,      snake_case_=0.1   ,      snake_case_=20   ,      snake_case_=2   ,      snake_case_=1   ,      snake_case_=0   ,      ):







										'''simple docstring'''


										A__       :     int								=			parent
										A__       :     Union[str, Any]								=			batch_size
										A__       :     Dict								=			seq_length
										A__       :     int								=			is_training
										A__       :     Any								=			use_labels
										A__       :     Tuple								=			vocab_size
										A__       :     str								=			hidden_size
										A__       :     Any								=			num_hidden_layers
										A__       :     Optional[int]								=			num_attention_heads
										A__       :     int								=			intermediate_size

										A__       :     str								=			hidden_dropout_prob
										A__       :     List[str]								=			attention_probs_dropout_prob
										A__       :     Tuple								=			max_position_embeddings
										A__       :     int								=			eos_token_id
										A__       :     Tuple								=			pad_token_id
										A__       :     Any								=			bos_token_id





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     List[Any]								=			ids_tensor([self.batch_size, self.seq_length - 1]   ,      self.vocab_size							)
										A__       :     Any								=			tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size							)   ,      1							)
										A__       :     Optional[int]								=			tf.concat([input_ids, eos_tensor]   ,      axis=1							)

										A__       :     Tuple								=			ids_tensor([self.batch_size, self.seq_length]   ,      self.vocab_size							)

										A__       :     int								=			self.config_cls(
										    vocab_size=self.vocab_size   ,      d_model=self.hidden_size   ,      encoder_layers=self.num_hidden_layers   ,      decoder_layers=self.num_hidden_layers   ,      encoder_attention_heads=self.num_attention_heads   ,      decoder_attention_heads=self.num_attention_heads   ,      encoder_ffn_dim=self.intermediate_size   ,      decoder_ffn_dim=self.intermediate_size   ,      dropout=self.hidden_dropout_prob   ,      attention_dropout=self.attention_probs_dropout_prob   ,      max_position_embeddings=self.max_position_embeddings   ,      eos_token_ids=[2]   ,      bos_token_id=self.bos_token_id   ,      pad_token_id=self.pad_token_id   ,      decoder_start_token_id=self.pad_token_id   ,      **self.config_updates   ,      )
										A__       :     Union[str, Any]								=			prepare_blenderbot_small_inputs_dict(snake_case__   ,      snake_case__   ,      snake_case__							)
										return config, inputs_dict





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_							):







										'''simple docstring'''


										A__       :     str								=			TFBlenderbotSmallModel(config=snake_case__							).get_decoder()
										A__       :     str								=			inputs_dict['input_ids']

										A__       :     Tuple								=			input_ids[:1, :]
										A__       :     str								=			inputs_dict['attention_mask'][:1, :]
										A__       :     Optional[int]								=			inputs_dict['head_mask']
										A__       :     Union[str, Any]								=			1

										# first forward pass
										A__       :     List[str]								=			model(snake_case__   ,      attention_mask=snake_case__   ,      head_mask=snake_case__   ,      use_cache=snake_case__							)

										A__       :     List[str]								=			outputs.to_tuple()

										# create hypothetical next token and extent to next_input_ids
										A__       :     Dict								=			ids_tensor((self.batch_size, 3)   ,      config.vocab_size							)
										A__       :     int								=			tf.cast(ids_tensor((self.batch_size, 3)   ,      2							)   ,      tf.inta							)

										# append to next input_ids and
										A__       :     Tuple								=			tf.concat([input_ids, next_tokens]   ,      axis=-1							)
										A__       :     Dict								=			tf.concat([attention_mask, next_attn_mask]   ,      axis=-1							)

										A__       :     List[Any]								=			model(snake_case__   ,      attention_mask=snake_case__							)[0]
										A__       :     Tuple								=			model(snake_case__   ,      attention_mask=snake_case__   ,      past_key_values=snake_case__							)[0]

										self.parent.assertEqual(next_tokens.shape[1]   ,      output_from_past.shape[1]							)

										# select random slice
										A__       :     str								=			int(ids_tensor((1,)   ,      output_from_past.shape[-1]							)							)
										A__       :     str								=			output_from_no_past[:, -3:, random_slice_idx]
										A__       :     Tuple								=			output_from_past[:, :, random_slice_idx]

										# test that outputs are equal for slice
										tf.debugging.assert_near(snake_case__   ,      snake_case__   ,      rtol=1E-3							)





def  _A(		lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase=None			,  lowerCAmelCase=None			,  lowerCAmelCase=None			,  lowerCAmelCase=None			,  lowerCAmelCase=None			,  ):







							if attention_mask is None:
														A__       :     Optional[Any]								=			tf.cast(tf.math.not_equal(_A			,  config.pad_token_id							)			,  tf.inta							)
							if decoder_attention_mask is None:
														A__       :     Dict								=			tf.concat(
														    [
														        tf.ones(decoder_input_ids[:, :1].shape			,  dtype=tf.inta							),
														        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]			,  config.pad_token_id							)			,  tf.inta							),
														    ]			,  axis=-1			,  )
							if head_mask is None:
														A__       :     int								=			tf.ones((config.encoder_layers, config.encoder_attention_heads)							)
							if decoder_head_mask is None:
														A__       :     Any								=			tf.ones((config.decoder_layers, config.decoder_attention_heads)							)
							if cross_attn_head_mask is None:
														A__       :     Tuple								=			tf.ones((config.decoder_layers, config.decoder_attention_heads)							)
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": decoder_attention_mask,
							    "head_mask": head_mask,
							    "decoder_head_mask": decoder_head_mask,
							    "cross_attn_head_mask": cross_attn_head_mask,
							}
@require_tf
class 						__UpperCAmelCase    (SCREAMING_SNAKE_CASE_							,   SCREAMING_SNAKE_CASE_							,   unittest.TestCase     ):







			'''simple docstring'''
			_UpperCamelCase  :						int					     =				(
			    (TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel) if is_tf_available() else ()
			)
			_UpperCamelCase  :						Optional[int]					     =				(TFBlenderbotSmallForConditionalGeneration,) if is_tf_available() else ()
			_UpperCamelCase  :						List[str]					     =				(
			    {
			        'conversational': TFBlenderbotSmallForConditionalGeneration,
			        'feature-extraction': TFBlenderbotSmallModel,
			        'summarization': TFBlenderbotSmallForConditionalGeneration,
			        'text2text-generation': TFBlenderbotSmallForConditionalGeneration,
			        'translation': TFBlenderbotSmallForConditionalGeneration,
			    }
			    if is_tf_available()
			    else {}
			)
			_UpperCamelCase  :						List[Any]					     =				True
			_UpperCamelCase  :						Union[str, Any]					     =				False
			_UpperCamelCase  :						List[str]					     =				False





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Union[str, Any]								=			TFBlenderbotSmallModelTester(self							)
										A__       :     Any								=			ConfigTester(self   ,      config_class=snake_case__							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										self.config_tester.run_common_tests()





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     int								=			self.model_tester.prepare_config_and_inputs_for_common()
										self.model_tester.check_decoder_model_past_large_inputs(*snake_case__							)





@require_tokenizers
@require_tf
class 						__UpperCAmelCase    (unittest.TestCase     ):







			'''simple docstring'''
			_UpperCamelCase  :						int					     =				[
			    'Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like '
			    '  i\'m going to throw up.\nand why is that?'
			]
			_UpperCamelCase  :						Union[str, Any]					     =				'facebook/blenderbot_small-90M'





			@cached_property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return BlenderbotSmallTokenizer.from_pretrained("""facebook/blenderbot-90M"""							)





			@cached_property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name							)
										return model





			@slow
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Any								=			self.tokenizer(self.src_text   ,      return_tensors="""tf"""							)
										A__       :     Tuple								=			self.model.generate(
										    model_inputs.input_ids   ,      attention_mask=model_inputs.attention_mask   ,      num_beams=2   ,      use_cache=snake_case__   ,      )
										A__       :     Optional[Any]								=			self.tokenizer.batch_decode(generated_ids.numpy()   ,      skip_special_tokens=snake_case__							)[0]
										assert generated_words in (
										    "i don't know. i just feel like i'm going to throw up. it's not fun.",
										    "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
										    "i'm not sure. i just feel like i've been in a bad situation.",
										)









'''simple docstring'''







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


lowerCAmelCase     :  Tuple											=    logging.get_logger(__name__)

lowerCAmelCase     :  Union[str, Any]											=    {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase     :  str											=    {
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

lowerCAmelCase     :  str											=    {
    """gpt2""": 10_24,
    """gpt2-medium""": 10_24,
    """gpt2-large""": 10_24,
    """gpt2-xl""": 10_24,
    """distilgpt2""": 10_24,
}
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_						):



     """simple docstring"""






     __magic_name__           =     VOCAB_FILES_NAMES
     __magic_name__           =     PRETRAINED_VOCAB_FILES_MAP
     __magic_name__           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__           =     ["input_ids", "attention_mask"]
     __magic_name__           =     GPTaTokenizer







     def __init__(			self   ,						snake_case__=None   ,						snake_case__=None   ,						snake_case__=None   ,						snake_case__="<|endoftext|>"   ,						snake_case__="<|endoftext|>"   ,						snake_case__="<|endoftext|>"   ,						snake_case__=False   ,						**snake_case__   ,						):

            '''simple docstring'''

            super().__init__(
                snake_case__   ,						snake_case__   ,						tokenizer_file=snake_case__   ,						unk_token=snake_case__   ,						bos_token=snake_case__   ,						eos_token=snake_case__   ,						add_prefix_space=snake_case__   ,						**snake_case__   ,						)

            _lowerCAmelCase     :				str       =							kwargs.pop('add_bos_token'   ,						snake_case__ )

            _lowerCAmelCase     :				List[Any]       =							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
            if pre_tok_state.get('add_prefix_space'   ,						snake_case__ ) != add_prefix_space:
                   _lowerCAmelCase     :				str       =							getattr(snake_case__   ,						pre_tok_state.pop('type' ) )
                   _lowerCAmelCase     :				List[str]       =							add_prefix_space
                   _lowerCAmelCase     :				Dict       =							pre_tok_class(**snake_case__ )

            _lowerCAmelCase     :				Any       =							add_prefix_space







     def a						(			self   ,						*snake_case__   ,						**snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				int       =							kwargs.get('is_split_into_words'   ,						snake_case__ )
            assert self.add_prefix_space or not is_split_into_words, (
                F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                "to use it with pretokenized inputs."
            )

            return super()._batch_encode_plus(*snake_case__   ,						**snake_case__ )







     def a						(			self   ,						*snake_case__   ,						**snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				int       =							kwargs.get('is_split_into_words'   ,						snake_case__ )

            assert self.add_prefix_space or not is_split_into_words, (
                F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                "to use it with pretokenized inputs."
            )

            return super()._encode_plus(*snake_case__   ,						**snake_case__ )







     def a						(			self   ,						snake_case__   ,						snake_case__ = None ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							self._tokenizer.model.save(snake_case__   ,						name=snake_case__ )
            return tuple(snake_case__ )







     def a						(			self   ,						snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							[]
            for is_user, text in conversation.iter_texts():
                   input_ids.extend(self.encode(snake_case__   ,						add_special_tokens=snake_case__ ) + [self.eos_token_id] )

            if len(snake_case__ ) > self.model_max_length:
                   _lowerCAmelCase     :				str       =							input_ids[-self.model_max_length :]
            return input_ids

