
"""simple docstring"""

from __future__ import annotations

import unittest

from transformers import OpenAIGPTConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers.models.openai.modeling_tf_openai import (
					    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
					    TFOpenAIGPTDoubleHeadsModel,
					    TFOpenAIGPTForSequenceClassification,
					    TFOpenAIGPTLMHeadModel,
					    TFOpenAIGPTModel,
					)






class        _lowercase   :




	def __init__( self		, UpperCAmelCase_		, )		->							Optional[Any]:
			lowerCamelCase     :			Any		=    parent
			lowerCamelCase     :			str		=    13
			lowerCamelCase     :			List[str]		=    7
			lowerCamelCase     :			List[str]		=    True
			lowerCamelCase     :			List[str]		=    True
			lowerCamelCase     :			str		=    True
			lowerCamelCase     :			Tuple		=    True
			lowerCamelCase     :			Optional[int]		=    True
			lowerCamelCase     :			Optional[int]		=    99
			lowerCamelCase     :			Tuple		=    32
			lowerCamelCase     :			Any		=    2
			lowerCamelCase     :			Optional[Any]		=    4
			lowerCamelCase     :			Any		=    37
			lowerCamelCase     :			int		=    'gelu'
			lowerCamelCase     :			Any		=    0.1
			lowerCamelCase     :			Any		=    0.1
			lowerCamelCase     :			Any		=    512
			lowerCamelCase     :			Any		=    16
			lowerCamelCase     :			int		=    2
			lowerCamelCase     :			Tuple		=    0.02
			lowerCamelCase     :			List[str]		=    3
			lowerCamelCase     :			str		=    4
			lowerCamelCase     :			str		=    None
			lowerCamelCase     :			Any		=    self.vocab_size - 1




	def 			_UpperCamelCase    ( self					)		->							Dict:
			lowerCamelCase     :			Tuple		=    ids_tensor([self.batch_size, self.seq_length]		, self.vocab_size					)

			lowerCamelCase     :			List[Any]		=    None
			if self.use_input_mask:
					lowerCamelCase     :			List[str]		=    random_attention_mask([self.batch_size, self.seq_length]					)

			lowerCamelCase     :			Union[str, Any]		=    None
			if self.use_token_type_ids:
					lowerCamelCase     :			List[Any]		=    ids_tensor([self.batch_size, self.seq_length]		, self.type_vocab_size					)

			lowerCamelCase     :			Union[str, Any]		=    None
			if self.use_mc_token_ids:
					lowerCamelCase     :			List[str]		=    ids_tensor([self.batch_size, self.num_choices]		, self.seq_length					)

			lowerCamelCase     :			List[Any]		=    None
			lowerCamelCase     :			Any		=    None
			lowerCamelCase     :			Dict		=    None
			if self.use_labels:
					lowerCamelCase     :			Optional[Any]		=    ids_tensor([self.batch_size]		, self.type_sequence_label_size					)
					lowerCamelCase     :			int		=    ids_tensor([self.batch_size, self.seq_length]		, self.num_labels					)
					lowerCamelCase     :			int		=    ids_tensor([self.batch_size]		, self.num_choices					)

			lowerCamelCase     :			Any		=    OpenAIGPTConfig(
			    vocab_size=self.vocab_size		, n_embd=self.hidden_size		, n_layer=self.num_hidden_layers		, n_head=self.num_attention_heads		, n_positions=self.max_position_embeddings		, pad_token_id=self.pad_token_id		, )

			lowerCamelCase     :			Optional[int]		=    ids_tensor([self.num_hidden_layers, self.num_attention_heads]		, 2					)

			return (
			    config,
			    input_ids,
			    input_mask,
			    head_mask,
			    token_type_ids,
			    mc_token_ids,
			    sequence_labels,
			    token_labels,
			    choice_labels,
			)




	def 			_UpperCamelCase    ( self		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, *UpperCAmelCase_					)		->							List[str]:
			lowerCamelCase     :			Any		=    TFOpenAIGPTModel(config=UpperCAmelCase_					)
			lowerCamelCase     :			Tuple		=    {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
			lowerCamelCase     :			Optional[int]		=    model(UpperCAmelCase_					)

			lowerCamelCase     :			Optional[int]		=    [input_ids, input_mask]
			lowerCamelCase     :			str		=    model(UpperCAmelCase_					)

			lowerCamelCase     :			Any		=    model(UpperCAmelCase_					)

			self.parent.assertEqual(result.last_hidden_state.shape		, (self.batch_size, self.seq_length, self.hidden_size)					)




	def 			_UpperCamelCase    ( self		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, *UpperCAmelCase_					)		->							Tuple:
			lowerCamelCase     :			Optional[int]		=    TFOpenAIGPTLMHeadModel(config=UpperCAmelCase_					)
			lowerCamelCase     :			str		=    {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
			lowerCamelCase     :			Optional[Any]		=    model(UpperCAmelCase_					)
			self.parent.assertEqual(result.logits.shape		, (self.batch_size, self.seq_length, self.vocab_size)					)




	def 			_UpperCamelCase    ( self		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, *UpperCAmelCase_					)		->							Tuple:
			lowerCamelCase     :			List[Any]		=    TFOpenAIGPTDoubleHeadsModel(config=UpperCAmelCase_					)

			lowerCamelCase     :			str		=    tf.tile(tf.expand_dims(UpperCAmelCase_		, 1					)		, (1, self.num_choices, 1)					)
			lowerCamelCase     :			List[Any]		=    tf.tile(tf.expand_dims(UpperCAmelCase_		, 1					)		, (1, self.num_choices, 1)					)
			lowerCamelCase     :			Union[str, Any]		=    tf.tile(tf.expand_dims(UpperCAmelCase_		, 1					)		, (1, self.num_choices, 1)					)

			lowerCamelCase     :			int		=    {
			    'input_ids': multiple_choice_inputs_ids,
			    'mc_token_ids': mc_token_ids,
			    'attention_mask': multiple_choice_input_mask,
			    'token_type_ids': multiple_choice_token_type_ids,
			}
			lowerCamelCase     :			Dict		=    model(UpperCAmelCase_					)
			self.parent.assertEqual(
			    result.logits.shape		, (self.batch_size, self.num_choices, self.seq_length, self.vocab_size)					)
			self.parent.assertEqual(result.mc_logits.shape		, (self.batch_size, self.num_choices)					)




	def 			_UpperCamelCase    ( self		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, *UpperCAmelCase_					)		->							Dict:
			lowerCamelCase     :			List[str]		=    self.num_labels
			lowerCamelCase     :			Dict		=    ids_tensor([self.batch_size]		, self.type_sequence_label_size					)
			lowerCamelCase     :			Tuple		=    {
			    'input_ids': input_ids,
			    'attention_mask': input_mask,
			    'token_type_ids': token_type_ids,
			    'labels': sequence_labels,
			}
			lowerCamelCase     :			Tuple		=    TFOpenAIGPTForSequenceClassification(UpperCAmelCase_					)
			lowerCamelCase     :			int		=    model(UpperCAmelCase_					)
			self.parent.assertEqual(result.logits.shape		, (self.batch_size, self.num_labels)					)




	def 			_UpperCamelCase    ( self					)		->							Union[str, Any]:
			lowerCamelCase     :			Dict		=    self.prepare_config_and_inputs()

			(
			    (
			    lowerCamelCase
			)	,    (
			    lowerCamelCase
			)	,    (
			    lowerCamelCase
			)	,    (
			    lowerCamelCase
			)	,    (
			    lowerCamelCase
			)	,    (
			    lowerCamelCase
			)	,    (
			    lowerCamelCase
			)	,    (
			    lowerCamelCase
			)	,    (
			    lowerCamelCase
			)	,    
			)     :			List[Any]		=    config_and_inputs

			lowerCamelCase     :			int		=    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
			return config, inputs_dict






@require_tf
class        _lowercase   (				__UpperCAmelCase				,       __UpperCAmelCase				,       unittest.TestCase		):
	lowercase_        =	(
	    (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
	    if is_tf_available()
	    else ()
	)
	lowercase_        =	(
	    (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
	)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
	lowercase_        =	(
	    {
	        'feature-extraction': TFOpenAIGPTModel,
	        'text-classification': TFOpenAIGPTForSequenceClassification,
	        'text-generation': TFOpenAIGPTLMHeadModel,
	        'zero-shot': TFOpenAIGPTForSequenceClassification,
	    }
	    if is_tf_available()
	    else {}
	)
	lowercase_        =	False
	lowercase_        =	False




	def 			_UpperCamelCase    ( self		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_					)		->							Tuple:
			if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
					# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
					# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
					# tiny config could not be created.
					return True

			return False




	def 			_UpperCamelCase    ( self					)		->							Tuple:
			lowerCamelCase     :			Union[str, Any]		=    TFOpenAIGPTModelTester(self					)
			lowerCamelCase     :			int		=    ConfigTester(self		, config_class=UpperCAmelCase_		, n_embd=37					)




	def 			_UpperCamelCase    ( self					)		->							Optional[int]:
			self.config_tester.run_common_tests()




	def 			_UpperCamelCase    ( self					)		->							List[str]:
			lowerCamelCase     :			Optional[int]		=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_openai_gpt_model(*UpperCAmelCase_					)




	def 			_UpperCamelCase    ( self					)		->							Optional[Any]:
			lowerCamelCase     :			Dict		=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_openai_gpt_lm_head(*UpperCAmelCase_					)




	def 			_UpperCamelCase    ( self					)		->							Optional[int]:
			lowerCamelCase     :			Union[str, Any]		=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_openai_gpt_double_head(*UpperCAmelCase_					)




	def 			_UpperCamelCase    ( self					)		->							Dict:
			lowerCamelCase     :			List[Any]		=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*UpperCAmelCase_					)




	@slow
	def 			_UpperCamelCase    ( self					)		->							Optional[int]:
			for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowerCamelCase     :			Tuple		=    TFOpenAIGPTModel.from_pretrained(UpperCAmelCase_					)
					self.assertIsNotNone(UpperCAmelCase_					)






@require_tf
class        _lowercase   (				unittest.TestCase		):




	@slow
	def 			_UpperCamelCase    ( self					)		->							Tuple:
			lowerCamelCase     :			Optional[Any]		=    TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt'					)
			lowerCamelCase     :			str		=    tf.convert_to_tensor([[481, 4735, 544]]		, dtype=tf.intaa					)  # the president is
			lowerCamelCase     :			List[Any]		=    [
			    481,
			    4735,
			    544,
			    246,
			    963,
			    870,
			    762,
			    239,
			    244,
			    40477,
			    244,
			    249,
			    719,
			    881,
			    487,
			    544,
			    240,
			    244,
			    603,
			    481,
			]  # the president is a very good man. " \n " i\'m sure he is, " said the

			lowerCamelCase     :			List[Any]		=    model.generate(UpperCAmelCase_		, do_sample=UpperCAmelCase_					)
			self.assertListEqual(output_ids[0].numpy().tolist()		, UpperCAmelCase_					)













import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
							UpperCAmelCase__  :	Union[str, Any]     =None


UpperCAmelCase__  :	str     =logging.get_logger(__name__)

UpperCAmelCase__  :	Optional[int]     ={'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__  :	str     ={
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__  :	int     ={
    '''xlm-roberta-base''': 5_12,
    '''xlm-roberta-large''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_12,
}

class 					__A   (  a			):
				__A            = VOCAB_FILES_NAMES
				__A            = PRETRAINED_VOCAB_FILES_MAP
				__A            = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A            = ["""input_ids""", """attention_mask"""]
				__A            = XLMRobertaTokenizer


				def __init__(						self      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_="<s>"      ,       UpperCAmelCase_="</s>"      ,       UpperCAmelCase_="</s>"      ,       UpperCAmelCase_="<s>"      ,       UpperCAmelCase_="<unk>"      ,       UpperCAmelCase_="<pad>"      ,       UpperCAmelCase_="<mask>"      ,       **UpperCAmelCase_      ,       ):
								# Mask token behave like a normal word, i.e. include the space before it
								lowerCamelCase		       =AddedToken(UpperCAmelCase_      ,       lstrip=UpperCAmelCase_      ,       rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_      ,       UpperCAmelCase_							) else mask_token

								super().__init__(
								    UpperCAmelCase_      ,       tokenizer_file=UpperCAmelCase_      ,       bos_token=UpperCAmelCase_      ,       eos_token=UpperCAmelCase_      ,       sep_token=UpperCAmelCase_      ,       cls_token=UpperCAmelCase_      ,       unk_token=UpperCAmelCase_      ,       pad_token=UpperCAmelCase_      ,       mask_token=UpperCAmelCase_      ,       **UpperCAmelCase_      ,       )

								lowerCamelCase		       =vocab_file
								lowerCamelCase		       =False if not self.vocab_file else True


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = None							):

								if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowerCamelCase		       =[self.cls_token_id]
								lowerCamelCase		       =[self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = None							):
								lowerCamelCase		       =[self.sep_token_id]
								lowerCamelCase		       =[self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep							) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = None							):
								if not self.can_save_slow_tokenizer:
												raise ValueError(
												    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
												    """tokenizer."""							)

								if not os.path.isdir(UpperCAmelCase_							):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""							)
												return
								lowerCamelCase		       =os.path.join(
								    UpperCAmelCase_      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

								if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCAmelCase_							):
												copyfile(self.vocab_file      ,       UpperCAmelCase_							)

								return (out_vocab_file,)





