




"""simple docstring"""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_mbart import MBartTokenizer
else:
       a							      =     None


a							      =     logging.get_logger(__name__)


a							      =     {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

a							      =     {
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-en-ro''': '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json''',
        '''facebook/mbart-large-cc25''': '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json''',
    },
}

a							      =     {
    '''facebook/mbart-large-en-ro''': 1_024,
    '''facebook/mbart-large-cc25''': 1_024,
}

# fmt: off
a							      =     ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']





class 							lowercase_			(	__lowerCAmelCase				):



 '''simple docstring'''
 UpperCAmelCase       :	int     =      VOCAB_FILES_NAMES
 UpperCAmelCase       :	Any     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase       :	Union[str, Any]     =      PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase       :	List[str]     =      ['''input_ids''', '''attention_mask''']
 UpperCAmelCase       :	Tuple     =      MBartTokenizer

 UpperCAmelCase       :	List[int]     =      []
 UpperCAmelCase       :	List[int]     =      []

 def __init__(					self			:			Dict       ,				_UpperCAmelCase			:			List[Any]=None       ,				_UpperCAmelCase			:			List[str]=None       ,				_UpperCAmelCase			:			List[str]="<s>"       ,				_UpperCAmelCase			:			Union[str, Any]="</s>"       ,				_UpperCAmelCase			:			Any="</s>"       ,				_UpperCAmelCase			:			Any="<s>"       ,				_UpperCAmelCase			:			List[str]="<unk>"       ,				_UpperCAmelCase			:			Tuple="<pad>"       ,				_UpperCAmelCase			:			str="<mask>"       ,				_UpperCAmelCase			:			str=None       ,				_UpperCAmelCase			:			str=None       ,				_UpperCAmelCase			:			Optional[int]=None       ,				**_UpperCAmelCase			:			Dict       ,				):
       # Mask token behave like a normal word, i.e. include the space before it
       _A     =  AddedToken(_UpperCAmelCase       ,				lstrip=_UpperCAmelCase       ,				rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase       ,				_UpperCAmelCase						) else mask_token

       super().__init__(
           vocab_file=_UpperCAmelCase       ,				tokenizer_file=_UpperCAmelCase       ,				bos_token=_UpperCAmelCase       ,				eos_token=_UpperCAmelCase       ,				sep_token=_UpperCAmelCase       ,				cls_token=_UpperCAmelCase       ,				unk_token=_UpperCAmelCase       ,				pad_token=_UpperCAmelCase       ,				mask_token=_UpperCAmelCase       ,				src_lang=_UpperCAmelCase       ,				tgt_lang=_UpperCAmelCase       ,				additional_special_tokens=_UpperCAmelCase       ,				**_UpperCAmelCase       ,				)

       _A     =  vocab_file
       _A     =  False if not self.vocab_file else True

       _A     =  FAIRSEQ_LANGUAGE_CODES.copy()

       if additional_special_tokens is not None:
             # Only add those special tokens if they are not already there.
             _additional_special_tokens.extend(
                 [t for t in additional_special_tokens if t not in _additional_special_tokens]						)

       self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}						)
       _A     =  {
           lang_code: self.convert_tokens_to_ids(_UpperCAmelCase						) for lang_code in FAIRSEQ_LANGUAGE_CODES
       }

       _A     =  src_lang if src_lang is not None else 'en_XX'
       _A     =  self.convert_tokens_to_ids(self._src_lang						)
       _A     =  tgt_lang
       self.set_src_lang_special_tokens(self._src_lang						)

 @property
 def      lowerCAmelCase_				(					self			:			int						):
       return self._src_lang

 @src_lang.setter
 def      lowerCAmelCase_				(					self			:			int       ,				_UpperCAmelCase			:			str						):
       _A     =  new_src_lang
       self.set_src_lang_special_tokens(self._src_lang						)

 def      lowerCAmelCase_				(					self			:			List[Any]       ,				_UpperCAmelCase			:			List[int]       ,				_UpperCAmelCase			:			Optional[List[int]] = None						):
       if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

 def      lowerCAmelCase_				(					self			:			Optional[Any]       ,				_UpperCAmelCase			:			List[int]       ,				_UpperCAmelCase			:			Optional[List[int]] = None						):
       _A     =  [self.sep_token_id]
       _A     =  [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]

 def      lowerCAmelCase_				(					self			:			List[Any]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Optional[str]       ,				_UpperCAmelCase			:			Optional[str]       ,				**_UpperCAmelCase			:			Dict						):
       if src_lang is None or tgt_lang is None:
             raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'						)
       _A     =  src_lang
       _A     =  self(_UpperCAmelCase       ,				add_special_tokens=_UpperCAmelCase       ,				return_tensors=_UpperCAmelCase       ,				**_UpperCAmelCase						)
       _A     =  self.convert_tokens_to_ids(_UpperCAmelCase						)
       _A     =  tgt_lang_id
       return inputs

 def      lowerCAmelCase_				(					self			:			Dict       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			str = "en_XX"       ,				_UpperCAmelCase			:			Optional[List[str]] = None       ,				_UpperCAmelCase			:			str = "ro_RO"       ,				**_UpperCAmelCase			:			Any       ,				):
       _A     =  src_lang
       _A     =  tgt_lang
       return super().prepare_seqaseq_batch(_UpperCAmelCase       ,				_UpperCAmelCase       ,				**_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			List[Any]						):
       return self.set_src_lang_special_tokens(self.src_lang						)

 def      lowerCAmelCase_				(					self			:			List[Any]						):
       return self.set_tgt_lang_special_tokens(self.tgt_lang						)

 def      lowerCAmelCase_				(					self			:			List[str]       ,				_UpperCAmelCase			:			Union[str, Any]						):
       _A     =  self.convert_tokens_to_ids(_UpperCAmelCase						)
       _A     =  []
       _A     =  [self.eos_token_id, self.cur_lang_code]

       _A     =  self.convert_ids_to_tokens(self.prefix_tokens						)
       _A     =  self.convert_ids_to_tokens(self.suffix_tokens						)

       _A     =  processors.TemplateProcessing(
           single=prefix_tokens_str + ['$A'] + suffix_tokens_str       ,				pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str       ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,				self.prefix_tokens + self.suffix_tokens						)						)       ,				)

 def      lowerCAmelCase_				(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			str						):
       _A     =  self.convert_tokens_to_ids(_UpperCAmelCase						)
       _A     =  []
       _A     =  [self.eos_token_id, self.cur_lang_code]

       _A     =  self.convert_ids_to_tokens(self.prefix_tokens						)
       _A     =  self.convert_ids_to_tokens(self.suffix_tokens						)

       _A     =  processors.TemplateProcessing(
           single=prefix_tokens_str + ['$A'] + suffix_tokens_str       ,				pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str       ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,				self.prefix_tokens + self.suffix_tokens						)						)       ,				)



 def      lowerCAmelCase_				(					self			:			str       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Optional[str] = None						):
       if not self.can_save_slow_tokenizer:
             raise ValueError(
                 'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                 'tokenizer.'						)

       if not os.path.isdir(_UpperCAmelCase						):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''						)
             return
       _A     =  os.path.join(
           _UpperCAmelCase       ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCAmelCase						):
             copyfile(self.vocab_file       ,				_UpperCAmelCase						)

       return (out_vocab_file,)



'''simple docstring'''

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
         OpenAIGPTConfig,
         OpenAIGPTDoubleHeadsModel,
         OpenAIGPTForSequenceClassification,
         OpenAIGPTLMHeadModel,
         OpenAIGPTModel,
     )




class    lowerCAmelCase       :


 def __init__(    self  : str			,							a__  : Union[str, Any]			,							a__  : Union[str, Any]=13			,							a__  : Union[str, Any]=7			,							a__  : Any=True			,							a__  : str=True			,							a__  : Optional[Any]=True			,							a__  : Any=99			,							a__  : Dict=32			,							a__  : Tuple=5			,							a__  : List[Any]=4			,							a__  : List[Any]=37			,							a__  : List[str]="gelu"			,							a__  : Dict=0.1			,							a__  : str=0.1			,							a__  : str=512			,							a__  : List[str]=16			,							a__  : Dict=2			,							a__  : Union[str, Any]=0.02			,							a__  : Dict=3			,							a__  : List[Any]=4			,							a__  : Optional[int]=None			,							):
  '''simple docstring'''

  lowerCAmelCase__     :   List[Any]     		=	parent
  lowerCAmelCase__     :   Optional[Any]     		=	batch_size
  lowerCAmelCase__     :   Optional[int]     		=	seq_length
  lowerCAmelCase__     :   int     		=	is_training
  lowerCAmelCase__     :   int     		=	use_token_type_ids
  lowerCAmelCase__     :   List[Any]     		=	use_labels
  lowerCAmelCase__     :   int     		=	vocab_size
  lowerCAmelCase__     :   Dict     		=	hidden_size
  lowerCAmelCase__     :   Tuple     		=	num_hidden_layers
  lowerCAmelCase__     :   Tuple     		=	num_attention_heads
  lowerCAmelCase__     :   Any     		=	intermediate_size
  lowerCAmelCase__     :   Optional[int]     		=	hidden_act
  lowerCAmelCase__     :   Optional[int]     		=	hidden_dropout_prob
  lowerCAmelCase__     :   Optional[Any]     		=	attention_probs_dropout_prob
  lowerCAmelCase__     :   Tuple     		=	max_position_embeddings
  lowerCAmelCase__     :   str     		=	type_vocab_size
  lowerCAmelCase__     :   List[Any]     		=	type_sequence_label_size
  lowerCAmelCase__     :   List[str]     		=	initializer_range
  lowerCAmelCase__     :   Tuple     		=	num_labels
  lowerCAmelCase__     :   Optional[Any]     		=	num_choices
  lowerCAmelCase__     :   int     		=	scope
  lowerCAmelCase__     :   List[str]     		=	self.vocab_size - 1


 def    _A	(    self  : Tuple    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Optional[int]     		=	ids_tensor([self.batch_size, self.seq_length]			,							self.vocab_size    )

  lowerCAmelCase__     :   Dict     		=	None
  if self.use_token_type_ids:
   lowerCAmelCase__     :   List[str]     		=	ids_tensor([self.batch_size, self.seq_length]			,							self.type_vocab_size    )

  lowerCAmelCase__     :   List[str]     		=	None
  lowerCAmelCase__     :   Tuple     		=	None
  lowerCAmelCase__     :   Optional[int]     		=	None
  if self.use_labels:
   lowerCAmelCase__     :   int     		=	ids_tensor([self.batch_size]			,							self.type_sequence_label_size    )
   lowerCAmelCase__     :   Any     		=	ids_tensor([self.batch_size, self.seq_length]			,							self.num_labels    )
   lowerCAmelCase__     :   Optional[Any]     		=	ids_tensor([self.batch_size]			,							self.num_choices    )

  lowerCAmelCase__     :   Any     		=	OpenAIGPTConfig(
      vocab_size=self.vocab_size			,							n_embd=self.hidden_size			,							n_layer=self.num_hidden_layers			,							n_head=self.num_attention_heads			,							n_positions=self.max_position_embeddings			,							pad_token_id=self.pad_token_id			,							)

  lowerCAmelCase__     :   Optional[Any]     		=	ids_tensor([self.num_hidden_layers, self.num_attention_heads]			,							2    )

  return (
      config,
      input_ids,
      head_mask,
      token_type_ids,
      sequence_labels,
      token_labels,
      choice_labels,
  )


 def    _A	(    self  : Dict			,							a__  : Any			,							a__  : Any			,							a__  : int			,							a__  : str			,							*a__  : Optional[Any]    ):
  '''simple docstring'''

  lowerCAmelCase__     :   str     		=	OpenAIGPTModel(config=a__    )
  model.to(a__    )
  model.eval()

  lowerCAmelCase__     :   Optional[Any]     		=	model(a__			,							token_type_ids=a__			,							head_mask=a__    )
  lowerCAmelCase__     :   Optional[int]     		=	model(a__			,							token_type_ids=a__    )
  lowerCAmelCase__     :   Any     		=	model(a__    )

  self.parent.assertEqual(result.last_hidden_state.shape			,							(self.batch_size, self.seq_length, self.hidden_size)    )


 def    _A	(    self  : Optional[int]			,							a__  : List[Any]			,							a__  : int			,							a__  : Dict			,							a__  : Any			,							*a__  : Any    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Tuple     		=	OpenAIGPTLMHeadModel(a__    )
  model.to(a__    )
  model.eval()

  lowerCAmelCase__     :   Dict     		=	model(a__			,							token_type_ids=a__			,							labels=a__    )
  self.parent.assertEqual(result.loss.shape			,							()    )
  self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.vocab_size)    )


 def    _A	(    self  : List[str]			,							a__  : str			,							a__  : Dict			,							a__  : Union[str, Any]			,							a__  : int			,							*a__  : Dict    ):
  '''simple docstring'''

  lowerCAmelCase__     :   List[str]     		=	OpenAIGPTDoubleHeadsModel(a__    )
  model.to(a__    )
  model.eval()

  lowerCAmelCase__     :   Optional[Any]     		=	model(a__			,							token_type_ids=a__			,							labels=a__    )
  self.parent.assertEqual(result.loss.shape			,							()    )
  self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.vocab_size)    )


 def    _A	(    self  : Any			,							a__  : Any			,							a__  : Union[str, Any]			,							a__  : Optional[int]			,							a__  : Any			,							*a__  : List[str]    ):
  '''simple docstring'''

  lowerCAmelCase__     :   List[Any]     		=	self.num_labels
  lowerCAmelCase__     :   Union[str, Any]     		=	OpenAIGPTForSequenceClassification(a__    )
  model.to(a__    )
  model.eval()

  lowerCAmelCase__     :   List[str]     		=	ids_tensor([self.batch_size]			,							self.type_sequence_label_size    )
  lowerCAmelCase__     :   Tuple     		=	model(a__			,							token_type_ids=a__			,							labels=a__    )
  self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.num_labels)    )


 def    _A	(    self  : Optional[int]    ):
  '''simple docstring'''

  lowerCAmelCase__     :   List[str]     		=	self.prepare_config_and_inputs()
  (
      (
      lowerCAmelCase__
  )		,     (
      lowerCAmelCase__
  )		,     (
      lowerCAmelCase__
  )		,     (
      lowerCAmelCase__
  )		,     (
      lowerCAmelCase__
  )		,     (
      lowerCAmelCase__
  )		,     (
      lowerCAmelCase__
  )		,     
  )     :   Union[str, Any]     		=	config_and_inputs
  lowerCAmelCase__     :   str     		=	{
      "input_ids": input_ids,
      "token_type_ids": token_type_ids,
      "head_mask": head_mask,
  }

  return config, inputs_dict




@require_torch
class    lowerCAmelCase       (     UpperCamelCase_    ,  UpperCamelCase_    ,  UpperCamelCase_    ,  unittest.TestCase  ):
 A_				:				str	=			(
     (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
     if is_torch_available()
     else ()
 )
 A_				:				Any	=			(
     (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
 )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
 A_				:				int	=			(
     {
         """feature-extraction""": OpenAIGPTModel,
         """text-classification""": OpenAIGPTForSequenceClassification,
         """text-generation""": OpenAIGPTLMHeadModel,
         """zero-shot""": OpenAIGPTForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )


 def    _A	(    self  : List[Any]			,							a__  : Optional[Any]			,							a__  : Optional[int]			,							a__  : Optional[Any]			,							a__  : List[Any]			,							a__  : int    ):
  '''simple docstring'''

  if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
   # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
   # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
   # tiny config could not be created.
   return True

  return False


 def    _A	(    self  : Dict			,							a__  : Optional[int]			,							a__  : List[Any]			,							a__  : List[Any]=False    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Any     		=	super()._prepare_for_class(a__			,							a__			,							return_labels=a__    )

  if return_labels:
   if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
    lowerCAmelCase__     :   Tuple     		=	torch.zeros(
        (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)			,							dtype=torch.long			,							device=a__			,							)
    lowerCAmelCase__     :   Union[str, Any]     		=	inputs_dict["labels"]
    lowerCAmelCase__     :   int     		=	inputs_dict["labels"]
    lowerCAmelCase__     :   Optional[int]     		=	torch.zeros(
        (self.model_tester.batch_size, self.model_tester.num_choices)			,							dtype=torch.long			,							device=a__			,							)
    lowerCAmelCase__     :   List[str]     		=	torch.zeros(
        self.model_tester.batch_size			,							dtype=torch.long			,							device=a__    )
  return inputs_dict


 def    _A	(    self  : str    ):
  '''simple docstring'''

  lowerCAmelCase__     :   str     		=	OpenAIGPTModelTester(self    )
  lowerCAmelCase__     :   Any     		=	ConfigTester(self			,							config_class=a__			,							n_embd=37    )


 def    _A	(    self  : List[str]    ):
  '''simple docstring'''

  self.config_tester.run_common_tests()


 def    _A	(    self  : Optional[int]    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Dict     		=	self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_openai_gpt_model(*a__    )


 def    _A	(    self  : Union[str, Any]    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Optional[int]     		=	self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_lm_head_model(*a__    )


 def    _A	(    self  : int    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Tuple     		=	self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_double_lm_head_model(*a__    )


 def    _A	(    self  : str    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Union[str, Any]     		=	self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*a__    )


 @slow
 def    _A	(    self  : Tuple    ):
  '''simple docstring'''

  for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
   lowerCAmelCase__     :   Any     		=	OpenAIGPTModel.from_pretrained(a__    )
   self.assertIsNotNone(a__    )




@require_torch
class    lowerCAmelCase       (     unittest.TestCase  ):


 @slow
 def    _A	(    self  : Tuple    ):
  '''simple docstring'''

  lowerCAmelCase__     :   int     		=	OpenAIGPTLMHeadModel.from_pretrained("openai-gpt"    )
  model.to(a__    )
  lowerCAmelCase__     :   Union[str, Any]     		=	torch.tensor([[481, 4735, 544]]			,							dtype=torch.long			,							device=a__    )  # the president is
  lowerCAmelCase__     :   Optional[int]     		=	[
      481,
      4735,
      544,
      246,
      963,
      870,
      762,
      239,
      244,
      4_0477,
      244,
      249,
      719,
      881,
      487,
      544,
      240,
      244,
      603,
      481,
  ]  # the president is a very good man. " \n " i\'m sure he is, " said the

  lowerCAmelCase__     :   Optional[int]     		=	model.generate(a__			,							do_sample=a__    )
  self.assertListEqual(output_ids[0].tolist()			,							a__    )





