

'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_mbart import MBartTokenizer
else:
     _UpperCamelCase						:  Dict							       =				None


_UpperCamelCase						:  Tuple							       =				logging.get_logger(__name__)


_UpperCamelCase						:  str							       =				{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

_UpperCamelCase						:  Union[str, Any]							       =				{
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

_UpperCamelCase						:  Dict							       =				{
    "facebook/mbart-large-en-ro": 1024,
    "facebook/mbart-large-cc25": 1024,
}

# fmt: off
_UpperCamelCase						:  str							       =				["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]



class 				_snake_case					(   a_			):
  SCREAMING_SNAKE_CASE			:       Tuple				=     VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE			:       int				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE			:       Tuple				=     PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE			:       Tuple				=     ['''input_ids''', '''attention_mask''']
  SCREAMING_SNAKE_CASE			:       Union[str, Any]				=     MBartTokenizer

  SCREAMING_SNAKE_CASE			:       List[int]				=     []
  SCREAMING_SNAKE_CASE			:       List[int]				=     []

  def __init__(				self      , _SCREAMING_SNAKE_CASE=None      , _SCREAMING_SNAKE_CASE=None      , _SCREAMING_SNAKE_CASE="<s>"      , _SCREAMING_SNAKE_CASE="</s>"      , _SCREAMING_SNAKE_CASE="</s>"      , _SCREAMING_SNAKE_CASE="<s>"      , _SCREAMING_SNAKE_CASE="<unk>"      , _SCREAMING_SNAKE_CASE="<pad>"      , _SCREAMING_SNAKE_CASE="<mask>"      , _SCREAMING_SNAKE_CASE=None      , _SCREAMING_SNAKE_CASE=None      , _SCREAMING_SNAKE_CASE=None      , **_SCREAMING_SNAKE_CASE      , ):





      '''simple docstring'''

      lowerCAmelCase 						=							AddedToken(_SCREAMING_SNAKE_CASE      , lstrip=_SCREAMING_SNAKE_CASE      , rstrip=_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE	) else mask_token

      super().__init__(
          vocab_file=_SCREAMING_SNAKE_CASE      , tokenizer_file=_SCREAMING_SNAKE_CASE      , bos_token=_SCREAMING_SNAKE_CASE      , eos_token=_SCREAMING_SNAKE_CASE      , sep_token=_SCREAMING_SNAKE_CASE      , cls_token=_SCREAMING_SNAKE_CASE      , unk_token=_SCREAMING_SNAKE_CASE      , pad_token=_SCREAMING_SNAKE_CASE      , mask_token=_SCREAMING_SNAKE_CASE      , src_lang=_SCREAMING_SNAKE_CASE      , tgt_lang=_SCREAMING_SNAKE_CASE      , additional_special_tokens=_SCREAMING_SNAKE_CASE      , **_SCREAMING_SNAKE_CASE      , )

      lowerCAmelCase 						=							vocab_file
      lowerCAmelCase 						=							False if not self.vocab_file else True

      lowerCAmelCase 						=							FAIRSEQ_LANGUAGE_CODES.copy()

      if additional_special_tokens is not None:
          # Only add those special tokens if they are not already there.
          _additional_special_tokens.extend(
              [t for t in additional_special_tokens if t not in _additional_special_tokens]	)

      self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}	)
      lowerCAmelCase 						=							{
          lang_code: self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE	) for lang_code in FAIRSEQ_LANGUAGE_CODES
      }

      lowerCAmelCase 						=							src_lang if src_lang is not None else 'en_XX'
      lowerCAmelCase 						=							self.convert_tokens_to_ids(self._src_lang	)
      lowerCAmelCase 						=							tgt_lang
      self.set_src_lang_special_tokens(self._src_lang	)

  @property
  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      return self._src_lang

  @src_lang.setter
  def   _SCREAMING_SNAKE_CASE				(				self      , _SCREAMING_SNAKE_CASE	):





      '''simple docstring'''

      lowerCAmelCase 						=							new_src_lang
      self.set_src_lang_special_tokens(self._src_lang	)

  def   _SCREAMING_SNAKE_CASE				(				self      , _SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE = None	):





      '''simple docstring'''

      if token_ids_a is None:
          return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

  def   _SCREAMING_SNAKE_CASE				(				self      , _SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE = None	):





      '''simple docstring'''

      lowerCAmelCase 						=							[self.sep_token_id]
      lowerCAmelCase 						=							[self.cls_token_id]

      if token_ids_a is None:
          return len(cls + token_ids_a + sep	) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]

  def   _SCREAMING_SNAKE_CASE				(				self      , _SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE      , **_SCREAMING_SNAKE_CASE	):





      '''simple docstring'''

      if src_lang is None or tgt_lang is None:
          raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'	)
      lowerCAmelCase 						=							src_lang
      lowerCAmelCase 						=							self(_SCREAMING_SNAKE_CASE      , add_special_tokens=_SCREAMING_SNAKE_CASE      , return_tensors=_SCREAMING_SNAKE_CASE      , **_SCREAMING_SNAKE_CASE	)
      lowerCAmelCase 						=							self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE	)
      lowerCAmelCase 						=							tgt_lang_id
      return inputs

  def   _SCREAMING_SNAKE_CASE				(				self      , _SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE = "en_XX"      , _SCREAMING_SNAKE_CASE = None      , _SCREAMING_SNAKE_CASE = "ro_RO"      , **_SCREAMING_SNAKE_CASE      , ):





      '''simple docstring'''

      lowerCAmelCase 						=							src_lang
      lowerCAmelCase 						=							tgt_lang
      return super().prepare_seqaseq_batch(_SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE      , **_SCREAMING_SNAKE_CASE	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      return self.set_src_lang_special_tokens(self.src_lang	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      return self.set_tgt_lang_special_tokens(self.tgt_lang	)

  def   _SCREAMING_SNAKE_CASE				(				self      , _SCREAMING_SNAKE_CASE	):





      '''simple docstring'''

      lowerCAmelCase 						=							self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE	)
      lowerCAmelCase 						=							[]
      lowerCAmelCase 						=							[self.eos_token_id, self.cur_lang_code]

      lowerCAmelCase 						=							self.convert_ids_to_tokens(self.prefix_tokens	)
      lowerCAmelCase 						=							self.convert_ids_to_tokens(self.suffix_tokens	)

      lowerCAmelCase 						=							processors.TemplateProcessing(
          single=prefix_tokens_str + ['$A'] + suffix_tokens_str      , pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str      , special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      , self.prefix_tokens + self.suffix_tokens	)	)      , )

  def   _SCREAMING_SNAKE_CASE				(				self      , _SCREAMING_SNAKE_CASE	):





      '''simple docstring'''

      lowerCAmelCase 						=							self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE	)
      lowerCAmelCase 						=							[]
      lowerCAmelCase 						=							[self.eos_token_id, self.cur_lang_code]

      lowerCAmelCase 						=							self.convert_ids_to_tokens(self.prefix_tokens	)
      lowerCAmelCase 						=							self.convert_ids_to_tokens(self.suffix_tokens	)

      lowerCAmelCase 						=							processors.TemplateProcessing(
          single=prefix_tokens_str + ['$A'] + suffix_tokens_str      , pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str      , special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      , self.prefix_tokens + self.suffix_tokens	)	)      , )


  def   _SCREAMING_SNAKE_CASE				(				self      , _SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE = None	):





      '''simple docstring'''

      if not self.can_save_slow_tokenizer:
          raise ValueError(
              'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
              'tokenizer.'	)

      if not os.path.isdir(_SCREAMING_SNAKE_CASE	):
          logger.error(F'Vocabulary path ({save_directory}) should be a directory.'	)
          return
      lowerCAmelCase 						=							os.path.join(
          _SCREAMING_SNAKE_CASE      , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

      if os.path.abspath(self.vocab_file	) != os.path.abspath(_SCREAMING_SNAKE_CASE	):
          copyfile(self.vocab_file      , _SCREAMING_SNAKE_CASE	)

      return (out_vocab_file,)








'''simple docstring'''



from typing import TYPE_CHECKING

from ...file_utils import _LazyModule, is_torch_available
from ...utils import OptionalDependencyNotAvailable


_UpperCamelCase						:  str							       =				{
    "configuration_gpt_neox_japanese": ["GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP", "GPTNeoXJapaneseConfig"],
    "tokenization_gpt_neox_japanese": ["GPTNeoXJapaneseTokenizer"],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _UpperCamelCase						:  Dict							       =				[
         "GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST",
         "GPTNeoXJapaneseForCausalLM",
         "GPTNeoXJapaneseLayer",
         "GPTNeoXJapaneseModel",
         "GPTNeoXJapanesePreTrainedModel",
     ]


if TYPE_CHECKING:
     from .configuration_gpt_neox_japanese import GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoXJapaneseConfig
     from .tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_gpt_neox_japanese import (
              GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST,
              GPTNeoXJapaneseForCausalLM,
              GPTNeoXJapaneseLayer,
              GPTNeoXJapaneseModel,
              GPTNeoXJapanesePreTrainedModel,
          )


else:
     import sys

     _UpperCamelCase						:  List[Any]							       =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





