




'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbart import MBartTokenizer
else:
			__UpperCAmelCase       =		None


__UpperCAmelCase       =		logging.get_logger(__name__)


__UpperCAmelCase       =		{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase       =		{
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

__UpperCAmelCase       =		{
    "facebook/mbart-large-en-ro": 1_024,
    "facebook/mbart-large-cc25": 1_024,
}

# fmt: off
__UpperCAmelCase       =		["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]
class 		SCREAMING_SNAKE_CASE			(     _a							):




				'''simple docstring'''





				__UpperCamelCase     				=   VOCAB_FILES_NAMES
				__UpperCamelCase     				=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase     				=   PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase     				=   ['''input_ids''', '''attention_mask''']
				__UpperCamelCase     				=   MBartTokenizer

				__UpperCamelCase     				=   []
				__UpperCamelCase     				=   []







				def __init__(   self  ,						SCREAMING_SNAKE_CASE__=None  ,						SCREAMING_SNAKE_CASE__=None  ,						SCREAMING_SNAKE_CASE__="<s>"  ,						SCREAMING_SNAKE_CASE__="</s>"  ,						SCREAMING_SNAKE_CASE__="</s>"  ,						SCREAMING_SNAKE_CASE__="<s>"  ,						SCREAMING_SNAKE_CASE__="<unk>"  ,						SCREAMING_SNAKE_CASE__="<pad>"  ,						SCREAMING_SNAKE_CASE__="<mask>"  ,						SCREAMING_SNAKE_CASE__=None  ,						SCREAMING_SNAKE_CASE__=None  ,						SCREAMING_SNAKE_CASE__=None  ,						**SCREAMING_SNAKE_CASE__  ,						):
							'''simple docstring'''







							snake_case:       Optional[Any]        =       AddedToken(SCREAMING_SNAKE_CASE__  ,						lstrip=SCREAMING_SNAKE_CASE__  ,						rstrip=SCREAMING_SNAKE_CASE__				) if isinstance(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__				) else mask_token

							super().__init__(
							    vocab_file=SCREAMING_SNAKE_CASE__  ,						tokenizer_file=SCREAMING_SNAKE_CASE__  ,						bos_token=SCREAMING_SNAKE_CASE__  ,						eos_token=SCREAMING_SNAKE_CASE__  ,						sep_token=SCREAMING_SNAKE_CASE__  ,						cls_token=SCREAMING_SNAKE_CASE__  ,						unk_token=SCREAMING_SNAKE_CASE__  ,						pad_token=SCREAMING_SNAKE_CASE__  ,						mask_token=SCREAMING_SNAKE_CASE__  ,						src_lang=SCREAMING_SNAKE_CASE__  ,						tgt_lang=SCREAMING_SNAKE_CASE__  ,						additional_special_tokens=SCREAMING_SNAKE_CASE__  ,						**SCREAMING_SNAKE_CASE__  ,						)

							snake_case:       List[Any]        =       vocab_file
							snake_case:       Tuple        =       False if not self.vocab_file else True

							snake_case:       Union[str, Any]        =       FAIRSEQ_LANGUAGE_CODES.copy()

							if additional_special_tokens is not None:
										# Only add those special tokens if they are not already there.
										_additional_special_tokens.extend(
										    [t for t in additional_special_tokens if t not in _additional_special_tokens]				)

							self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}				)
							snake_case:       Dict        =       {
							    lang_code: self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				) for lang_code in FAIRSEQ_LANGUAGE_CODES
							}

							snake_case:       Optional[int]        =       src_lang if src_lang is not None else 'en_XX'
							snake_case:       Any        =       self.convert_tokens_to_ids(self._src_lang				)
							snake_case:       Union[str, Any]        =       tgt_lang
							self.set_src_lang_special_tokens(self._src_lang				)







				@property
				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							return self._src_lang







				@src_lang.setter
				def        _UpperCamelCase							(   self  ,						SCREAMING_SNAKE_CASE__				):
							'''simple docstring'''







							snake_case:       Optional[int]        =       new_src_lang
							self.set_src_lang_special_tokens(self._src_lang				)







				def        _UpperCamelCase							(   self  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ = None				):
							'''simple docstring'''







							if token_ids_a is None:
										return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







				def        _UpperCamelCase							(   self  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ = None				):
							'''simple docstring'''







							snake_case:       List[Any]        =       [self.sep_token_id]
							snake_case:       Any        =       [self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep				) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]







				def        _UpperCamelCase							(   self  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						**SCREAMING_SNAKE_CASE__				):
							'''simple docstring'''







							if src_lang is None or tgt_lang is None:
										raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'				)
							snake_case:       Any        =       src_lang
							snake_case:       Any        =       self(SCREAMING_SNAKE_CASE__  ,						add_special_tokens=SCREAMING_SNAKE_CASE__  ,						return_tensors=SCREAMING_SNAKE_CASE__  ,						**SCREAMING_SNAKE_CASE__				)
							snake_case:       Dict        =       self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				)
							snake_case:       Optional[int]        =       tgt_lang_id
							return inputs







				def        _UpperCamelCase							(   self  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ = "en_XX"  ,						SCREAMING_SNAKE_CASE__ = None  ,						SCREAMING_SNAKE_CASE__ = "ro_RO"  ,						**SCREAMING_SNAKE_CASE__  ,						):
							'''simple docstring'''







							snake_case:       str        =       src_lang
							snake_case:       Tuple        =       tgt_lang
							return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						**SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							return self.set_src_lang_special_tokens(self.src_lang				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							return self.set_tgt_lang_special_tokens(self.tgt_lang				)







				def        _UpperCamelCase							(   self  ,						SCREAMING_SNAKE_CASE__				):
							'''simple docstring'''







							snake_case:       Union[str, Any]        =       self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				)
							snake_case:       Any        =       []
							snake_case:       str        =       [self.eos_token_id, self.cur_lang_code]

							snake_case:       Tuple        =       self.convert_ids_to_tokens(self.prefix_tokens				)
							snake_case:       Dict        =       self.convert_ids_to_tokens(self.suffix_tokens				)

							snake_case:       Dict        =       processors.TemplateProcessing(
							    single=prefix_tokens_str + ['$A'] + suffix_tokens_str  ,						pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str  ,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str  ,						self.prefix_tokens + self.suffix_tokens				)				)  ,						)







				def        _UpperCamelCase							(   self  ,						SCREAMING_SNAKE_CASE__				):
							'''simple docstring'''







							snake_case:       Tuple        =       self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				)
							snake_case:       List[str]        =       []
							snake_case:       List[Any]        =       [self.eos_token_id, self.cur_lang_code]

							snake_case:       int        =       self.convert_ids_to_tokens(self.prefix_tokens				)
							snake_case:       Any        =       self.convert_ids_to_tokens(self.suffix_tokens				)

							snake_case:       Tuple        =       processors.TemplateProcessing(
							    single=prefix_tokens_str + ['$A'] + suffix_tokens_str  ,						pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str  ,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str  ,						self.prefix_tokens + self.suffix_tokens				)				)  ,						)







				def        _UpperCamelCase							(   self  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ = None				):
							'''simple docstring'''







							if not self.can_save_slow_tokenizer:
										raise ValueError(
										    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
										    'tokenizer.'				)

							if not os.path.isdir(SCREAMING_SNAKE_CASE__				):
										logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""				)
										return
							snake_case:       Tuple        =       os.path.join(
							    SCREAMING_SNAKE_CASE__  ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)

							if os.path.abspath(self.vocab_file				) != os.path.abspath(SCREAMING_SNAKE_CASE__				):
										copyfile(self.vocab_file  ,						SCREAMING_SNAKE_CASE__				)

							return (out_vocab_file,)







def        _snake_case   (_snake_case				:       list      ,    _snake_case				:       int      ,    _snake_case				:       int = 0      ,    _snake_case				:       int = 0)					->	int:
   _lowercase              =right or len(_snake_case) - 1
   if left > right:
      return -1
   elif list_data[left] == key:
      return left
   elif list_data[right] == key:
      return right
   else:
      return search(_snake_case      ,    _snake_case      ,    left + 1      ,    right - 1)


if __name__ == "__main__":
    import doctest

    doctest.testmod()


