


import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


lowerCAmelCase__			:      List[str]								=threading.Lock()
lowerCAmelCase__			:      Optional[logging.Handler]								=None

lowerCAmelCase__			:      List[Any]								={
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

lowerCAmelCase__			:      Tuple								=logging.WARNING

lowerCAmelCase__			:      Dict								=True






def 						a__			(				):
			SCREAMING_SNAKE_CASE_		:     Optional[int]					=					os.getenv('TRANSFORMERS_VERBOSITY',      A__				)
			if env_level_str:
						if env_level_str in log_levels:
									return log_levels[env_level_str]
						else:
									logging.getLogger().warning(
									    F'''Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, '''
									    F'''has to be one of: { ', '.join(log_levels.keys()				) }'''				)
			return _default_log_level






def 						a__			(				):
			return __name__.split('.'				)[0]






def 						a__			(				):
			return logging.getLogger(_get_library_name()				)






def 						a__			(				):
			global _default_handler

			with _lock:
						if _default_handler:
									# This library has already configured the library root logger.
									return
						SCREAMING_SNAKE_CASE_		:     Optional[int]					=					logging.StreamHandler()  # Set sys.stderr as stream.
						SCREAMING_SNAKE_CASE_		:     Any					=					sys.stderr.flush

						# Apply our default configuration to the library root logger.
						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					_get_library_root_logger()
						library_root_logger.addHandler(_default_handler				)
						library_root_logger.setLevel(_get_default_logging_level()				)
						SCREAMING_SNAKE_CASE_		:     int					=					False






def 						a__			(				):
			global _default_handler

			with _lock:
						if not _default_handler:
									return

						SCREAMING_SNAKE_CASE_		:     List[str]					=					_get_library_root_logger()
						library_root_logger.removeHandler(_default_handler				)
						library_root_logger.setLevel(logging.NOTSET				)
						SCREAMING_SNAKE_CASE_		:     int					=					None






def 						a__			(				):
			return log_levels






def 						a__			(				A__ = None				):

			if name is None:
						SCREAMING_SNAKE_CASE_		:     List[str]					=					_get_library_name()

			_configure_library_root_logger()
			return logging.getLogger(A__				)






def 						a__			(				):
			_configure_library_root_logger()
			return _get_library_root_logger().getEffectiveLevel()






def 						a__			(				A__				):
			_configure_library_root_logger()
			_get_library_root_logger().setLevel(A__				)






def 						a__			(				):
			return set_verbosity(A__				)






def 						a__			(				):
			return set_verbosity(A__				)






def 						a__			(				):
			return set_verbosity(A__				)






def 						a__			(				):
			return set_verbosity(A__				)






def 						a__			(				):
			_configure_library_root_logger()

			assert _default_handler is not None
			_get_library_root_logger().removeHandler(_default_handler				)






def 						a__			(				):
			_configure_library_root_logger()

			assert _default_handler is not None
			_get_library_root_logger().addHandler(_default_handler				)






def 						a__			(				A__				):
			_configure_library_root_logger()

			assert handler is not None
			_get_library_root_logger().addHandler(A__				)






def 						a__			(				A__				):
			_configure_library_root_logger()

			assert handler is not None and handler not in _get_library_root_logger().handlers
			_get_library_root_logger().removeHandler(A__				)






def 						a__			(				):
			_configure_library_root_logger()
			SCREAMING_SNAKE_CASE_		:     str					=					False






def 						a__			(				):
			_configure_library_root_logger()
			SCREAMING_SNAKE_CASE_		:     str					=					True






def 						a__			(				):
			SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					_get_library_root_logger().handlers

			for handler in handlers:
						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					logging.Formatter('[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s'				)
						handler.setFormatter(A__				)






def 						a__			(				):
			SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					_get_library_root_logger().handlers

			for handler in handlers:
						handler.setFormatter(A__				)






def 						a__			(				self,      *A__,      **A__				):
			SCREAMING_SNAKE_CASE_		:     Dict					=					os.getenv('TRANSFORMERS_NO_ADVISORY_WARNINGS',      A__				)
			if no_advisory_warnings:
						return
			self.warning(*A__,      **A__				)


lowerCAmelCase__			:      Optional[Any]								=warning_advice






@functools.lru_cache(A__				)
def 						a__			(				self,      *A__,      **A__				):
			self.warning(*A__,      **A__				)


lowerCAmelCase__			:      List[str]								=warning_once







class 	__lowercase    :





							"""simple docstring"""







							def __init__(      self			,						*lowerCAmelCase__			,						**lowerCAmelCase__       ):  # pylint: disable=unused-argument





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[str]					=					args[0] if args else None






							def __iter__(      self       ):





										"""simple docstring"""




										return iter(self._iterator       )






							def __getattr__(      self			,						lowerCAmelCase__       ):





										"""simple docstring"""




										def empty_fn(*lowerCAmelCase__			,						**lowerCAmelCase__       ):  # pylint: disable=unused-argument
													return

										return empty_fn






							def __enter__(      self       ):





										"""simple docstring"""




										return self






							def __exit__(      self			,						lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__       ):





										"""simple docstring"""




										return







class 	__lowercase    :





							"""simple docstring"""







							def __call__(      self			,						*lowerCAmelCase__			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										if _tqdm_active:
													return tqdm_lib.tqdm(*lowerCAmelCase__			,						**lowerCAmelCase__       )
										else:
													return EmptyTqdm(*lowerCAmelCase__			,						**lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self			,						*lowerCAmelCase__			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					None
										if _tqdm_active:
													return tqdm_lib.tqdm.set_lock(*lowerCAmelCase__			,						**lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										if _tqdm_active:
													return tqdm_lib.tqdm.get_lock()







lowerCAmelCase__			:      str								=_tqdm_cls()






def 						a__			(				):
			global _tqdm_active
			return bool(_tqdm_active				)






def 						a__			(				):
			global _tqdm_active
			SCREAMING_SNAKE_CASE_		:     Any					=					True
			hf_hub_utils.enable_progress_bars()






def 						a__			(				):
			global _tqdm_active
			SCREAMING_SNAKE_CASE_		:     Optional[int]					=					False
			hf_hub_utils.disable_progress_bars()









'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_mbart import MBartTokenizer
else:
				UpperCAmelCase_						:  List[str]				  =						None


UpperCAmelCase_						:  Tuple				  =						logging.get_logger(__name__)


UpperCAmelCase_						:  List[Any]				  =						{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_						:  Dict				  =						{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

UpperCAmelCase_						:  str				  =						{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
UpperCAmelCase_						:  List[str]				  =						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']





class        lowercase__    (		_snake_case   ):





							'''simple docstring'''


							A_				: Any       =    VOCAB_FILES_NAMES
							A_				: List[str]       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_				: Any       =    PRETRAINED_VOCAB_FILES_MAP
							A_				: Tuple       =    ["""input_ids""", """attention_mask"""]
							A_				: Dict       =    MBartTokenizer

							A_				: List[int]       =    []
							A_				: List[int]       =    []






							def __init__(       self       ,   __snake_case=None       ,   __snake_case=None       ,   __snake_case="<s>"       ,   __snake_case="</s>"       ,   __snake_case="</s>"       ,   __snake_case="<s>"       ,   __snake_case="<unk>"       ,   __snake_case="<pad>"       ,   __snake_case="<mask>"       ,   __snake_case=None       ,   __snake_case=None       ,   __snake_case=None       ,   **__snake_case       ,   ):
											# Mask token behave like a normal word, i.e. include the space before it
											_SCREAMING_SNAKE_CASE					:			List[Any]							= AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case    ) if isinstance(__snake_case       ,   __snake_case    ) else mask_token

											super().__init__(
											    vocab_file=__snake_case       ,   tokenizer_file=__snake_case       ,   bos_token=__snake_case       ,   eos_token=__snake_case       ,   sep_token=__snake_case       ,   cls_token=__snake_case       ,   unk_token=__snake_case       ,   pad_token=__snake_case       ,   mask_token=__snake_case       ,   src_lang=__snake_case       ,   tgt_lang=__snake_case       ,   additional_special_tokens=__snake_case       ,   **__snake_case       ,   )

											_SCREAMING_SNAKE_CASE					:			List[str]							= vocab_file
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= False if not self.vocab_file else True

											_SCREAMING_SNAKE_CASE					:			int							= FAIRSEQ_LANGUAGE_CODES.copy()

											if additional_special_tokens is not None:
															# Only add those special tokens if they are not already there.
															_additional_special_tokens.extend(
															    [t for t in additional_special_tokens if t not in _additional_special_tokens]    )

											self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}    )
											_SCREAMING_SNAKE_CASE					:			str							= {
											    lang_code: self.convert_tokens_to_ids(__snake_case    ) for lang_code in FAIRSEQ_LANGUAGE_CODES
											}

											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= src_lang if src_lang is not None else """en_XX"""
											_SCREAMING_SNAKE_CASE					:			str							= self.convert_tokens_to_ids(self._src_lang    )
											_SCREAMING_SNAKE_CASE					:			Any							= tgt_lang
											self.set_src_lang_special_tokens(self._src_lang    )






							@property
							def UpperCAmelCase_      (       self    ):
											return self._src_lang






							@src_lang.setter
							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			Any							= new_src_lang
											self.set_src_lang_special_tokens(self._src_lang    )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= [self.sep_token_id]
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= [self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case       ,   **__snake_case    ):
											if src_lang is None or tgt_lang is None:
															raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
											_SCREAMING_SNAKE_CASE					:			Dict							= src_lang
											_SCREAMING_SNAKE_CASE					:			List[Any]							= self(__snake_case       ,   add_special_tokens=__snake_case       ,   return_tensors=__snake_case       ,   **__snake_case    )
											_SCREAMING_SNAKE_CASE					:			Dict							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tgt_lang_id
											return inputs






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = "en_XX"       ,   __snake_case = None       ,   __snake_case = "ro_RO"       ,   **__snake_case       ,   ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= src_lang
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= tgt_lang
											return super().prepare_seqaseq_batch(__snake_case       ,   __snake_case       ,   **__snake_case    )






							def UpperCAmelCase_      (       self    ):
											return self.set_src_lang_special_tokens(self.src_lang    )






							def UpperCAmelCase_      (       self    ):
											return self.set_tgt_lang_special_tokens(self.tgt_lang    )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			str							= []
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= [self.eos_token_id, self.cur_lang_code]

											_SCREAMING_SNAKE_CASE					:			int							= self.convert_ids_to_tokens(self.prefix_tokens    )
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= self.convert_ids_to_tokens(self.suffix_tokens    )

											_SCREAMING_SNAKE_CASE					:			Optional[int]							= processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,   pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,   self.prefix_tokens + self.suffix_tokens    )    )       ,   )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.convert_tokens_to_ids(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			List[str]							= []
											_SCREAMING_SNAKE_CASE					:			Tuple							= [self.eos_token_id, self.cur_lang_code]

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= self.convert_ids_to_tokens(self.prefix_tokens    )
											_SCREAMING_SNAKE_CASE					:			str							= self.convert_ids_to_tokens(self.suffix_tokens    )

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,   pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,   self.prefix_tokens + self.suffix_tokens    )    )       ,   )






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""    )

											if not os.path.isdir(__snake_case    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""    )
															return
											_SCREAMING_SNAKE_CASE					:			List[str]							= os.path.join(
											    __snake_case       ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(__snake_case    ):
															copyfile(self.vocab_file       ,   __snake_case    )

											return (out_vocab_file,)




