


'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_nllb import NllbTokenizer
else:
		__A	=None


__A	=logging.get_logger(__name__)


__A	={'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

__A	={
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'
        ),
    },
}

__A	={
    'facebook/nllb-large-en-ro': 10_24,
    'facebook/nllb-200-distilled-600M': 10_24,
}

# fmt: off
__A	=['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']


class 		_snake_case					(     a__	):
					lowerCAmelCase      :str   =					VOCAB_FILES_NAMES
					lowerCAmelCase      :int   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase      :Optional[Any]   =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase      :List[Any]   =					['''input_ids''', '''attention_mask''']
					lowerCAmelCase      :List[str]   =					NllbTokenizer

					lowerCAmelCase      :List[int]   =					[]
					lowerCAmelCase      :List[int]   =					[]




					def __init__(      self							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase="<s>"							,							_lowerCamelCase="</s>"							,							_lowerCamelCase="</s>"							,							_lowerCamelCase="<s>"							,							_lowerCamelCase="<unk>"							,							_lowerCamelCase="<pad>"							,							_lowerCamelCase="<mask>"							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=False							,							**_lowerCamelCase							,							):
								# Mask token behave like a normal word, i.e. include the space before it
								UpperCAmelCase__  :					Tuple        =     AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else mask_token
								UpperCAmelCase__  :					int        =     legacy_behaviour
								super().__init__(
								    vocab_file=_lowerCamelCase							,							tokenizer_file=_lowerCamelCase							,							bos_token=_lowerCamelCase							,							eos_token=_lowerCamelCase							,							sep_token=_lowerCamelCase							,							cls_token=_lowerCamelCase							,							unk_token=_lowerCamelCase							,							pad_token=_lowerCamelCase							,							mask_token=_lowerCamelCase							,							src_lang=_lowerCamelCase							,							tgt_lang=_lowerCamelCase							,							additional_special_tokens=_lowerCamelCase							,							legacy_behaviour=_lowerCamelCase							,							**_lowerCamelCase							,							)

								UpperCAmelCase__  :					Any        =     vocab_file
								UpperCAmelCase__  :					List[str]        =     False if not self.vocab_file else True

								UpperCAmelCase__  :					Union[str, Any]        =     FAIRSEQ_LANGUAGE_CODES.copy()

								if additional_special_tokens is not None:
											# Only add those special tokens if they are not already there.
											_additional_special_tokens.extend(
											    [t for t in additional_special_tokens if t not in _additional_special_tokens])

								self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens})
								UpperCAmelCase__  :					Union[str, Any]        =     {
								    lang_code: self.convert_tokens_to_ids(_lowerCamelCase) for lang_code in FAIRSEQ_LANGUAGE_CODES
								}

								UpperCAmelCase__  :					List[Any]        =     src_lang if src_lang is not None else """eng_Latn"""
								UpperCAmelCase__  :					Tuple        =     self.convert_tokens_to_ids(self._src_lang)
								UpperCAmelCase__  :					Dict        =     tgt_lang
								self.set_src_lang_special_tokens(self._src_lang)




					@property
					def 				snake_case__						(      self):
								return self._src_lang




					@src_lang.setter
					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__  :					Union[str, Any]        =     new_src_lang
								self.set_src_lang_special_tokens(self._src_lang)




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__  :					Optional[int]        =     [self.sep_token_id]
								UpperCAmelCase__  :					Optional[Any]        =     [self.cls_token_id]

								if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							**_lowerCamelCase):
								if src_lang is None or tgt_lang is None:
											raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""")
								UpperCAmelCase__  :					str        =     src_lang
								UpperCAmelCase__  :					Dict        =     self(_lowerCamelCase							,							add_special_tokens=_lowerCamelCase							,							return_tensors=_lowerCamelCase							,							**_lowerCamelCase)
								UpperCAmelCase__  :					List[str]        =     self.convert_tokens_to_ids(_lowerCamelCase)
								UpperCAmelCase__  :					Optional[int]        =     tgt_lang_id
								return inputs




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = "eng_Latn"							,							_lowerCamelCase = None							,							_lowerCamelCase = "fra_Latn"							,							**_lowerCamelCase							,							):
								UpperCAmelCase__  :					Tuple        =     src_lang
								UpperCAmelCase__  :					List[Any]        =     tgt_lang
								return super().prepare_seqaseq_batch(_lowerCamelCase							,							_lowerCamelCase							,							**_lowerCamelCase)




					def 				snake_case__						(      self):
								return self.set_src_lang_special_tokens(self.src_lang)




					def 				snake_case__						(      self):
								return self.set_tgt_lang_special_tokens(self.tgt_lang)




					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__  :					List[Any]        =     self.convert_tokens_to_ids(_lowerCamelCase)

								if self.legacy_behaviour:
											UpperCAmelCase__  :					List[str]        =     []
											UpperCAmelCase__  :					Tuple        =     [self.eos_token_id, self.cur_lang_code]
								else:
											UpperCAmelCase__  :					Optional[int]        =     [self.cur_lang_code]
											UpperCAmelCase__  :					Dict        =     [self.eos_token_id]

								UpperCAmelCase__  :					Optional[int]        =     self.convert_ids_to_tokens(self.prefix_tokens)
								UpperCAmelCase__  :					Dict        =     self.convert_ids_to_tokens(self.suffix_tokens)

								UpperCAmelCase__  :					Optional[Any]        =     processors.TemplateProcessing(
								    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str							,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str							,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,							self.prefix_tokens + self.suffix_tokens))							,							)




					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__  :					int        =     self.convert_tokens_to_ids(_lowerCamelCase)
								if self.legacy_behaviour:
											UpperCAmelCase__  :					Tuple        =     []
											UpperCAmelCase__  :					Optional[Any]        =     [self.eos_token_id, self.cur_lang_code]
								else:
											UpperCAmelCase__  :					Any        =     [self.cur_lang_code]
											UpperCAmelCase__  :					Tuple        =     [self.eos_token_id]

								UpperCAmelCase__  :					List[str]        =     self.convert_ids_to_tokens(self.prefix_tokens)
								UpperCAmelCase__  :					Optional[int]        =     self.convert_ids_to_tokens(self.suffix_tokens)

								UpperCAmelCase__  :					Optional[int]        =     processors.TemplateProcessing(
								    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str							,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str							,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,							self.prefix_tokens + self.suffix_tokens))							,							)




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								if not self.can_save_slow_tokenizer:
											raise ValueError(
											    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
											    """tokenizer.""")

								if not os.path.isdir(_lowerCamelCase):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory.''')
											return
								UpperCAmelCase__  :					List[Any]        =     os.path.join(
								    _lowerCamelCase							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

								if os.path.abspath(self.vocab_file) != os.path.abspath(_lowerCamelCase):
											copyfile(self.vocab_file							,							_lowerCamelCase)

								return (out_vocab_file,)



'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__A	={
    'configuration_bert': ['BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'BertConfig', 'BertOnnxConfig'],
    'tokenization_bert': ['BasicTokenizer', 'BertTokenizer', 'WordpieceTokenizer'],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=['BertTokenizerFast']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=[
		    'BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'BertForMaskedLM',
		    'BertForMultipleChoice',
		    'BertForNextSentencePrediction',
		    'BertForPreTraining',
		    'BertForQuestionAnswering',
		    'BertForSequenceClassification',
		    'BertForTokenClassification',
		    'BertLayer',
		    'BertLMHeadModel',
		    'BertModel',
		    'BertPreTrainedModel',
		    'load_tf_weights_in_bert',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=[
		    'TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'TFBertEmbeddings',
		    'TFBertForMaskedLM',
		    'TFBertForMultipleChoice',
		    'TFBertForNextSentencePrediction',
		    'TFBertForPreTraining',
		    'TFBertForQuestionAnswering',
		    'TFBertForSequenceClassification',
		    'TFBertForTokenClassification',
		    'TFBertLMHeadModel',
		    'TFBertMainLayer',
		    'TFBertModel',
		    'TFBertPreTrainedModel',
		]
try:
		if not is_tensorflow_text_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=['TFBertTokenizer']

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=[
		    'FlaxBertForCausalLM',
		    'FlaxBertForMaskedLM',
		    'FlaxBertForMultipleChoice',
		    'FlaxBertForNextSentencePrediction',
		    'FlaxBertForPreTraining',
		    'FlaxBertForQuestionAnswering',
		    'FlaxBertForSequenceClassification',
		    'FlaxBertForTokenClassification',
		    'FlaxBertModel',
		    'FlaxBertPreTrainedModel',
		]

if TYPE_CHECKING:
		from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
		from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_bert_fast import BertTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_bert import (
				    BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    BertForMaskedLM,
				    BertForMultipleChoice,
				    BertForNextSentencePrediction,
				    BertForPreTraining,
				    BertForQuestionAnswering,
				    BertForSequenceClassification,
				    BertForTokenClassification,
				    BertLayer,
				    BertLMHeadModel,
				    BertModel,
				    BertPreTrainedModel,
				    load_tf_weights_in_bert,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_bert import (
				    TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFBertEmbeddings,
				    TFBertForMaskedLM,
				    TFBertForMultipleChoice,
				    TFBertForNextSentencePrediction,
				    TFBertForPreTraining,
				    TFBertForQuestionAnswering,
				    TFBertForSequenceClassification,
				    TFBertForTokenClassification,
				    TFBertLMHeadModel,
				    TFBertMainLayer,
				    TFBertModel,
				    TFBertPreTrainedModel,
				)

		try:
				if not is_tensorflow_text_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_bert_tf import TFBertTokenizer

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_bert import (
				    FlaxBertForCausalLM,
				    FlaxBertForMaskedLM,
				    FlaxBertForMultipleChoice,
				    FlaxBertForNextSentencePrediction,
				    FlaxBertForPreTraining,
				    FlaxBertForQuestionAnswering,
				    FlaxBertForSequenceClassification,
				    FlaxBertForTokenClassification,
				    FlaxBertModel,
				    FlaxBertPreTrainedModel,
				)

else:
		import sys

		__A	=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)