




"""simple docstring"""




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


a_        =       logging.get_logger(__name__)

a_        =       '▁'

a_        =       {'vocab_file': 'sentencepiece.bpe.model'}

a_        =       {
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model'
        ),
    }
}

a_        =       {
    'facebook/nllb-200-distilled-600M': 1_0_2_4,
}

# fmt: off
a_        =       ['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']

class   UpperCAmelCase_    (			__lowercase		):
		UpperCamelCase							=VOCAB_FILES_NAMES
		UpperCamelCase							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCamelCase							=PRETRAINED_VOCAB_FILES_MAP
		UpperCamelCase							=["input_ids", "attention_mask"]

		UpperCamelCase							=[]
		UpperCamelCase							=[]
		def __init__(						self				,						UpperCamelCase_				,						UpperCamelCase_="<s>"				,						UpperCamelCase_="</s>"				,						UpperCamelCase_="</s>"				,						UpperCamelCase_="<s>"				,						UpperCamelCase_="<unk>"				,						UpperCamelCase_="<pad>"				,						UpperCamelCase_="<mask>"				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_ = None				,						UpperCamelCase_=None				,						UpperCamelCase_=False				,						**UpperCamelCase_				,						)						->       Dict:
					# Mask token behave like a normal word, i.e. include the space before it
					__lowercase			:  Union[str, Any]								=       AddedToken(_A				,						lstrip=_A				,						rstrip=_A       ) if isinstance(_A				,						_A       ) else mask_token

					__lowercase			:  Optional[Any]								=       {} if sp_model_kwargs is None else sp_model_kwargs
					__lowercase			:  str								=       legacy_behaviour

					super().__init__(
					    bos_token=_A				,						eos_token=_A				,						unk_token=_A				,						sep_token=_A				,						cls_token=_A				,						pad_token=_A				,						mask_token=_A				,						tokenizer_file=_A				,						src_lang=_A				,						tgt_lang=_A				,						additional_special_tokens=_A				,						sp_model_kwargs=self.sp_model_kwargs				,						legacy_behaviour=_A				,						**_A				,						)

					__lowercase			:  int								=       spm.SentencePieceProcessor(**self.sp_model_kwargs       )
					self.sp_model.Load(str(_A       )       )
					__lowercase			:  Optional[int]								=       vocab_file
					# Original fairseq vocab and spm vocab must be "aligned":
					# Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
					# -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
					# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
					# spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

					# Mimic fairseq token-to-id alignment for the first 4 token
					__lowercase			:  Dict								=       {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

					# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
					__lowercase			:  Union[str, Any]								=       1

					__lowercase			:  Union[str, Any]								=       len(self.sp_model       )
					__lowercase			:  Union[str, Any]								=       {
					    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_A       )
					}
					__lowercase			:  List[Any]								=       {v: k for k, v in self.lang_code_to_id.items()}
					__lowercase			:  Union[str, Any]								=       len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset

					self.fairseq_tokens_to_ids.update(self.lang_code_to_id       )
					__lowercase			:  Optional[int]								=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}
					__lowercase			:  List[Any]								=       list(self.lang_code_to_id.keys()       )

					if additional_special_tokens is not None:
								# Only add those special tokens if they are not already there.
								self._additional_special_tokens.extend(
								    [t for t in additional_special_tokens if t not in self._additional_special_tokens]       )

					__lowercase			:  str								=       src_lang if src_lang is not None else '''eng_Latn'''
					__lowercase			:  Optional[int]								=       self.lang_code_to_id[self._src_lang]
					__lowercase			:  Tuple								=       tgt_lang
					self.set_src_lang_special_tokens(self._src_lang       )
		def __getstate__(						self       )						->       List[Any]:
					__lowercase			:  Optional[int]								=       self.__dict__.copy()
					__lowercase			:  List[str]								=       None
					__lowercase			:  Optional[int]								=       self.sp_model.serialized_model_proto()
					return state
		def __setstate__(						self				,						UpperCamelCase_       )						->       List[Any]:
					__lowercase			:  Optional[Any]								=       d

					# for backward compatibility
					if not hasattr(self				,						'''sp_model_kwargs'''       ):
								__lowercase			:  List[str]								=       {}

					__lowercase			:  List[Any]								=       spm.SentencePieceProcessor(**self.sp_model_kwargs       )
					self.sp_model.LoadFromSerializedProto(self.sp_model_proto       )
		@property
		def 	_lowerCamelCase							(						self       )						->       Optional[Any]:
					return len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset + 1  # Plus 1 for the mask token
		@property
		def 	_lowerCamelCase							(						self       )						->       Dict:
					return self._src_lang
		@src_lang.setter
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Optional[int]:
					__lowercase			:  Dict								=       new_src_lang
					self.set_src_lang_special_tokens(self._src_lang       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None				,						UpperCamelCase_ = False       )						->       int:

					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=_A				,						token_ids_a=_A				,						already_has_special_tokens=_A       )

					__lowercase			:  int								=       [1] * len(self.prefix_tokens       )
					__lowercase			:  Union[str, Any]								=       [1] * len(self.suffix_tokens       )
					if token_ids_a is None:
								return prefix_ones + ([0] * len(_A       )) + suffix_ones
					return prefix_ones + ([0] * len(_A       )) + ([0] * len(_A       )) + suffix_ones
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None       )						->       List[str]:
					if token_ids_a is None:
								return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None       )						->       Dict:
					__lowercase			:  Any								=       [self.sep_token_id]
					__lowercase			:  Optional[int]								=       [self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_				,						**UpperCamelCase_       )						->       List[str]:
					if src_lang is None or tgt_lang is None:
								raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''       )
					__lowercase			:  Union[str, Any]								=       src_lang
					__lowercase			:  List[Any]								=       self(_A				,						add_special_tokens=_A				,						return_tensors=_A				,						**_A       )
					__lowercase			:  Optional[Any]								=       self.convert_tokens_to_ids(_A       )
					__lowercase			:  Union[str, Any]								=       tgt_lang_id
					return inputs
		def 	_lowerCamelCase							(						self       )						->       List[Any]:
					__lowercase			:  Union[str, Any]								=       {self.convert_ids_to_tokens(_A       ): i for i in range(self.vocab_size       )}
					vocab.update(self.added_tokens_encoder       )
					return vocab
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Optional[Any]:
					return self.sp_model.encode(_A				,						out_type=_A       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Any:
					if token in self.fairseq_tokens_to_ids:
								return self.fairseq_tokens_to_ids[token]
					__lowercase			:  int								=       self.sp_model.PieceToId(_A       )

					# Need to return unknown token if the SP model returned 0
					return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Any:
					if index in self.fairseq_ids_to_tokens:
								return self.fairseq_ids_to_tokens[index]
					return self.sp_model.IdToPiece(index - self.fairseq_offset       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Union[str, Any]:
					__lowercase			:  List[Any]								=       ''''''.join(_A       ).replace(_A				,						''' '''       ).strip()
					return out_string
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None       )						->       List[str]:
					if not os.path.isdir(_A       ):
								logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
								return
					__lowercase			:  Optional[Any]								=       os.path.join(
					    _A				,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

					if os.path.abspath(self.vocab_file       ) != os.path.abspath(_A       ) and os.path.isfile(self.vocab_file       ):
								copyfile(self.vocab_file				,						_A       )
					elif not os.path.isfile(self.vocab_file       ):
								with open(_A				,						'''wb'''       ) as fi:
											__lowercase			:  Union[str, Any]								=       self.sp_model.serialized_model_proto()
											fi.write(_A       )

					return (out_vocab_file,)
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = "eng_Latn"				,						UpperCamelCase_ = None				,						UpperCamelCase_ = "fra_Latn"				,						**UpperCamelCase_				,						)						->       List[str]:
					__lowercase			:  int								=       src_lang
					__lowercase			:  List[str]								=       tgt_lang
					return super().prepare_seqaseq_batch(_A				,						_A				,						**_A       )
		def 	_lowerCamelCase							(						self       )						->       List[Any]:
					return self.set_src_lang_special_tokens(self.src_lang       )
		def 	_lowerCamelCase							(						self       )						->       Optional[int]:
					return self.set_tgt_lang_special_tokens(self.tgt_lang       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Any:
					__lowercase			:  Tuple								=       self.lang_code_to_id[src_lang]
					if self.legacy_behaviour:
								__lowercase			:  Optional[Any]								=       []
								__lowercase			:  Optional[int]								=       [self.eos_token_id, self.cur_lang_code]
					else:
								__lowercase			:  int								=       [self.cur_lang_code]
								__lowercase			:  str								=       [self.eos_token_id]
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Any:
					__lowercase			:  Tuple								=       self.lang_code_to_id[lang]
					if self.legacy_behaviour:
								__lowercase			:  Optional[Any]								=       []
								__lowercase			:  int								=       [self.eos_token_id, self.cur_lang_code]
					else:
								__lowercase			:  Any								=       [self.cur_lang_code]
								__lowercase			:  List[str]								=       [self.eos_token_id]




import logging
import os

from .state import PartialState
class     lowerCAmelCase_     (       logging.LoggerAdapter ):







							@staticmethod
							def  UpperCamelCase_						(   _A      :     Any		):
												_UpperCamelCase			     =     PartialState()
												return not main_process_only or (main_process_only and state.is_main_process)







							def  UpperCamelCase_						(   self      :     Union[str, Any] ,			_A      :     Optional[Any] ,			_A      :     str ,			*_A      :     int ,			**_A      :     List[Any]		):
												if PartialState._shared_state == {}:
																	raise RuntimeError(
																	    '''You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility.'''		)
												_UpperCamelCase			     =     kwargs.pop('''main_process_only''' ,			_A		)
												_UpperCamelCase			     =     kwargs.pop('''in_order''' ,			_A		)

												if self.isEnabledFor(_A		):
																	if self._should_log(_A		):
																						_UpperCamelCase     ,    _UpperCamelCase			     =     self.process(_A ,			_A		)
																						self.logger.log(_A ,			_A ,			*_A ,			**_A		)

																	elif in_order:
																						_UpperCamelCase			     =     PartialState()
																						for i in range(state.num_processes		):
																											if i == state.process_index:
																																_UpperCamelCase     ,    _UpperCamelCase			     =     self.process(_A ,			_A		)
																																self.logger.log(_A ,			_A ,			*_A ,			**_A		)
																											state.wait_for_everyone()




def 					_snake_case	(  __snake_case  ,     __snake_case = None						):
					if log_level is None:
										_UpperCamelCase			     =     os.environ.get('''ACCELERATE_LOG_LEVEL'''  ,     __snake_case						)
					_UpperCamelCase			     =     logging.getLogger(__snake_case						)
					if log_level is not None:
										logger.setLevel(log_level.upper()						)
										logger.root.setLevel(log_level.upper()						)
					return MultiProcessAdapter(__snake_case  ,     {}						)
