




"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_	     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_	     =       '▁'

SCREAMING_SNAKE_CASE_	     =       {'vocab_file': 'sentencepiece.bpe.model'}

SCREAMING_SNAKE_CASE_	     =       {
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model'
        ),
    }
}

SCREAMING_SNAKE_CASE_	     =       {
    'facebook/nllb-200-distilled-600M': 1024,
}

# fmt: off
SCREAMING_SNAKE_CASE_	     =       ['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']

class 				snake_case_			(					lowerCamelCase_			):







   """simple docstring"""




   A_				      =       VOCAB_FILES_NAMES
   A_				      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A_				      =       PRETRAINED_VOCAB_FILES_MAP
   A_				      =       ['''input_ids''', '''attention_mask''']

   A_				      =       []
   A_				      =       []







   def __init__(		self       ,					lowerCamelCase_       ,					lowerCamelCase_="<s>"       ,					lowerCamelCase_="</s>"       ,					lowerCamelCase_="</s>"       ,					lowerCamelCase_="<s>"       ,					lowerCamelCase_="<unk>"       ,					lowerCamelCase_="<pad>"       ,					lowerCamelCase_="<mask>"       ,					lowerCamelCase_=None       ,					lowerCamelCase_=None       ,					lowerCamelCase_=None       ,					lowerCamelCase_ = None       ,					lowerCamelCase_=None       ,					lowerCamelCase_=False       ,					**lowerCamelCase_       ,					)				->   Optional[Any]:
       UpperCamelCase              =	AddedToken(__lowercase       ,					lstrip=__lowercase       ,					rstrip=__lowercase) if isinstance(__lowercase       ,					__lowercase) else mask_token

       UpperCamelCase              =	{} if sp_model_kwargs is None else sp_model_kwargs
       UpperCamelCase              =	legacy_behaviour

       super().__init__(
           bos_token=__lowercase       ,					eos_token=__lowercase       ,					unk_token=__lowercase       ,					sep_token=__lowercase       ,					cls_token=__lowercase       ,					pad_token=__lowercase       ,					mask_token=__lowercase       ,					tokenizer_file=__lowercase       ,					src_lang=__lowercase       ,					tgt_lang=__lowercase       ,					additional_special_tokens=__lowercase       ,					sp_model_kwargs=self.sp_model_kwargs       ,					legacy_behaviour=__lowercase       ,					**__lowercase       ,					)

       UpperCamelCase              =	spm.SentencePieceProcessor(**self.sp_model_kwargs)
       self.sp_model.Load(str(__lowercase))
       UpperCamelCase              =	vocab_file
       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
       # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
       # spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

       # Mimic fairseq token-to-id alignment for the first 4 token
       UpperCamelCase              =	{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       UpperCamelCase              =	1

       UpperCamelCase              =	len(self.sp_model)
       UpperCamelCase              =	{
           code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__lowercase)
       }
       UpperCamelCase              =	{v: k for k, v in self.lang_code_to_id.items()}
       UpperCamelCase              =	len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

       self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
       UpperCamelCase              =	{v: k for k, v in self.fairseq_tokens_to_ids.items()}
       UpperCamelCase              =	list(self.lang_code_to_id.keys())

       if additional_special_tokens is not None:
           # Only add those special tokens if they are not already there.
           self._additional_special_tokens.extend(
               [t for t in additional_special_tokens if t not in self._additional_special_tokens])

       UpperCamelCase              =	src_lang if src_lang is not None else '''eng_Latn'''
       UpperCamelCase              =	self.lang_code_to_id[self._src_lang]
       UpperCamelCase              =	tgt_lang
       self.set_src_lang_special_tokens(self._src_lang)







   def __getstate__(		self)				->   Union[str, Any]:
       UpperCamelCase              =	self.__dict__.copy()
       UpperCamelCase              =	None
       UpperCamelCase              =	self.sp_model.serialized_model_proto()
       return state







   def __setstate__(		self       ,					lowerCamelCase_)				->   Any:
       UpperCamelCase              =	d

       # for backward compatibility
       if not hasattr(self       ,					'''sp_model_kwargs'''):
           UpperCamelCase              =	{}

       UpperCamelCase              =	spm.SentencePieceProcessor(**self.sp_model_kwargs)
       self.sp_model.LoadFromSerializedProto(self.sp_model_proto)







   @property
   def   UpperCAmelCase__			(		self)				->   Union[str, Any]:
       return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token







   @property
   def   UpperCAmelCase__			(		self)				->   Dict:
       return self._src_lang







   @src_lang.setter
   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Optional[int]:
       UpperCamelCase              =	new_src_lang
       self.set_src_lang_special_tokens(self._src_lang)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None       ,					lowerCamelCase_ = False)				->   int:







       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__lowercase       ,					token_ids_a=__lowercase       ,					already_has_special_tokens=__lowercase)

       UpperCamelCase              =	[1] * len(self.prefix_tokens)
       UpperCamelCase              =	[1] * len(self.suffix_tokens)
       if token_ids_a is None:
           return prefix_ones + ([0] * len(__lowercase)) + suffix_ones
       return prefix_ones + ([0] * len(__lowercase)) + ([0] * len(__lowercase)) + suffix_ones







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   Optional[int]:







       if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   str:
       UpperCamelCase              =	[self.sep_token_id]
       UpperCamelCase              =	[self.cls_token_id]

       if token_ids_a is None:
           return len(cls + token_ids_a + sep) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					**lowerCamelCase_)				->   Optional[int]:







       if src_lang is None or tgt_lang is None:
           raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model''')
       UpperCamelCase              =	src_lang
       UpperCamelCase              =	self(__lowercase       ,					add_special_tokens=__lowercase       ,					return_tensors=__lowercase       ,					**__lowercase)
       UpperCamelCase              =	self.convert_tokens_to_ids(__lowercase)
       UpperCamelCase              =	tgt_lang_id
       return inputs







   def   UpperCAmelCase__			(		self)				->   List[str]:
       UpperCamelCase              =	{self.convert_ids_to_tokens(__lowercase): i for i in range(self.vocab_size)}
       vocab.update(self.added_tokens_encoder)
       return vocab







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Dict:
       return self.sp_model.encode(__lowercase       ,					out_type=__lowercase)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   int:







       if token in self.fairseq_tokens_to_ids:
           return self.fairseq_tokens_to_ids[token]
       UpperCamelCase              =	self.sp_model.PieceToId(__lowercase)

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Any:







       if index in self.fairseq_ids_to_tokens:
           return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Any:
       UpperCamelCase              =	''''''.join(__lowercase).replace(__lowercase       ,					''' ''').strip()
       return out_string







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   Optional[Any]:







       if not os.path.isdir(__lowercase):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory')
           return
       UpperCamelCase              =	os.path.join(
           __lowercase       ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

       if os.path.abspath(self.vocab_file) != os.path.abspath(__lowercase) and os.path.isfile(self.vocab_file):
           copyfile(self.vocab_file       ,					__lowercase)
       elif not os.path.isfile(self.vocab_file):
           with open(__lowercase       ,					'''wb''') as fi:
               UpperCamelCase              =	self.sp_model.serialized_model_proto()
               fi.write(__lowercase)

       return (out_vocab_file,)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = "eng_Latn"       ,					lowerCamelCase_ = None       ,					lowerCamelCase_ = "fra_Latn"       ,					**lowerCamelCase_       ,					)				->   Union[str, Any]:
       UpperCamelCase              =	src_lang
       UpperCamelCase              =	tgt_lang
       return super().prepare_seqaseq_batch(__lowercase       ,					__lowercase       ,					**__lowercase)







   def   UpperCAmelCase__			(		self)				->   Tuple:
       return self.set_src_lang_special_tokens(self.src_lang)







   def   UpperCAmelCase__			(		self)				->   Dict:
       return self.set_tgt_lang_special_tokens(self.tgt_lang)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   int:
       UpperCamelCase              =	self.lang_code_to_id[src_lang]
       if self.legacy_behaviour:
           UpperCamelCase              =	[]
           UpperCamelCase              =	[self.eos_token_id, self.cur_lang_code]
       else:
           UpperCamelCase              =	[self.cur_lang_code]
           UpperCamelCase              =	[self.eos_token_id]







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   str:
       UpperCamelCase              =	self.lang_code_to_id[lang]
       if self.legacy_behaviour:
           UpperCamelCase              =	[]
           UpperCamelCase              =	[self.eos_token_id, self.cur_lang_code]
       else:
           UpperCamelCase              =	[self.cur_lang_code]
           UpperCamelCase              =	[self.eos_token_id]


'''simple docstring'''

import os
from collections.abc import Iterator

def 					__UpperCamelCase				(						lowercase__    :	str = "."			):






     '''simple docstring'''





     for dir_path, dir_names, filenames in os.walk(lowercase__			):
          __lowercase      		=[d for d in dir_names if d != 'scripts' and d[0] not in '._']
          for filename in filenames:
               if filename == "__init__.py":
                    continue
               if os.path.splitext(lowercase__			)[1] in (".py", ".ipynb"):
                    yield os.path.join(lowercase__,       lowercase__			).lstrip('./'			)

def 					__UpperCamelCase				(						lowercase__    :	Optional[Any]			):






     '''simple docstring'''





     return F'''{i * "  "}*''' if i else "\n##"

def 					__UpperCamelCase				(						lowercase__    :	str,       lowercase__    :	str			):






     '''simple docstring'''





     __lowercase      		=old_path.split(os.sep			)
     for i, new_part in enumerate(new_path.split(os.sep			)			):
          if (i + 1 > len(lowercase__			) or old_parts[i] != new_part) and new_part:
               print(F'''{md_prefix(lowercase__			)} {new_part.replace("_",       " "			).title()}'''			)
     return new_path

def 					__UpperCamelCase				(						lowercase__    :	str = "."			):






     '''simple docstring'''





     __lowercase      		=''
     for filepath in sorted(good_file_paths(lowercase__			)			):
          __lowercase		,  __lowercase      		=os.path.split(lowercase__			)
          if filepath != old_path:
               __lowercase      		=print_path(lowercase__,       lowercase__			)
          __lowercase      		=(filepath.count(os.sep			) + 1) if filepath else 0
          __lowercase      		=F'''{filepath}/{filename}'''.replace(' ',       '%20'			)
          __lowercase      		=os.path.splitext(filename.replace('_',       ' '			).title()			)[0]
          print(F'''{md_prefix(lowercase__			)} [{filename}]({url})'''			)


if __name__ == "__main__":
       print_directory_md('''.''')


