





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase_         =						logging.get_logger(__name__)

lowercase_         =						'▁'

lowercase_         =						{'vocab_file': 'sentencepiece.bpe.model'}

lowercase_         =						{
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model'
        ),
    }
}

lowercase_         =						{
    'facebook/nllb-200-distilled-600M': 1_024,
}

# fmt: off
lowercase_         =						['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']
class 							SCREAMING_SNAKE_CASE  (__SCREAMING_SNAKE_CASE ):
   _UpperCamelCase       :						Optional[int]					   =			VOCAB_FILES_NAMES
   _UpperCamelCase       :						Optional[int]					   =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase       :						Dict					   =			PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase       :						Union[str, Any]					   =			['input_ids', 'attention_mask']

   _UpperCamelCase       :						List[str]					   =			[]
   _UpperCamelCase       :						Tuple					   =			[]


   def __init__(       self     :							Optional[int]     ,	a     :							Tuple     ,	a     :							Union[str, Any]="<s>"     ,	a     :							Union[str, Any]="</s>"     ,	a     :							Dict="</s>"     ,	a     :							str="<s>"     ,	a     :							Dict="<unk>"     ,	a     :							Optional[int]="<pad>"     ,	a     :							int="<mask>"     ,	a     :							Tuple=None     ,	a     :							Tuple=None     ,	a     :							Optional[Any]=None     ,	a     :							Optional[Dict[str, Any]] = None     ,	a     :							Union[str, Any]=None     ,	a     :							Tuple=False     ,	**a     :							Optional[int]     ,	)->  List[str]:





     """simple docstring"""







     lowercase__           =    AddedToken(_SCREAMING_SNAKE_CASE     ,	lstrip=_SCREAMING_SNAKE_CASE     ,	rstrip=_SCREAMING_SNAKE_CASE			) if isinstance(_SCREAMING_SNAKE_CASE     ,	_SCREAMING_SNAKE_CASE			) else mask_token

     lowercase__           =    {} if sp_model_kwargs is None else sp_model_kwargs
     lowercase__           =    legacy_behaviour

     super().__init__(
         bos_token=_SCREAMING_SNAKE_CASE     ,	eos_token=_SCREAMING_SNAKE_CASE     ,	unk_token=_SCREAMING_SNAKE_CASE     ,	sep_token=_SCREAMING_SNAKE_CASE     ,	cls_token=_SCREAMING_SNAKE_CASE     ,	pad_token=_SCREAMING_SNAKE_CASE     ,	mask_token=_SCREAMING_SNAKE_CASE     ,	tokenizer_file=_SCREAMING_SNAKE_CASE     ,	src_lang=_SCREAMING_SNAKE_CASE     ,	tgt_lang=_SCREAMING_SNAKE_CASE     ,	additional_special_tokens=_SCREAMING_SNAKE_CASE     ,	sp_model_kwargs=self.sp_model_kwargs     ,	legacy_behaviour=_SCREAMING_SNAKE_CASE     ,	**_SCREAMING_SNAKE_CASE     ,	)

     lowercase__           =    spm.SentencePieceProcessor(**self.sp_model_kwargs			)
     self.sp_model.Load(str(_SCREAMING_SNAKE_CASE			)			)
     lowercase__           =    vocab_file
     # Original fairseq vocab and spm vocab must be "aligned":
     # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
     # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
     # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
     # spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

     # Mimic fairseq token-to-id alignment for the first 4 token
     lowercase__           =    {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

     # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
     lowercase__           =    1

     lowercase__           =    len(self.sp_model			)
     lowercase__           =    {
         code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_SCREAMING_SNAKE_CASE			)
     }
     lowercase__           =    {v: k for k, v in self.lang_code_to_id.items()}
     lowercase__           =    len(self.sp_model			) + len(self.lang_code_to_id			) + self.fairseq_offset

     self.fairseq_tokens_to_ids.update(self.lang_code_to_id			)
     lowercase__           =    {v: k for k, v in self.fairseq_tokens_to_ids.items()}
     lowercase__           =    list(self.lang_code_to_id.keys()			)

     if additional_special_tokens is not None:
       # Only add those special tokens if they are not already there.
       self._additional_special_tokens.extend(
           [t for t in additional_special_tokens if t not in self._additional_special_tokens]			)

     lowercase__           =    src_lang if src_lang is not None else "eng_Latn"
     lowercase__           =    self.lang_code_to_id[self._src_lang]
     lowercase__           =    tgt_lang
     self.set_src_lang_special_tokens(self._src_lang			)


   def __getstate__(       self     :							Any			)->  str:





     """simple docstring"""







     lowercase__           =    self.__dict__.copy()
     lowercase__           =    None
     lowercase__           =    self.sp_model.serialized_model_proto()
     return state


   def __setstate__(       self     :							Optional[Any]     ,	a     :							List[Any]			)->  Tuple:





     """simple docstring"""







     lowercase__           =    d

     # for backward compatibility
     if not hasattr(self     ,	'sp_model_kwargs'			):
       lowercase__           =    {}

     lowercase__           =    spm.SentencePieceProcessor(**self.sp_model_kwargs			)
     self.sp_model.LoadFromSerializedProto(self.sp_model_proto			)


   @property
   def 			SCREAMING_SNAKE_CASE_				(       self     :							str			)->  Tuple:





     """simple docstring"""







     return len(self.sp_model			) + len(self.lang_code_to_id			) + self.fairseq_offset + 1  # Plus 1 for the mask token


   @property
   def 			SCREAMING_SNAKE_CASE_				(       self     :							List[Any]			)->  str:





     """simple docstring"""







     return self._src_lang


   @src_lang.setter
   def 			SCREAMING_SNAKE_CASE_				(       self     :							Any     ,	a     :							str			)->  None:





     """simple docstring"""







     lowercase__           =    new_src_lang
     self.set_src_lang_special_tokens(self._src_lang			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Union[str, Any]     ,	a     :							List[int]     ,	a     :							Optional[List[int]] = None     ,	a     :							bool = False			)->  List[int]:





     """simple docstring"""







     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=_SCREAMING_SNAKE_CASE     ,	token_ids_a=_SCREAMING_SNAKE_CASE     ,	already_has_special_tokens=_SCREAMING_SNAKE_CASE			)

     lowercase__           =    [1] * len(self.prefix_tokens			)
     lowercase__           =    [1] * len(self.suffix_tokens			)
     if token_ids_a is None:
       return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE			)) + suffix_ones
     return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE			)) + ([0] * len(_SCREAMING_SNAKE_CASE			)) + suffix_ones


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Optional[int]     ,	a     :							List[int]     ,	a     :							Optional[List[int]] = None			)->  List[int]:





     """simple docstring"""







     if token_ids_a is None:
       return self.prefix_tokens + token_ids_a + self.suffix_tokens
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Union[str, Any]     ,	a     :							List[int]     ,	a     :							Optional[List[int]] = None			)->  List[int]:





     """simple docstring"""







     lowercase__           =    [self.sep_token_id]
     lowercase__           =    [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Optional[Any]     ,	a     :							Union[str, Any]     ,	a     :							str     ,	a     :							Optional[str]     ,	a     :							Optional[str]     ,	**a     :							Union[str, Any]			)->  Union[str, Any]:





     """simple docstring"""







     if src_lang is None or tgt_lang is None:
       raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'			)
     lowercase__           =    src_lang
     lowercase__           =    self(_SCREAMING_SNAKE_CASE     ,	add_special_tokens=_SCREAMING_SNAKE_CASE     ,	return_tensors=_SCREAMING_SNAKE_CASE     ,	**_SCREAMING_SNAKE_CASE			)
     lowercase__           =    self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE			)
     lowercase__           =    tgt_lang_id
     return inputs


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Optional[Any]			)->  Optional[Any]:





     """simple docstring"""







     lowercase__           =    {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE			): i for i in range(self.vocab_size			)}
     vocab.update(self.added_tokens_encoder			)
     return vocab


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Union[str, Any]     ,	a     :							str			)->  List[str]:





     """simple docstring"""







     return self.sp_model.encode(_SCREAMING_SNAKE_CASE     ,	out_type=_SCREAMING_SNAKE_CASE			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							List[Any]     ,	a     :							List[Any]			)->  int:





     """simple docstring"""







     if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
     lowercase__           =    self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE			)

     # Need to return unknown token if the SP model returned 0
     return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


   def 			SCREAMING_SNAKE_CASE_				(       self     :							int     ,	a     :							Optional[Any]			)->  Tuple:





     """simple docstring"""







     if index in self.fairseq_ids_to_tokens:
       return self.fairseq_ids_to_tokens[index]
     return self.sp_model.IdToPiece(index - self.fairseq_offset			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							int     ,	a     :							List[str]			)->  int:





     """simple docstring"""







     lowercase__           =    "".join(_SCREAMING_SNAKE_CASE			).replace(_SCREAMING_SNAKE_CASE     ,	' '			).strip()
     return out_string


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Tuple     ,	a     :							str     ,	a     :							Optional[str] = None			)->  Tuple[str]:





     """simple docstring"""







     if not os.path.isdir(_SCREAMING_SNAKE_CASE			):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
       return
     lowercase__           =    os.path.join(
         _SCREAMING_SNAKE_CASE     ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

     if os.path.abspath(self.vocab_file			) != os.path.abspath(_SCREAMING_SNAKE_CASE			) and os.path.isfile(self.vocab_file			):
       copyfile(self.vocab_file     ,	_SCREAMING_SNAKE_CASE			)
     elif not os.path.isfile(self.vocab_file			):
       with open(_SCREAMING_SNAKE_CASE     ,	'wb'			) as fi:
         lowercase__           =    self.sp_model.serialized_model_proto()
         fi.write(_SCREAMING_SNAKE_CASE			)

     return (out_vocab_file,)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Optional[Any]     ,	a     :							List[str]     ,	a     :							str = "eng_Latn"     ,	a     :							Optional[List[str]] = None     ,	a     :							str = "fra_Latn"     ,	**a     :							Optional[Any]     ,	)->  BatchEncoding:





     """simple docstring"""







     lowercase__           =    src_lang
     lowercase__           =    tgt_lang
     return super().prepare_seqaseq_batch(_SCREAMING_SNAKE_CASE     ,	_SCREAMING_SNAKE_CASE     ,	**_SCREAMING_SNAKE_CASE			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							List[str]			)->  Any:





     """simple docstring"""







     return self.set_src_lang_special_tokens(self.src_lang			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Tuple			)->  int:





     """simple docstring"""







     return self.set_tgt_lang_special_tokens(self.tgt_lang			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Optional[Any]     ,	a     :							Optional[Any]			)->  None:





     """simple docstring"""







     lowercase__           =    self.lang_code_to_id[src_lang]
     if self.legacy_behaviour:
       lowercase__           =    []
       lowercase__           =    [self.eos_token_id, self.cur_lang_code]
     else:
       lowercase__           =    [self.cur_lang_code]
       lowercase__           =    [self.eos_token_id]


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Union[str, Any]     ,	a     :							str			)->  None:





     """simple docstring"""







     lowercase__           =    self.lang_code_to_id[lang]
     if self.legacy_behaviour:
       lowercase__           =    []
       lowercase__           =    [self.eos_token_id, self.cur_lang_code]
     else:
       lowercase__           =    [self.cur_lang_code]
       lowercase__           =    [self.eos_token_id]







"""simple docstring"""






def      _lowercase							(    __snake_case   ,__snake_case      )		->  int:
			return int((input_a, input_a).count(1      ) != 0      )


def      _lowercase							(    )		->  None:
			assert or_gate(0   ,0      ) == 0
			assert or_gate(0   ,1      ) == 1
			assert or_gate(1   ,0      ) == 1
			assert or_gate(1   ,1      ) == 1


if __name__ == "__main__":
	print(or_gate(0, 1))
	print(or_gate(1, 0))
	print(or_gate(0, 0))
	print(or_gate(1, 1))