






"""simple docstring"""


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE   =   logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE   =   '▁'

__SCREAMING_SNAKE_CASE   =   {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

__SCREAMING_SNAKE_CASE   =   {
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

__SCREAMING_SNAKE_CASE   =   {
    'facebook/m2m100_418M': 10_24,
}

# fmt: off
__SCREAMING_SNAKE_CASE   =   {
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}




class 						__UpperCamelCase     (     UpperCamelCase						):
					lowercase_    :				Optional[Any]      =  VOCAB_FILES_NAMES
					lowercase_    :				Tuple      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase_    :				Optional[Any]      =  PRETRAINED_VOCAB_FILES_MAP
					lowercase_    :				List[Any]      =  ["""input_ids""", """attention_mask"""]

					lowercase_    :				List[int]      =  []
					lowercase_    :				List[int]      =  []
					def __init__(							self							:	str     , UpperCAmelCase							:	Optional[Any]     , UpperCAmelCase							:	Optional[Any]     , UpperCAmelCase							:	Optional[Any]=None     , UpperCAmelCase							:	List[str]=None     , UpperCAmelCase							:	Union[str, Any]="<s>"     , UpperCAmelCase							:	Optional[int]="</s>"     , UpperCAmelCase							:	Optional[Any]="</s>"     , UpperCAmelCase							:	Tuple="<pad>"     , UpperCAmelCase							:	List[str]="<unk>"     , UpperCAmelCase							:	str="m2m100"     , UpperCAmelCase							:	Optional[Dict[str, Any]] = None     , UpperCAmelCase							:	Any=8     , **UpperCAmelCase							:	Dict     , )	->       None:
											lowerCAmelCase   :List[Any]						=			{} if sp_model_kwargs is None else sp_model_kwargs

											lowerCAmelCase   :List[str]						=			language_codes
											lowerCAmelCase   :int						=			FAIRSEQ_LANGUAGE_CODES[language_codes]
											lowerCAmelCase   :Tuple						=			{lang_code: f"""__{lang_code}__""" for lang_code in fairseq_language_code}

											lowerCAmelCase   :Tuple						=			kwargs.get('additional_special_tokens'     , []   )
											kwargs["additional_special_tokens"] += [
											    self.get_lang_token(UpperCAmelCase   )
											    for lang_code in fairseq_language_code
											    if self.get_lang_token(UpperCAmelCase   ) not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    src_lang=UpperCAmelCase     , tgt_lang=UpperCAmelCase     , bos_token=UpperCAmelCase     , eos_token=UpperCAmelCase     , sep_token=UpperCAmelCase     , unk_token=UpperCAmelCase     , pad_token=UpperCAmelCase     , language_codes=UpperCAmelCase     , sp_model_kwargs=self.sp_model_kwargs     , num_madeup_words=UpperCAmelCase     , **UpperCAmelCase     , )

											lowerCAmelCase   :Optional[Any]						=			vocab_file
											lowerCAmelCase   :Union[str, Any]						=			load_json(UpperCAmelCase   )
											lowerCAmelCase   :str						=			{v: k for k, v in self.encoder.items()}
											lowerCAmelCase   :List[str]						=			spm_file
											lowerCAmelCase   :Optional[int]						=			load_spm(UpperCAmelCase     , self.sp_model_kwargs   )

											lowerCAmelCase   :Optional[Any]						=			len(self.encoder   )

											lowerCAmelCase   :Optional[int]						=			{
											    self.get_lang_token(UpperCAmelCase   ): self.encoder_size + i for i, lang_code in enumerate(UpperCAmelCase   )
											}
											lowerCAmelCase   :Optional[int]						=			{lang_code: self.encoder_size + i for i, lang_code in enumerate(UpperCAmelCase   )}
											lowerCAmelCase   :List[str]						=			{v: k for k, v in self.lang_token_to_id.items()}

											lowerCAmelCase   :List[Any]						=			src_lang if src_lang is not None else 'en'
											lowerCAmelCase   :Any						=			tgt_lang
											lowerCAmelCase   :Optional[int]						=			self.get_lang_id(self._src_lang   )
											self.set_src_lang_special_tokens(self._src_lang   )

											lowerCAmelCase   :str						=			num_madeup_words
					@property
					def        UpperCAmelCase__ (							self							:	Optional[Any]   )	->       int:
											return len(self.encoder   ) + len(self.lang_token_to_id   )
					@property
					def        UpperCAmelCase__ (							self							:	Optional[Any]   )	->       str:
											return self._src_lang
					@src_lang.setter
					def        UpperCAmelCase__ (							self							:	Any     , UpperCAmelCase							:	str   )	->       None:
											lowerCAmelCase   :Optional[Any]						=			new_src_lang
											self.set_src_lang_special_tokens(self._src_lang   )
					def        UpperCAmelCase__ (							self							:	Union[str, Any]     , UpperCAmelCase							:	str   )	->       List[str]:
											return self.sp_model.encode(UpperCAmelCase     , out_type=UpperCAmelCase   )
					def        UpperCAmelCase__ (							self							:	List[Any]     , UpperCAmelCase							:	Dict   )	->       Dict:
											if token in self.lang_token_to_id:
																	return self.lang_token_to_id[token]
											return self.encoder.get(UpperCAmelCase     , self.encoder[self.unk_token]   )
					def        UpperCAmelCase__ (							self							:	Optional[Any]     , UpperCAmelCase							:	int   )	->       str:
											if index in self.id_to_lang_token:
																	return self.id_to_lang_token[index]
											return self.decoder.get(UpperCAmelCase     , self.unk_token   )
					def        UpperCAmelCase__ (							self							:	str     , UpperCAmelCase							:	Tuple   )	->       Dict:
											lowerCAmelCase   :Optional[int]						=			[]
											lowerCAmelCase   :Optional[Any]						=			''
											for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																							out_string += self.sp_model.decode(UpperCAmelCase   ) + token
																							lowerCAmelCase   :Any						=			[]
																	else:
																							current_sub_tokens.append(UpperCAmelCase   )
											out_string += self.sp_model.decode(UpperCAmelCase   )
											return out_string.strip()
					def        UpperCAmelCase__ (							self							:	Tuple     , UpperCAmelCase							:	List[int]     , UpperCAmelCase							:	Optional[List[int]] = None     , UpperCAmelCase							:	bool = False   )	->       List[int]:

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=UpperCAmelCase     , token_ids_a=UpperCAmelCase     , already_has_special_tokens=UpperCAmelCase   )

											lowerCAmelCase   :List[Any]						=			[1] * len(self.prefix_tokens   )
											lowerCAmelCase   :Dict						=			[1] * len(self.suffix_tokens   )
											if token_ids_a is None:
																	return prefix_ones + ([0] * len(UpperCAmelCase   )) + suffix_ones
											return prefix_ones + ([0] * len(UpperCAmelCase   )) + ([0] * len(UpperCAmelCase   )) + suffix_ones
					def        UpperCAmelCase__ (							self							:	List[Any]     , UpperCAmelCase							:	List[int]     , UpperCAmelCase							:	Optional[List[int]] = None   )	->       List[int]:
											if token_ids_a is None:
																	return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
					def        UpperCAmelCase__ (							self							:	Optional[Any]   )	->       Dict:
											lowerCAmelCase   :Any						=			{self.convert_ids_to_tokens(UpperCAmelCase   ): i for i in range(self.vocab_size   )}
											vocab.update(self.added_tokens_encoder   )
											return vocab
					def __getstate__(							self							:	Optional[int]   )	->       Dict:
											lowerCAmelCase   :Optional[Any]						=			self.__dict__.copy()
											lowerCAmelCase   :Dict						=			None
											return state
					def __setstate__(							self							:	Union[str, Any]     , UpperCAmelCase							:	Dict   )	->       None:
											lowerCAmelCase   :Any						=			d

											# for backward compatibility
											if not hasattr(self     , 'sp_model_kwargs'   ):
																	lowerCAmelCase   :Any						=			{}

											lowerCAmelCase   :Union[str, Any]						=			load_spm(self.spm_file     , self.sp_model_kwargs   )
					def        UpperCAmelCase__ (							self							:	Optional[int]     , UpperCAmelCase							:	str     , UpperCAmelCase							:	Optional[str] = None   )	->       Tuple[str]:
											lowerCAmelCase   :str						=			Path(UpperCAmelCase   )
											if not save_dir.is_dir():
																	raise OSError(f"""{save_directory} should be a directory"""   )
											lowerCAmelCase   :str						=			save_dir / (
											    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
											)
											lowerCAmelCase   :List[str]						=			save_dir / (
											    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
											)

											save_json(self.encoder     , UpperCAmelCase   )

											if os.path.abspath(self.spm_file   ) != os.path.abspath(UpperCAmelCase   ) and os.path.isfile(self.spm_file   ):
																	copyfile(self.spm_file     , UpperCAmelCase   )
											elif not os.path.isfile(self.spm_file   ):
																	with open(UpperCAmelCase     , 'wb'   ) as fi:
																							lowerCAmelCase   :List[str]						=			self.sp_model.serialized_model_proto()
																							fi.write(UpperCAmelCase   )

											return (str(UpperCAmelCase   ), str(UpperCAmelCase   ))
					def        UpperCAmelCase__ (							self							:	Tuple     , UpperCAmelCase							:	List[str]     , UpperCAmelCase							:	str = "en"     , UpperCAmelCase							:	Optional[List[str]] = None     , UpperCAmelCase							:	str = "ro"     , **UpperCAmelCase							:	Dict     , )	->       BatchEncoding:
											lowerCAmelCase   :Any						=			src_lang
											lowerCAmelCase   :Any						=			tgt_lang
											self.set_src_lang_special_tokens(self.src_lang   )
											return super().prepare_seqaseq_batch(UpperCAmelCase     , UpperCAmelCase     , **UpperCAmelCase   )
					def        UpperCAmelCase__ (							self							:	Any     , UpperCAmelCase							:	Tuple     , UpperCAmelCase							:	Optional[str]     , UpperCAmelCase							:	Optional[str]     , **UpperCAmelCase							:	Any   )	->       Optional[int]:
											if src_lang is None or tgt_lang is None:
																	raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'   )
											lowerCAmelCase   :List[str]						=			src_lang
											lowerCAmelCase   :Union[str, Any]						=			self(UpperCAmelCase     , add_special_tokens=UpperCAmelCase     , **UpperCAmelCase   )
											lowerCAmelCase   :Any						=			self.get_lang_id(UpperCAmelCase   )
											lowerCAmelCase   :Optional[int]						=			tgt_lang_id
											return inputs
					def        UpperCAmelCase__ (							self							:	Optional[int]   )	->       Optional[int]:
											self.set_src_lang_special_tokens(self.src_lang   )
					def        UpperCAmelCase__ (							self							:	str   )	->       Optional[int]:
											self.set_tgt_lang_special_tokens(self.tgt_lang   )
					def        UpperCAmelCase__ (							self							:	List[Any]     , UpperCAmelCase							:	str   )	->       None:
											lowerCAmelCase   :str						=			self.get_lang_token(UpperCAmelCase   )
											lowerCAmelCase   :int						=			self.lang_token_to_id[lang_token]
											lowerCAmelCase   :int						=			[self.cur_lang_id]
											lowerCAmelCase   :Dict						=			[self.eos_token_id]
					def        UpperCAmelCase__ (							self							:	Tuple     , UpperCAmelCase							:	str   )	->       None:
											lowerCAmelCase   :Optional[int]						=			self.get_lang_token(UpperCAmelCase   )
											lowerCAmelCase   :List[Any]						=			self.lang_token_to_id[lang_token]
											lowerCAmelCase   :Optional[int]						=			[self.cur_lang_id]
											lowerCAmelCase   :Dict						=			[self.eos_token_id]
					def        UpperCAmelCase__ (							self							:	Any     , UpperCAmelCase							:	str   )	->       str:
											return self.lang_code_to_token[lang]
					def        UpperCAmelCase__ (							self							:	str     , UpperCAmelCase							:	str   )	->       int:
											lowerCAmelCase   :List[Any]						=			self.get_lang_token(UpperCAmelCase   )
											return self.lang_token_to_id[lang_token]


def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :int						=			sentencepiece.SentencePieceProcessor(**a__ )
						spm.Load(str(a__ ) )
						return spm


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						with open(a__     ,      'r' ) as f:
												return json.load(a__ )







def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						with open(a__     ,      'w' ) as f:
												json.dump(a__     ,      a__     ,      indent=2 )







"""simple docstring"""


from math import loga


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						if a < 0:
												raise ValueError('Input value must be a positive integer' )
						elif isinstance(a__     ,      a__ ):
												raise TypeError('Input value must be a \'int\' type' )
						return 0 if (a == 0) else int(loga(a & -a ) )


if __name__ == "__main__":
		import doctest

		doctest.testmod()