



"""simple docstring"""
import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_				= logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_				= """▁"""

SCREAMING_SNAKE_CASE_				= {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

SCREAMING_SNAKE_CASE_				= {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

SCREAMING_SNAKE_CASE_				= {
    """facebook/m2m100_418M""": 10_24,
}

# fmt: off
SCREAMING_SNAKE_CASE_				= {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}




class 			snake_case_						(						a_				):
							__lowerCAmelCase    					=					VOCAB_FILES_NAMES
							__lowerCAmelCase    					=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowerCAmelCase    					=					PRETRAINED_VOCAB_FILES_MAP
							__lowerCAmelCase    					=					["input_ids", "attention_mask"]

							__lowerCAmelCase    					=					[]
							__lowerCAmelCase    					=					[]






							def __init__(   self			,     a_			,     a_			,     a_=None			,     a_=None			,     a_="<s>"			,     a_="</s>"			,     a_="</s>"			,     a_="<pad>"			,     a_="<unk>"			,     a_="m2m100"			,     a_ = None			,     a_=8			,     **a_			,     ):
									a_     :      Optional[Any] 						=     {} if sp_model_kwargs is None else sp_model_kwargs

									a_     :      List[Any] 						=     language_codes
									a_     :      Dict 						=     FAIRSEQ_LANGUAGE_CODES[language_codes]
									a_     :      Dict 						=     {lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

									a_     :      Tuple 						=     kwargs.get("additional_special_tokens"			,     []      )
									kwargs["additional_special_tokens"] += [
									    self.get_lang_token(a_      )
									    for lang_code in fairseq_language_code
									    if self.get_lang_token(a_      ) not in kwargs["additional_special_tokens"]
									]

									super().__init__(
									    src_lang=a_			,     tgt_lang=a_			,     bos_token=a_			,     eos_token=a_			,     sep_token=a_			,     unk_token=a_			,     pad_token=a_			,     language_codes=a_			,     sp_model_kwargs=self.sp_model_kwargs			,     num_madeup_words=a_			,     **a_			,     )

									a_     :      Dict 						=     vocab_file
									a_     :      List[str] 						=     load_json(a_      )
									a_     :      Any 						=     {v: k for k, v in self.encoder.items()}
									a_     :      Dict 						=     spm_file
									a_     :      Dict 						=     load_spm(a_			,     self.sp_model_kwargs      )

									a_     :      Optional[Any] 						=     len(self.encoder      )

									a_     :      Optional[int] 						=     {
									    self.get_lang_token(a_      ): self.encoder_size + i for i, lang_code in enumerate(a_      )
									}
									a_     :      int 						=     {lang_code: self.encoder_size + i for i, lang_code in enumerate(a_      )}
									a_     :      List[Any] 						=     {v: k for k, v in self.lang_token_to_id.items()}

									a_     :      List[str] 						=     src_lang if src_lang is not None else "en"
									a_     :      Tuple 						=     tgt_lang
									a_     :      Any 						=     self.get_lang_id(self._src_lang      )
									self.set_src_lang_special_tokens(self._src_lang      )

									a_     :      str 						=     num_madeup_words






							@property
							def snake_case_  (   self      ):
									return len(self.encoder      ) + len(self.lang_token_to_id      )






							@property
							def snake_case_  (   self      ):
									return self._src_lang






							@src_lang.setter
							def snake_case_  (   self			,     a_      ):
									a_     :      Optional[int] 						=     new_src_lang
									self.set_src_lang_special_tokens(self._src_lang      )






							def snake_case_  (   self			,     a_      ):
									return self.sp_model.encode(a_			,     out_type=a_      )






							def snake_case_  (   self			,     a_      ):
									if token in self.lang_token_to_id:
											return self.lang_token_to_id[token]
									return self.encoder.get(a_			,     self.encoder[self.unk_token]      )






							def snake_case_  (   self			,     a_      ):
									if index in self.id_to_lang_token:
											return self.id_to_lang_token[index]
									return self.decoder.get(a_			,     self.unk_token      )






							def snake_case_  (   self			,     a_      ):
									a_     :      int 						=     []
									a_     :      List[str] 						=     ""
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													out_string += self.sp_model.decode(a_      ) + token
													a_     :      int 						=     []
											else:
													current_sub_tokens.append(a_      )
									out_string += self.sp_model.decode(a_      )
									return out_string.strip()






							def snake_case_  (   self			,     a_			,     a_ = None			,     a_ = False      ):

									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=a_			,     token_ids_a=a_			,     already_has_special_tokens=a_      )

									a_     :      Optional[Any] 						=     [1] * len(self.prefix_tokens      )
									a_     :      Any 						=     [1] * len(self.suffix_tokens      )
									if token_ids_a is None:
											return prefix_ones + ([0] * len(a_      )) + suffix_ones
									return prefix_ones + ([0] * len(a_      )) + ([0] * len(a_      )) + suffix_ones






							def snake_case_  (   self			,     a_			,     a_ = None      ):
									if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     {self.convert_ids_to_tokens(a_      ): i for i in range(self.vocab_size      )}
									vocab.update(self.added_tokens_encoder      )
									return vocab






							def __getstate__(   self      ):
									a_     :      Any 						=     self.__dict__.copy()
									a_     :      Union[str, Any] 						=     None
									return state






							def __setstate__(   self			,     a_      ):
									a_     :      Tuple 						=     d

									# for backward compatibility
									if not hasattr(self			,     "sp_model_kwargs"      ):
											a_     :      Union[str, Any] 						=     {}

									a_     :      Union[str, Any] 						=     load_spm(self.spm_file			,     self.sp_model_kwargs      )






							def snake_case_  (   self			,     a_			,     a_ = None      ):
									a_     :      Optional[Any] 						=     Path(a_      )
									if not save_dir.is_dir():
											raise OSError(F"""{save_directory} should be a directory"""      )
									a_     :      str 						=     save_dir / (
									    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
									)
									a_     :      str 						=     save_dir / (
									    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
									)

									save_json(self.encoder			,     a_      )

									if os.path.abspath(self.spm_file      ) != os.path.abspath(a_      ) and os.path.isfile(self.spm_file      ):
											copyfile(self.spm_file			,     a_      )
									elif not os.path.isfile(self.spm_file      ):
											with open(a_			,     "wb"      ) as fi:
													a_     :      List[str] 						=     self.sp_model.serialized_model_proto()
													fi.write(a_      )

									return (str(a_      ), str(a_      ))






							def snake_case_  (   self			,     a_			,     a_ = "en"			,     a_ = None			,     a_ = "ro"			,     **a_			,     ):
									a_     :      Union[str, Any] 						=     src_lang
									a_     :      List[Any] 						=     tgt_lang
									self.set_src_lang_special_tokens(self.src_lang      )
									return super().prepare_seqaseq_batch(a_			,     a_			,     **a_      )






							def snake_case_  (   self			,     a_			,     a_			,     a_			,     **a_      ):
									if src_lang is None or tgt_lang is None:
											raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"      )
									a_     :      Optional[Any] 						=     src_lang
									a_     :      List[Any] 						=     self(a_			,     add_special_tokens=a_			,     **a_      )
									a_     :      Optional[int] 						=     self.get_lang_id(a_      )
									a_     :      Any 						=     tgt_lang_id
									return inputs






							def snake_case_  (   self      ):
									self.set_src_lang_special_tokens(self.src_lang      )






							def snake_case_  (   self      ):
									self.set_tgt_lang_special_tokens(self.tgt_lang      )






							def snake_case_  (   self			,     a_      ):
									a_     :      Union[str, Any] 						=     self.get_lang_token(a_      )
									a_     :      List[Any] 						=     self.lang_token_to_id[lang_token]
									a_     :      List[Any] 						=     [self.cur_lang_id]
									a_     :      Dict 						=     [self.eos_token_id]






							def snake_case_  (   self			,     a_      ):
									a_     :      Any 						=     self.get_lang_token(a_      )
									a_     :      str 						=     self.lang_token_to_id[lang_token]
									a_     :      Dict 						=     [self.cur_lang_id]
									a_     :      Tuple 						=     [self.eos_token_id]






							def snake_case_  (   self			,     a_      ):
									return self.lang_code_to_token[lang]






							def snake_case_  (   self			,     a_      ):
									a_     :      Any 						=     self.get_lang_token(a_      )
									return self.lang_token_to_id[lang_token]



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      sentencepiece.SentencePieceProcessor:
		a_     :      Any 						=     sentencepiece.SentencePieceProcessor(**SCREAMING_SNAKE_CASE__	)
		spm.Load(str(SCREAMING_SNAKE_CASE__	)	)
		return spm



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__	)			->      Union[Dict, List]:
		with open(SCREAMING_SNAKE_CASE__, "r"	) as f:
				return json.load(SCREAMING_SNAKE_CASE__	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      None:
		with open(SCREAMING_SNAKE_CASE__, "w"	) as f:
				json.dump(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, indent=2	)




"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


SCREAMING_SNAKE_CASE_				= {
    """configuration_bigbird_pegasus""": [
        """BIGBIRD_PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """BigBirdPegasusConfig""",
        """BigBirdPegasusOnnxConfig""",
    ],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					SCREAMING_SNAKE_CASE_				= [
					    """BIGBIRD_PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """BigBirdPegasusForCausalLM""",
					    """BigBirdPegasusForConditionalGeneration""",
					    """BigBirdPegasusForQuestionAnswering""",
					    """BigBirdPegasusForSequenceClassification""",
					    """BigBirdPegasusModel""",
					    """BigBirdPegasusPreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_bigbird_pegasus import (
					    BIGBIRD_PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    BigBirdPegasusConfig,
					    BigBirdPegasusOnnxConfig,
					)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_bigbird_pegasus import (
										    BIGBIRD_PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST,
										    BigBirdPegasusForCausalLM,
										    BigBirdPegasusForConditionalGeneration,
										    BigBirdPegasusForQuestionAnswering,
										    BigBirdPegasusForSequenceClassification,
										    BigBirdPegasusModel,
										    BigBirdPegasusPreTrainedModel,
										)


else:
					import sys

					SCREAMING_SNAKE_CASE_				= _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)