




'''simple docstring'''







import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase      :   List[str]      =       logging.get_logger(__name__)

lowercase      :   Optional[Any]      =       """▁"""

lowercase      :   Optional[int]      =       {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

lowercase      :   Dict      =       {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

lowercase      :   Union[str, Any]      =       {
    """facebook/m2m100_418M""": 1024,
}

# fmt: off
lowercase      :   Optional[int]      =       {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     int          =					VOCAB_FILES_NAMES
     lowerCAmelCase_  :     Any          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_  :     Optional[Any]          =					PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_  :     Tuple          =					["""input_ids""", """attention_mask"""]

     lowerCAmelCase_  :     List[int]          =					[]
     lowerCAmelCase_  :     List[int]          =					[]







     def __init__(				self							,__a							,__a							,__a=None							,__a=None							,__a="<s>"							,__a="</s>"							,__a="</s>"							,__a="<pad>"							,__a="<unk>"							,__a="m2m100"							,__a = None							,__a=8							,**__a							,)      ->				None:
          snake_case			:      Tuple        =			{} if sp_model_kwargs is None else sp_model_kwargs

          snake_case			:      Dict        =			language_codes
          snake_case			:      Tuple        =			FAIRSEQ_LANGUAGE_CODES[language_codes]
          snake_case			:      Tuple        =			{lang_code: F'''__{lang_code}__''' for lang_code in fairseq_language_code}

          snake_case			:      List[Any]        =			kwargs.get("""additional_special_tokens"""							,[]							)
          kwargs["additional_special_tokens"] += [
              self.get_lang_token(__a							)
              for lang_code in fairseq_language_code
              if self.get_lang_token(__a							) not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              src_lang=__a							,tgt_lang=__a							,bos_token=__a							,eos_token=__a							,sep_token=__a							,unk_token=__a							,pad_token=__a							,language_codes=__a							,sp_model_kwargs=self.sp_model_kwargs							,num_madeup_words=__a							,**__a							,)

          snake_case			:      Tuple        =			vocab_file
          snake_case			:      Any        =			load_json(__a							)
          snake_case			:      Dict        =			{v: k for k, v in self.encoder.items()}
          snake_case			:      Optional[Any]        =			spm_file
          snake_case			:      int        =			load_spm(__a							,self.sp_model_kwargs							)

          snake_case			:      str        =			len(self.encoder							)

          snake_case			:      Any        =			{
              self.get_lang_token(__a							): self.encoder_size + i for i, lang_code in enumerate(__a							)
          }
          snake_case			:      Dict        =			{lang_code: self.encoder_size + i for i, lang_code in enumerate(__a							)}
          snake_case			:      Any        =			{v: k for k, v in self.lang_token_to_id.items()}

          snake_case			:      Union[str, Any]        =			src_lang if src_lang is not None else """en"""
          snake_case			:      int        =			tgt_lang
          snake_case			:      Union[str, Any]        =			self.get_lang_id(self._src_lang							)
          self.set_src_lang_special_tokens(self._src_lang							)

          snake_case			:      List[str]        =			num_madeup_words







     @property
     def 			snake_case_      (				self							)      ->				int:
          return len(self.encoder							) + len(self.lang_token_to_id							)







     @property
     def 			snake_case_      (				self							)      ->				str:
          return self._src_lang







     @src_lang.setter
     def 			snake_case_      (				self							,__a							)      ->				None:
          snake_case			:      Any        =			new_src_lang
          self.set_src_lang_special_tokens(self._src_lang							)







     def 			snake_case_      (				self							,__a							)      ->				List[str]:
          return self.sp_model.encode(__a							,out_type=__a							)







     def 			snake_case_      (				self							,__a							)      ->				Optional[Any]:
          if token in self.lang_token_to_id:
               return self.lang_token_to_id[token]
          return self.encoder.get(__a							,self.encoder[self.unk_token]							)







     def 			snake_case_      (				self							,__a							)      ->				str:
          if index in self.id_to_lang_token:
               return self.id_to_lang_token[index]
          return self.decoder.get(__a							,self.unk_token							)







     def 			snake_case_      (				self							,__a							)      ->				List[str]:
          snake_case			:      List[str]        =			[]
          snake_case			:      List[str]        =			""""""
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    out_string += self.sp_model.decode(__a							) + token
                    snake_case			:      int        =			[]
               else:
                    current_sub_tokens.append(__a							)
          out_string += self.sp_model.decode(__a							)
          return out_string.strip()







     def 			snake_case_      (				self							,__a							,__a = None							,__a = False							)      ->				List[int]:

          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__a							,token_ids_a=__a							,already_has_special_tokens=__a							)

          snake_case			:      Optional[Any]        =			[1] * len(self.prefix_tokens							)
          snake_case			:      Any        =			[1] * len(self.suffix_tokens							)
          if token_ids_a is None:
               return prefix_ones + ([0] * len(__a							)) + suffix_ones
          return prefix_ones + ([0] * len(__a							)) + ([0] * len(__a							)) + suffix_ones







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				List[int]:
          if token_ids_a is None:
               return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







     def 			snake_case_      (				self							)      ->				Dict:
          snake_case			:      Optional[Any]        =			{self.convert_ids_to_tokens(__a							): i for i in range(self.vocab_size							)}
          vocab.update(self.added_tokens_encoder							)
          return vocab







     def __getstate__(				self							)      ->				Dict:
          snake_case			:      Any        =			self.__dict__.copy()
          snake_case			:      Optional[Any]        =			None
          return state







     def __setstate__(				self							,__a							)      ->				None:
          snake_case			:      int        =			d

          # for backward compatibility
          if not hasattr(self							,"""sp_model_kwargs"""							):
               snake_case			:      str        =			{}

          snake_case			:      List[Any]        =			load_spm(self.spm_file							,self.sp_model_kwargs							)







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				Tuple[str]:
          snake_case			:      Union[str, Any]        =			Path(__a							)
          if not save_dir.is_dir():
               raise OSError(F'''{save_directory} should be a directory'''							)
          snake_case			:      List[str]        =			save_dir / (
              (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
          )
          snake_case			:      List[Any]        =			save_dir / (
              (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
          )

          save_json(self.encoder							,__a							)

          if os.path.abspath(self.spm_file							) != os.path.abspath(__a							) and os.path.isfile(self.spm_file							):
               copyfile(self.spm_file							,__a							)
          elif not os.path.isfile(self.spm_file							):
               with open(__a							,"""wb"""							) as fi:
                    snake_case			:      str        =			self.sp_model.serialized_model_proto()
                    fi.write(__a							)

          return (str(__a							), str(__a							))







     def 			snake_case_      (				self							,__a							,__a = "en"							,__a = None							,__a = "ro"							,**__a							,)      ->				BatchEncoding:
          snake_case			:      str        =			src_lang
          snake_case			:      Dict        =			tgt_lang
          self.set_src_lang_special_tokens(self.src_lang							)
          return super().prepare_seqaseq_batch(__a							,__a							,**__a							)







     def 			snake_case_      (				self							,__a							,__a							,__a							,**__a							)      ->				Optional[Any]:
          if src_lang is None or tgt_lang is None:
               raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""							)
          snake_case			:      Tuple        =			src_lang
          snake_case			:      Optional[int]        =			self(__a							,add_special_tokens=__a							,**__a							)
          snake_case			:      Any        =			self.get_lang_id(__a							)
          snake_case			:      str        =			tgt_lang_id
          return inputs







     def 			snake_case_      (				self							)      ->				str:
          self.set_src_lang_special_tokens(self.src_lang							)







     def 			snake_case_      (				self							)      ->				Union[str, Any]:
          self.set_tgt_lang_special_tokens(self.tgt_lang							)







     def 			snake_case_      (				self							,__a							)      ->				None:
          snake_case			:      Any        =			self.get_lang_token(__a							)
          snake_case			:      Dict        =			self.lang_token_to_id[lang_token]
          snake_case			:      List[Any]        =			[self.cur_lang_id]
          snake_case			:      Optional[Any]        =			[self.eos_token_id]







     def 			snake_case_      (				self							,__a							)      ->				None:
          snake_case			:      Dict        =			self.get_lang_token(__a							)
          snake_case			:      Optional[int]        =			self.lang_token_to_id[lang_token]
          snake_case			:      List[Any]        =			[self.cur_lang_id]
          snake_case			:      Optional[Any]        =			[self.eos_token_id]







     def 			snake_case_      (				self							,__a							)      ->				str:
          return self.lang_code_to_token[lang]







     def 			snake_case_      (				self							,__a							)      ->				int:
          snake_case			:      List[str]        =			self.get_lang_token(__a							)
          return self.lang_token_to_id[lang_token]



def        lowerCamelCase__				(      __lowercase     ,   __lowercase				):
     snake_case			:      Any        =			sentencepiece.SentencePieceProcessor(**__lowercase				)
     spm.Load(str(__lowercase				)				)
     return spm



def        lowerCamelCase__				(      __lowercase				):
     with open(__lowercase     ,   """r"""				) as f:
          return json.load(__lowercase				)







def        lowerCamelCase__				(      __lowercase     ,   __lowercase				):
     with open(__lowercase     ,   """w"""				) as f:
          json.dump(__lowercase     ,   __lowercase     ,   indent=2				)









'''simple docstring'''







from ..utils import DummyObject, requires_backends







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Any          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Tuple:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Tuple:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Dict:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[int]          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				List[Any]:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				List[str]:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				str:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Any          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Any:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				List[Any]:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Any:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     List[str]          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Optional[Any]:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Dict:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				int:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     int          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Optional[int]:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Optional[Any]:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				List[str]:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Dict          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Optional[Any]:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				List[Any]:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				int:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Union[str, Any]          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				List[str]:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Optional[Any]:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				str:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[Any]          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				str:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				str:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Union[str, Any]:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     List[Any]          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Union[str, Any]:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Any:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Optional[Any]:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Dict          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Dict:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				List[str]:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Optional[Any]:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     List[str]          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Tuple:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Tuple:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Optional[int]:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Tuple          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				List[str]:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Union[str, Any]:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				Dict:
          requires_backends(cls							,["""flax"""]							)







class 					_a (metaclass=a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Tuple          =					["""flax"""]







     def __init__(				self							,*__a							,**__a							)      ->				Tuple:
          requires_backends(self							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				List[str]:
          requires_backends(cls							,["""flax"""]							)







     @classmethod
     def 			snake_case_      (				cls							,*__a							,**__a							)      ->				int:
          requires_backends(cls							,["""flax"""]							)



