

"""simple docstring"""
import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_A							=  logging.get_logger(__name__)

_A							=  """▁"""

_A							=  {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

_A							=  {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

_A							=  {
    """facebook/m2m100_418M""": 10_24,
}

# fmt: off
_A							=  {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}

class       lowerCamelCase	(       lowerCAmelCase__   ):


  '''simple docstring'''







  SCREAMING_SNAKE_CASE					      =  VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE					      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE					      =  PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE					      =  ['input_ids', 'attention_mask']

  SCREAMING_SNAKE_CASE					      =  []
  SCREAMING_SNAKE_CASE					      =  []



  def __init__(self							,       _lowerCamelCase							,       _lowerCamelCase							,       _lowerCamelCase=None							,       _lowerCamelCase=None							,       _lowerCamelCase="<s>"							,       _lowerCamelCase="</s>"							,       _lowerCamelCase="</s>"							,       _lowerCamelCase="<pad>"							,       _lowerCamelCase="<unk>"							,       _lowerCamelCase="m2m100"							,       _lowerCamelCase = None							,       _lowerCamelCase=8							,       **_lowerCamelCase							,       ):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    {} if sp_model_kwargs is None else sp_model_kwargs

        UpperCAmelCase__			:							List[str]							=    language_codes
        UpperCAmelCase__			:							Optional[int]							=    FAIRSEQ_LANGUAGE_CODES[language_codes]
        UpperCAmelCase__			:							Tuple							=    {lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

        UpperCAmelCase__			:							str							=    kwargs.get("""additional_special_tokens"""							,       []	)
        kwargs["additional_special_tokens"] += [
            self.get_lang_token(_lowerCamelCase	)
            for lang_code in fairseq_language_code
            if self.get_lang_token(_lowerCamelCase	) not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            src_lang=_lowerCamelCase							,       tgt_lang=_lowerCamelCase							,       bos_token=_lowerCamelCase							,       eos_token=_lowerCamelCase							,       sep_token=_lowerCamelCase							,       unk_token=_lowerCamelCase							,       pad_token=_lowerCamelCase							,       language_codes=_lowerCamelCase							,       sp_model_kwargs=self.sp_model_kwargs							,       num_madeup_words=_lowerCamelCase							,       **_lowerCamelCase							,       )

        UpperCAmelCase__			:							Dict							=    vocab_file
        UpperCAmelCase__			:							str							=    load_json(_lowerCamelCase	)
        UpperCAmelCase__			:							Optional[int]							=    {v: k for k, v in self.encoder.items()}
        UpperCAmelCase__			:							List[str]							=    spm_file
        UpperCAmelCase__			:							Any							=    load_spm(_lowerCamelCase							,       self.sp_model_kwargs	)

        UpperCAmelCase__			:							Tuple							=    len(self.encoder	)

        UpperCAmelCase__			:							int							=    {
            self.get_lang_token(_lowerCamelCase	): self.encoder_size + i for i, lang_code in enumerate(_lowerCamelCase	)
        }
        UpperCAmelCase__			:							str							=    {lang_code: self.encoder_size + i for i, lang_code in enumerate(_lowerCamelCase	)}
        UpperCAmelCase__			:							Dict							=    {v: k for k, v in self.lang_token_to_id.items()}

        UpperCAmelCase__			:							str							=    src_lang if src_lang is not None else """en"""
        UpperCAmelCase__			:							Any							=    tgt_lang
        UpperCAmelCase__			:							Union[str, Any]							=    self.get_lang_id(self._src_lang	)
        self.set_src_lang_special_tokens(self._src_lang	)

        UpperCAmelCase__			:							Optional[int]							=    num_madeup_words



  @property
  def  _a      (self	):


        """simple docstring"""

        return len(self.encoder	) + len(self.lang_token_to_id	)



  @property
  def  _a      (self	):


        """simple docstring"""

        return self._src_lang



  @src_lang.setter
  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							Dict							=    new_src_lang
        self.set_src_lang_special_tokens(self._src_lang	)



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        return self.sp_model.encode(_lowerCamelCase							,       out_type=_lowerCamelCase	)



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        if token in self.lang_token_to_id:
              return self.lang_token_to_id[token]
        return self.encoder.get(_lowerCamelCase							,       self.encoder[self.unk_token]	)



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        if index in self.id_to_lang_token:
              return self.id_to_lang_token[index]
        return self.decoder.get(_lowerCamelCase							,       self.unk_token	)



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    []
        UpperCAmelCase__			:							Optional[int]							=    """"""
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    out_string += self.sp_model.decode(_lowerCamelCase	) + token
                    UpperCAmelCase__			:							List[str]							=    []
              else:
                    current_sub_tokens.append(_lowerCamelCase	)
        out_string += self.sp_model.decode(_lowerCamelCase	)
        return out_string.strip()



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = None							,       _lowerCamelCase = False	):


        """simple docstring"""

        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_lowerCamelCase							,       token_ids_a=_lowerCamelCase							,       already_has_special_tokens=_lowerCamelCase	)

        UpperCAmelCase__			:							List[Any]							=    [1] * len(self.prefix_tokens	)
        UpperCAmelCase__			:							Union[str, Any]							=    [1] * len(self.suffix_tokens	)
        if token_ids_a is None:
              return prefix_ones + ([0] * len(_lowerCamelCase	)) + suffix_ones
        return prefix_ones + ([0] * len(_lowerCamelCase	)) + ([0] * len(_lowerCamelCase	)) + suffix_ones



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = None	):


        """simple docstring"""

        if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Any							=    {self.convert_ids_to_tokens(_lowerCamelCase	): i for i in range(self.vocab_size	)}
        vocab.update(self.added_tokens_encoder	)
        return vocab



  def __getstate__(self	):


        """simple docstring"""

        UpperCAmelCase__			:							Union[str, Any]							=    self.__dict__.copy()
        UpperCAmelCase__			:							Any							=    None
        return state



  def __setstate__(self							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							int							=    d

        # for backward compatibility
        if not hasattr(self							,       """sp_model_kwargs"""	):
              UpperCAmelCase__			:							str							=    {}

        UpperCAmelCase__			:							List[str]							=    load_spm(self.spm_file							,       self.sp_model_kwargs	)



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = None	):


        """simple docstring"""

        UpperCAmelCase__			:							Dict							=    Path(_lowerCamelCase	)
        if not save_dir.is_dir():
              raise OSError(F"""{save_directory} should be a directory"""	)
        UpperCAmelCase__			:							Any							=    save_dir / (
            (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
        )
        UpperCAmelCase__			:							Dict							=    save_dir / (
            (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
        )

        save_json(self.encoder							,       _lowerCamelCase	)

        if os.path.abspath(self.spm_file	) != os.path.abspath(_lowerCamelCase	) and os.path.isfile(self.spm_file	):
              copyfile(self.spm_file							,       _lowerCamelCase	)
        elif not os.path.isfile(self.spm_file	):
              with open(_lowerCamelCase							,       """wb"""	) as fi:
                    UpperCAmelCase__			:							Dict							=    self.sp_model.serialized_model_proto()
                    fi.write(_lowerCamelCase	)

        return (str(_lowerCamelCase	), str(_lowerCamelCase	))



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = "en"							,       _lowerCamelCase = None							,       _lowerCamelCase = "ro"							,       **_lowerCamelCase							,       ):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[int]							=    src_lang
        UpperCAmelCase__			:							Optional[int]							=    tgt_lang
        self.set_src_lang_special_tokens(self.src_lang	)
        return super().prepare_seqaseq_batch(_lowerCamelCase							,       _lowerCamelCase							,       **_lowerCamelCase	)



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase							,       _lowerCamelCase							,       **_lowerCamelCase	):


        """simple docstring"""

        if src_lang is None or tgt_lang is None:
              raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""	)
        UpperCAmelCase__			:							Union[str, Any]							=    src_lang
        UpperCAmelCase__			:							Union[str, Any]							=    self(_lowerCamelCase							,       add_special_tokens=_lowerCamelCase							,       **_lowerCamelCase	)
        UpperCAmelCase__			:							Any							=    self.get_lang_id(_lowerCamelCase	)
        UpperCAmelCase__			:							Any							=    tgt_lang_id
        return inputs



  def  _a      (self	):


        """simple docstring"""

        self.set_src_lang_special_tokens(self.src_lang	)



  def  _a      (self	):


        """simple docstring"""

        self.set_tgt_lang_special_tokens(self.tgt_lang	)



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							List[Any]							=    self.get_lang_token(_lowerCamelCase	)
        UpperCAmelCase__			:							Optional[int]							=    self.lang_token_to_id[lang_token]
        UpperCAmelCase__			:							Union[str, Any]							=    [self.cur_lang_id]
        UpperCAmelCase__			:							Optional[int]							=    [self.eos_token_id]



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    self.get_lang_token(_lowerCamelCase	)
        UpperCAmelCase__			:							Optional[Any]							=    self.lang_token_to_id[lang_token]
        UpperCAmelCase__			:							Union[str, Any]							=    [self.cur_lang_id]
        UpperCAmelCase__			:							Optional[int]							=    [self.eos_token_id]



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        return self.lang_code_to_token[lang]




  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[int]							=    self.get_lang_token(_lowerCamelCase	)
        return self.lang_token_to_id[lang_token]




def 					a__						(						lowerCAmelCase			,	lowerCAmelCase    )	-> sentencepiece.SentencePieceProcessor:
      UpperCAmelCase__			:							Dict							=    sentencepiece.SentencePieceProcessor(**lowerCAmelCase    )
      spm.Load(str(lowerCAmelCase    )    )
      return spm




def 					a__						(						lowerCAmelCase    )	-> Union[Dict, List]:
      with open(lowerCAmelCase			,	"""r"""    ) as f:
            return json.load(lowerCAmelCase    )





def 					a__						(						lowerCAmelCase			,	lowerCAmelCase    )	-> None:
      with open(lowerCAmelCase			,	"""w"""    ) as f:
            json.dump(lowerCAmelCase			,	lowerCAmelCase			,	indent=2    )




"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


_A							=  {}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _A							=  ["""BartphoTokenizer"""]

if TYPE_CHECKING:
    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_bartpho import BartphoTokenizer

else:
    import sys

    _A							=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

