

"""simple docstring"""

import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase		     =    logging.get_logger(__name__)

_UpperCamelCase		     =    """▁"""

_UpperCamelCase		     =    {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

_UpperCamelCase		     =    {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

_UpperCamelCase		     =    {
    """facebook/m2m100_418M""": 1_0_2_4,
}

# fmt: off
_UpperCamelCase		     =    {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}






class 			__a					(						__magic_name__       ):





       """simple docstring"""


       __UpperCamelCase					:      Dict    =						VOCAB_FILES_NAMES
       __UpperCamelCase					:      int    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __UpperCamelCase					:      Union[str, Any]    =						PRETRAINED_VOCAB_FILES_MAP
       __UpperCamelCase					:      List[str]    =						['input_ids', 'attention_mask']

       __UpperCamelCase					:      List[int]    =						[]
       __UpperCamelCase					:      List[int]    =						[]






       def __init__(   self			,					snake_case			,					snake_case			,					snake_case=None			,					snake_case=None			,					snake_case="<s>"			,					snake_case="</s>"			,					snake_case="</s>"			,					snake_case="<pad>"			,					snake_case="<unk>"			,					snake_case="m2m100"			,					snake_case = None			,					snake_case=8			,					**snake_case			,					):





        """simple docstring"""

        lowerCAmelCase__   :				int 			=				{} if sp_model_kwargs is None else sp_model_kwargs

        lowerCAmelCase__   :				List[Any] 			=				language_codes
        lowerCAmelCase__   :				List[str] 			=				FAIRSEQ_LANGUAGE_CODES[language_codes]
        lowerCAmelCase__   :				Optional[Any] 			=				{lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

        lowerCAmelCase__   :				Union[str, Any] 			=				kwargs.get("additional_special_tokens"			,					[]   )
        kwargs["additional_special_tokens"] += [
            self.get_lang_token(snake_case   )
            for lang_code in fairseq_language_code
            if self.get_lang_token(snake_case   ) not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            src_lang=snake_case			,					tgt_lang=snake_case			,					bos_token=snake_case			,					eos_token=snake_case			,					sep_token=snake_case			,					unk_token=snake_case			,					pad_token=snake_case			,					language_codes=snake_case			,					sp_model_kwargs=self.sp_model_kwargs			,					num_madeup_words=snake_case			,					**snake_case			,					)

        lowerCAmelCase__   :				Any 			=				vocab_file
        lowerCAmelCase__   :				Optional[int] 			=				load_json(snake_case   )
        lowerCAmelCase__   :				Optional[int] 			=				{v: k for k, v in self.encoder.items()}
        lowerCAmelCase__   :				int 			=				spm_file
        lowerCAmelCase__   :				List[str] 			=				load_spm(snake_case			,					self.sp_model_kwargs   )

        lowerCAmelCase__   :				Optional[int] 			=				len(self.encoder   )

        lowerCAmelCase__   :				str 			=				{
            self.get_lang_token(snake_case   ): self.encoder_size + i for i, lang_code in enumerate(snake_case   )
        }
        lowerCAmelCase__   :				Dict 			=				{lang_code: self.encoder_size + i for i, lang_code in enumerate(snake_case   )}
        lowerCAmelCase__   :				Optional[Any] 			=				{v: k for k, v in self.lang_token_to_id.items()}

        lowerCAmelCase__   :				Tuple 			=				src_lang if src_lang is not None else "en"
        lowerCAmelCase__   :				Tuple 			=				tgt_lang
        lowerCAmelCase__   :				Optional[Any] 			=				self.get_lang_id(self._src_lang   )
        self.set_src_lang_special_tokens(self._src_lang   )

        lowerCAmelCase__   :				Union[str, Any] 			=				num_madeup_words






       @property
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        return len(self.encoder   ) + len(self.lang_token_to_id   )






       @property
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        return self._src_lang






       @src_lang.setter
       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				str 			=				new_src_lang
        self.set_src_lang_special_tokens(self._src_lang   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        return self.sp_model.encode(snake_case			,					out_type=snake_case   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        if token in self.lang_token_to_id:
         return self.lang_token_to_id[token]
        return self.encoder.get(snake_case			,					self.encoder[self.unk_token]   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        if index in self.id_to_lang_token:
         return self.id_to_lang_token[index]
        return self.decoder.get(snake_case			,					self.unk_token   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Tuple 			=				[]
        lowerCAmelCase__   :				Dict 			=				""
        for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
          out_string += self.sp_model.decode(snake_case   ) + token
          lowerCAmelCase__   :				Tuple 			=				[]
         else:
          current_sub_tokens.append(snake_case   )
        out_string += self.sp_model.decode(snake_case   )
        return out_string.strip()






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None			,					snake_case = False   ):





        """simple docstring"""

        if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=snake_case			,					token_ids_a=snake_case			,					already_has_special_tokens=snake_case   )

        lowerCAmelCase__   :				Optional[Any] 			=				[1] * len(self.prefix_tokens   )
        lowerCAmelCase__   :				Any 			=				[1] * len(self.suffix_tokens   )
        if token_ids_a is None:
         return prefix_ones + ([0] * len(snake_case   )) + suffix_ones
        return prefix_ones + ([0] * len(snake_case   )) + ([0] * len(snake_case   )) + suffix_ones






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None   ):





        """simple docstring"""

        if token_ids_a is None:
         return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Tuple 			=				{self.convert_ids_to_tokens(snake_case   ): i for i in range(self.vocab_size   )}
        vocab.update(self.added_tokens_encoder   )
        return vocab






       def __getstate__(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Optional[int] 			=				self.__dict__.copy()
        lowerCAmelCase__   :				Union[str, Any] 			=				None
        return state






       def __setstate__(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Union[str, Any] 			=				d

        # for backward compatibility
        if not hasattr(self			,					"sp_model_kwargs"   ):
         lowerCAmelCase__   :				Dict 			=				{}

        lowerCAmelCase__   :				str 			=				load_spm(self.spm_file			,					self.sp_model_kwargs   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None   ):





        """simple docstring"""

        lowerCAmelCase__   :				Dict 			=				Path(snake_case   )
        if not save_dir.is_dir():
         raise OSError(F"""{save_directory} should be a directory"""   )
        lowerCAmelCase__   :				List[str] 			=				save_dir / (
            (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
        )
        lowerCAmelCase__   :				List[str] 			=				save_dir / (
            (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
        )

        save_json(self.encoder			,					snake_case   )

        if os.path.abspath(self.spm_file   ) != os.path.abspath(snake_case   ) and os.path.isfile(self.spm_file   ):
         copyfile(self.spm_file			,					snake_case   )
        elif not os.path.isfile(self.spm_file   ):
         with open(snake_case			,					"wb"   ) as fi:
          lowerCAmelCase__   :				Union[str, Any] 			=				self.sp_model.serialized_model_proto()
          fi.write(snake_case   )

        return (str(snake_case   ), str(snake_case   ))






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = "en"			,					snake_case = None			,					snake_case = "ro"			,					**snake_case			,					):





        """simple docstring"""

        lowerCAmelCase__   :				int 			=				src_lang
        lowerCAmelCase__   :				List[str] 			=				tgt_lang
        self.set_src_lang_special_tokens(self.src_lang   )
        return super().prepare_seqaseq_batch(snake_case			,					snake_case			,					**snake_case   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case			,					snake_case			,					**snake_case   ):





        """simple docstring"""

        if src_lang is None or tgt_lang is None:
         raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"   )
        lowerCAmelCase__   :				str 			=				src_lang
        lowerCAmelCase__   :				str 			=				self(snake_case			,					add_special_tokens=snake_case			,					**snake_case   )
        lowerCAmelCase__   :				Dict 			=				self.get_lang_id(snake_case   )
        lowerCAmelCase__   :				Optional[Any] 			=				tgt_lang_id
        return inputs






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        self.set_src_lang_special_tokens(self.src_lang   )






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        self.set_tgt_lang_special_tokens(self.tgt_lang   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Optional[Any] 			=				self.get_lang_token(snake_case   )
        lowerCAmelCase__   :				List[Any] 			=				self.lang_token_to_id[lang_token]
        lowerCAmelCase__   :				str 			=				[self.cur_lang_id]
        lowerCAmelCase__   :				Any 			=				[self.eos_token_id]






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Union[str, Any] 			=				self.get_lang_token(snake_case   )
        lowerCAmelCase__   :				int 			=				self.lang_token_to_id[lang_token]
        lowerCAmelCase__   :				Optional[Any] 			=				[self.cur_lang_id]
        lowerCAmelCase__   :				Any 			=				[self.eos_token_id]






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        return self.lang_code_to_token[lang]






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Optional[Any] 			=				self.get_lang_token(snake_case   )
        return self.lang_token_to_id[lang_token]




def       SCREAMING_SNAKE_CASE							( lowercase__					,					lowercase__   )			->						sentencepiece.SentencePieceProcessor:
 lowerCAmelCase__   :				Any 			=				sentencepiece.SentencePieceProcessor(**lowercase__   )
 spm.Load(str(lowercase__   )   )
 return spm



def       SCREAMING_SNAKE_CASE							( lowercase__   )			->						Union[Dict, List]:
 with open(lowercase__					,					"r"   ) as f:
  return json.load(lowercase__   )






def       SCREAMING_SNAKE_CASE							( lowercase__					,					lowercase__   )			->						None:
 with open(lowercase__					,					"w"   ) as f:
  json.dump(lowercase__					,					lowercase__					,					indent=2   )









"""simple docstring"""

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase		     =    logging.get_logger(__name__)

_UpperCamelCase		     =    """▁"""

_UpperCamelCase		     =    {"""vocab_file""": """sentencepiece.bpe.model""", """monolingual_vocab_file""": """dict.txt"""}

_UpperCamelCase		     =    {
    """vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model""",
    },
    """monolingual_vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt""",
    },
}

_UpperCamelCase		     =    {"""vinai/bartpho-syllable""": 1_0_2_4}






class 			__a					(						__magic_name__       ):





       """simple docstring"""


       __UpperCamelCase					:      int    =						VOCAB_FILES_NAMES
       __UpperCamelCase					:      List[str]    =						PRETRAINED_VOCAB_FILES_MAP
       __UpperCamelCase					:      List[Any]    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __UpperCamelCase					:      int    =						['input_ids', 'attention_mask']






       def __init__(   self			,					snake_case			,					snake_case			,					snake_case="<s>"			,					snake_case="</s>"			,					snake_case="</s>"			,					snake_case="<s>"			,					snake_case="<unk>"			,					snake_case="<pad>"			,					snake_case="<mask>"			,					snake_case = None			,					**snake_case			,					):





        """simple docstring"""

        lowerCAmelCase__   :				Union[str, Any] 			=				AddedToken(snake_case			,					lstrip=snake_case			,					rstrip=snake_case   ) if isinstance(snake_case			,					snake_case   ) else mask_token

        lowerCAmelCase__   :				Union[str, Any] 			=				{} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=snake_case			,					eos_token=snake_case			,					unk_token=snake_case			,					sep_token=snake_case			,					cls_token=snake_case			,					pad_token=snake_case			,					mask_token=snake_case			,					sp_model_kwargs=self.sp_model_kwargs			,					**snake_case			,					)

        lowerCAmelCase__   :				Union[str, Any] 			=				vocab_file
        lowerCAmelCase__   :				Optional[Any] 			=				monolingual_vocab_file
        lowerCAmelCase__   :				Optional[Any] 			=				spm.SentencePieceProcessor(**self.sp_model_kwargs   )
        self.sp_model.Load(str(snake_case   )   )

        # Load the reduced vocab

        # Keep order of special tokens for backward compatibility
        lowerCAmelCase__   :				Optional[int] 			=				{}
        lowerCAmelCase__   :				int 			=				0
        for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
         if str(snake_case   ) not in self.fairseq_tokens_to_ids:
          lowerCAmelCase__   :				Optional[int] 			=				cnt
          cnt += 1
        with open(snake_case			,					"r"			,					encoding="utf-8"   ) as f:
         for line in f.readlines():
          lowerCAmelCase__   :				Union[str, Any] 			=				line.strip().split()[0]
          lowerCAmelCase__   :				List[str] 			=				len(self.fairseq_tokens_to_ids   )
        if str(snake_case   ) not in self.fairseq_tokens_to_ids:
         lowerCAmelCase__   :				int 			=				len(self.fairseq_tokens_to_ids   )

        lowerCAmelCase__   :				Union[str, Any] 			=				{v: k for k, v in self.fairseq_tokens_to_ids.items()}






       def __getstate__(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[Any] 			=				self.__dict__.copy()
        lowerCAmelCase__   :				str 			=				None
        lowerCAmelCase__   :				Union[str, Any] 			=				self.sp_model.serialized_model_proto()
        return state






       def __setstate__(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Any 			=				d

        # for backward compatibility
        if not hasattr(self			,					"sp_model_kwargs"   ):
         lowerCAmelCase__   :				Any 			=				{}

        lowerCAmelCase__   :				Dict 			=				spm.SentencePieceProcessor(**self.sp_model_kwargs   )
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None   ):





        """simple docstring"""

        if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        lowerCAmelCase__   :				Any 			=				[self.cls_token_id]
        lowerCAmelCase__   :				Dict 			=				[self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None			,					snake_case = False   ):





        """simple docstring"""

        if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=snake_case			,					token_ids_a=snake_case			,					already_has_special_tokens=snake_case   )

        if token_ids_a is None:
         return [1] + ([0] * len(snake_case   )) + [1]
        return [1] + ([0] * len(snake_case   )) + [1, 1] + ([0] * len(snake_case   )) + [1]






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None   ):





        """simple docstring"""

        lowerCAmelCase__   :				Optional[Any] 			=				[self.sep_token_id]
        lowerCAmelCase__   :				Optional[int] 			=				[self.cls_token_id]

        if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






       @property
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        return len(self.fairseq_ids_to_tokens   )






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[str] 			=				{self.convert_ids_to_tokens(snake_case   ): i for i in range(self.vocab_size   )}
        vocab.update(self.added_tokens_encoder   )
        return vocab






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        return self.sp_model.encode(snake_case			,					out_type=snake_case   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        if token in self.fairseq_tokens_to_ids:
         return self.fairseq_tokens_to_ids[token]
        else:
         return self.unk_token_id






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        return self.fairseq_ids_to_tokens[index]






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[str] 			=				"".join(snake_case   ).replace(snake_case			,					" "   ).strip()
        return out_string






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None   ):





        """simple docstring"""

        if not os.path.isdir(snake_case   ):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
         return
        lowerCAmelCase__   :				List[str] 			=				os.path.join(
            snake_case			,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
        lowerCAmelCase__   :				Optional[Any] 			=				os.path.join(
            snake_case			,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["monolingual_vocab_file"]			,					)

        if os.path.abspath(self.vocab_file   ) != os.path.abspath(snake_case   ) and os.path.isfile(self.vocab_file   ):
         copyfile(self.vocab_file			,					snake_case   )
        elif not os.path.isfile(self.vocab_file   ):
         with open(snake_case			,					"wb"   ) as fi:
          lowerCAmelCase__   :				str 			=				self.sp_model.serialized_model_proto()
          fi.write(snake_case   )

        if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
            snake_case   ) and os.path.isfile(self.monolingual_vocab_file   ):
         copyfile(self.monolingual_vocab_file			,					snake_case   )
        elif not os.path.isfile(self.monolingual_vocab_file   ):
         with open(snake_case			,					"w"			,					encoding="utf-8"   ) as fp:
          for token in self.fairseq_tokens_to_ids:
           if token not in self.all_special_tokens:
            fp.write(F"""{str(snake_case   )} \n"""   )

        return out_vocab_file, out_monolingual_vocab_file






