
'''simple docstring'''


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


snake_case_ :  int     =					logging.get_logger(__name__)

snake_case_ :  Optional[Any]     =					'▁'

snake_case_ :  Union[str, Any]     =					{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

snake_case_ :  Optional[Any]     =					{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

snake_case_ :  Optional[int]     =					{
    'facebook/m2m100_418M': 1_024,
}

# fmt: off
snake_case_ :  int     =					{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							['''input_ids''', '''attention_mask''']

   _snake_case		   =							[]
   _snake_case		   =							[]







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__="<s>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="<pad>" ,   lowerCamelCase__="<unk>" ,   lowerCamelCase__="m2m100" ,   lowerCamelCase__ = None ,   lowerCamelCase__=8 ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  {} if sp_model_kwargs is None else sp_model_kwargs

      UpperCamelCase       =  language_codes
      UpperCamelCase       =  FAIRSEQ_LANGUAGE_CODES[language_codes]
      UpperCamelCase       =  {lang_code: f'__{lang_code}__' for lang_code in fairseq_language_code}

      UpperCamelCase       =  kwargs.get('''additional_special_tokens''' ,   []   )
      kwargs["additional_special_tokens"] += [
          self.get_lang_token(lowerCamelCase__   )
          for lang_code in fairseq_language_code
          if self.get_lang_token(lowerCamelCase__   ) not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          src_lang=lowerCamelCase__ ,   tgt_lang=lowerCamelCase__ ,   bos_token=lowerCamelCase__ ,   eos_token=lowerCamelCase__ ,   sep_token=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   language_codes=lowerCamelCase__ ,   sp_model_kwargs=self.sp_model_kwargs ,   num_madeup_words=lowerCamelCase__ ,   **lowerCamelCase__ ,   )

      UpperCamelCase       =  vocab_file
      UpperCamelCase       =  load_json(lowerCamelCase__   )
      UpperCamelCase       =  {v: k for k, v in self.encoder.items()}
      UpperCamelCase       =  spm_file
      UpperCamelCase       =  load_spm(lowerCamelCase__ ,   self.sp_model_kwargs   )

      UpperCamelCase       =  len(self.encoder   )

      UpperCamelCase       =  {
          self.get_lang_token(lowerCamelCase__   ): self.encoder_size + i for i, lang_code in enumerate(lowerCamelCase__   )
      }
      UpperCamelCase       =  {lang_code: self.encoder_size + i for i, lang_code in enumerate(lowerCamelCase__   )}
      UpperCamelCase       =  {v: k for k, v in self.lang_token_to_id.items()}

      UpperCamelCase       =  src_lang if src_lang is not None else '''en'''
      UpperCamelCase       =  tgt_lang
      UpperCamelCase       =  self.get_lang_id(self._src_lang   )
      self.set_src_lang_special_tokens(self._src_lang   )

      UpperCamelCase       =  num_madeup_words







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return len(self.encoder   ) + len(self.lang_token_to_id   )







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return self._src_lang







   @src_lang.setter
   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  new_src_lang
      self.set_src_lang_special_tokens(self._src_lang   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.encode(lowerCamelCase__ ,   out_type=lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if token in self.lang_token_to_id:
         return self.lang_token_to_id[token]
      return self.encoder.get(lowerCamelCase__ ,   self.encoder[self.unk_token]   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if index in self.id_to_lang_token:
         return self.id_to_lang_token[index]
      return self.decoder.get(lowerCamelCase__ ,   self.unk_token   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  []
      UpperCamelCase       =  ''''''
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            out_string += self.sp_model.decode(lowerCamelCase__   ) + token
            UpperCamelCase       =  []
         else:
            current_sub_tokens.append(lowerCamelCase__   )
      out_string += self.sp_model.decode(lowerCamelCase__   )
      return out_string.strip()







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None ,   lowerCamelCase__ = False   ):

      '''simple docstring'''







      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowerCamelCase__ ,   token_ids_a=lowerCamelCase__ ,   already_has_special_tokens=lowerCamelCase__   )

      UpperCamelCase       =  [1] * len(self.prefix_tokens   )
      UpperCamelCase       =  [1] * len(self.suffix_tokens   )
      if token_ids_a is None:
         return prefix_ones + ([0] * len(lowerCamelCase__   )) + suffix_ones
      return prefix_ones + ([0] * len(lowerCamelCase__   )) + ([0] * len(lowerCamelCase__   )) + suffix_ones







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if token_ids_a is None:
         return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  {self.convert_ids_to_tokens(lowerCamelCase__   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab







   def __getstate__(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.__dict__.copy()
      UpperCamelCase       =  None
      return state







   def __setstate__(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  d

      # for backward compatibility
      if not hasattr(self ,   '''sp_model_kwargs'''   ):
         UpperCamelCase       =  {}

      UpperCamelCase       =  load_spm(self.spm_file ,   self.sp_model_kwargs   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      UpperCamelCase       =  Path(lowerCamelCase__   )
      if not save_dir.is_dir():
         raise OSError(f'{save_directory} should be a directory'   )
      UpperCamelCase       =  save_dir / (
          (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
      )
      UpperCamelCase       =  save_dir / (
          (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
      )

      save_json(self.encoder ,   lowerCamelCase__   )

      if os.path.abspath(self.spm_file   ) != os.path.abspath(lowerCamelCase__   ) and os.path.isfile(self.spm_file   ):
         copyfile(self.spm_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.spm_file   ):
         with open(lowerCamelCase__ ,   '''wb'''   ) as fi:
            UpperCamelCase       =  self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase__   )

      return (str(lowerCamelCase__   ), str(lowerCamelCase__   ))







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = "en" ,   lowerCamelCase__ = None ,   lowerCamelCase__ = "ro" ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  src_lang
      UpperCamelCase       =  tgt_lang
      self.set_src_lang_special_tokens(self.src_lang   )
      return super().prepare_seqaseq_batch(lowerCamelCase__ ,   lowerCamelCase__ ,   **lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      if src_lang is None or tgt_lang is None:
         raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''   )
      UpperCamelCase       =  src_lang
      UpperCamelCase       =  self(lowerCamelCase__ ,   add_special_tokens=lowerCamelCase__ ,   **lowerCamelCase__   )
      UpperCamelCase       =  self.get_lang_id(lowerCamelCase__   )
      UpperCamelCase       =  tgt_lang_id
      return inputs







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      self.set_src_lang_special_tokens(self.src_lang   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      self.set_tgt_lang_special_tokens(self.tgt_lang   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self.get_lang_token(lowerCamelCase__   )
      UpperCamelCase       =  self.lang_token_to_id[lang_token]
      UpperCamelCase       =  [self.cur_lang_id]
      UpperCamelCase       =  [self.eos_token_id]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self.get_lang_token(lowerCamelCase__   )
      UpperCamelCase       =  self.lang_token_to_id[lang_token]
      UpperCamelCase       =  [self.cur_lang_id]
      UpperCamelCase       =  [self.eos_token_id]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.lang_code_to_token[lang]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self.get_lang_token(lowerCamelCase__   )
      return self.lang_token_to_id[lang_token]




def  __snake_case     (    _UpperCAmelCase					:       str,      _UpperCAmelCase					:       Dict[str, Any]):
   UpperCamelCase       =  sentencepiece.SentencePieceProcessor(**_UpperCAmelCase)
   spm.Load(str(_UpperCAmelCase))
   return spm



def  __snake_case     (    _UpperCAmelCase					:       str):
   with open(_UpperCAmelCase,      '''r''') as f:
      return json.load(_UpperCAmelCase)



def  __snake_case     (    _UpperCAmelCase					:       Dict,      _UpperCAmelCase					:       str):
   with open(_UpperCAmelCase,      '''w''') as f:
      json.dump(_UpperCAmelCase,      _UpperCAmelCase,      indent=2)




'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case_ :  Optional[int]     =					logging.get_logger(__name__)

snake_case_ :  int     =					'▁'

snake_case_ :  str     =					{'vocab_file': 'sentencepiece.bpe.model', 'monolingual_vocab_file': 'dict.txt'}

snake_case_ :  Optional[Any]     =					{
    'vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model',
    },
    'monolingual_vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt',
    },
}

snake_case_ :  Dict     =					{'vinai/bartpho-syllable': 1_024}


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							['''input_ids''', '''attention_mask''']







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__="<s>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="<s>" ,   lowerCamelCase__="<unk>" ,   lowerCamelCase__="<pad>" ,   lowerCamelCase__="<mask>" ,   lowerCamelCase__ = None ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  AddedToken(lowerCamelCase__ ,   lstrip=lowerCamelCase__ ,   rstrip=lowerCamelCase__   ) if isinstance(lowerCamelCase__ ,   lowerCamelCase__   ) else mask_token

      UpperCamelCase       =  {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=lowerCamelCase__ ,   eos_token=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   sep_token=lowerCamelCase__ ,   cls_token=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   mask_token=lowerCamelCase__ ,   sp_model_kwargs=self.sp_model_kwargs ,   **lowerCamelCase__ ,   )

      UpperCamelCase       =  vocab_file
      UpperCamelCase       =  monolingual_vocab_file
      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(str(lowerCamelCase__   )   )

      # Load the reduced vocab

      # Keep order of special tokens for backward compatibility
      UpperCamelCase       =  {}
      UpperCamelCase       =  0
      for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
         if str(lowerCamelCase__   ) not in self.fairseq_tokens_to_ids:
            UpperCamelCase       =  cnt
            cnt += 1
      with open(lowerCamelCase__ ,   '''r''' ,   encoding='''utf-8'''   ) as f:
         for line in f.readlines():
            UpperCamelCase       =  line.strip().split()[0]
            UpperCamelCase       =  len(self.fairseq_tokens_to_ids   )
      if str(lowerCamelCase__   ) not in self.fairseq_tokens_to_ids:
         UpperCamelCase       =  len(self.fairseq_tokens_to_ids   )

      UpperCamelCase       =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}







   def __getstate__(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.__dict__.copy()
      UpperCamelCase       =  None
      UpperCamelCase       =  self.sp_model.serialized_model_proto()
      return state







   def __setstate__(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  d

      # for backward compatibility
      if not hasattr(self ,   '''sp_model_kwargs'''   ):
         UpperCamelCase       =  {}

      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      UpperCamelCase       =  [self.cls_token_id]
      UpperCamelCase       =  [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None ,   lowerCamelCase__ = False   ):

      '''simple docstring'''







      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowerCamelCase__ ,   token_ids_a=lowerCamelCase__ ,   already_has_special_tokens=lowerCamelCase__   )

      if token_ids_a is None:
         return [1] + ([0] * len(lowerCamelCase__   )) + [1]
      return [1] + ([0] * len(lowerCamelCase__   )) + [1, 1] + ([0] * len(lowerCamelCase__   )) + [1]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      UpperCamelCase       =  [self.sep_token_id]
      UpperCamelCase       =  [self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return len(self.fairseq_ids_to_tokens   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  {self.convert_ids_to_tokens(lowerCamelCase__   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.encode(lowerCamelCase__ ,   out_type=lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if token in self.fairseq_tokens_to_ids:
         return self.fairseq_tokens_to_ids[token]
      else:
         return self.unk_token_id







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.fairseq_ids_to_tokens[index]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  ''''''.join(lowerCamelCase__   ).replace(lowerCamelCase__ ,   ''' '''   ).strip()
      return out_string







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if not os.path.isdir(lowerCamelCase__   ):
         logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
         return
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''monolingual_vocab_file'''] ,   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase__   ) and os.path.isfile(self.vocab_file   ):
         copyfile(self.vocab_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.vocab_file   ):
         with open(lowerCamelCase__ ,   '''wb'''   ) as fi:
            UpperCamelCase       =  self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase__   )

      if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
          lowerCamelCase__   ) and os.path.isfile(self.monolingual_vocab_file   ):
         copyfile(self.monolingual_vocab_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.monolingual_vocab_file   ):
         with open(lowerCamelCase__ ,   '''w''' ,   encoding='''utf-8'''   ) as fp:
            for token in self.fairseq_tokens_to_ids:
               if token not in self.all_special_tokens:
                  fp.write(f'{str(lowerCamelCase__   )} \n'   )

      return out_vocab_file, out_monolingual_vocab_file





