



'''simple docstring'''






import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase:   List[str]					=		logging.get_logger(__name__)

lowerCAmelCase:   Dict					=		'▁'

lowerCAmelCase:   int					=		{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

lowerCAmelCase:   int					=		{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

lowerCAmelCase:   Union[str, Any]					=		{
    'facebook/m2m100_418M': 1_0_2_4,
}

# fmt: off
lowerCAmelCase:   str					=		{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       VOCAB_FILES_NAMES
    lowercase__					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__					=       PRETRAINED_VOCAB_FILES_MAP
    lowercase__					=       ["""input_ids""", """attention_mask"""]

    lowercase__					=       []
    lowercase__					=       []







    def __init__( self					:    List[str]		,     __snake_case					:    List[Any]		,     __snake_case					:    str		,     __snake_case					:    str=None		,     __snake_case					:    str=None		,     __snake_case					:    Dict="<s>"		,     __snake_case					:    Optional[Any]="</s>"		,     __snake_case					:    Any="</s>"		,     __snake_case					:    Optional[int]="<pad>"		,     __snake_case					:    Any="<unk>"		,     __snake_case					:    Optional[Any]="m2m100"		,     __snake_case					:    Optional[Dict[str, Any]] = None		,     __snake_case					:    Union[str, Any]=8		,     **__snake_case					:    Dict		,     ):
      a		:   List[str]	  =	{} if sp_model_kwargs is None else sp_model_kwargs

      a		:   Union[str, Any]	  =	language_codes
      a		:   Optional[Any]	  =	FAIRSEQ_LANGUAGE_CODES[language_codes]
      a		:   Any	  =	{lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

      a		:   int	  =	kwargs.get('additional_special_tokens'		,     []      )
      kwargs["additional_special_tokens"] += [
          self.get_lang_token(__snake_case      )
          for lang_code in fairseq_language_code
          if self.get_lang_token(__snake_case      ) not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          src_lang=__snake_case		,     tgt_lang=__snake_case		,     bos_token=__snake_case		,     eos_token=__snake_case		,     sep_token=__snake_case		,     unk_token=__snake_case		,     pad_token=__snake_case		,     language_codes=__snake_case		,     sp_model_kwargs=self.sp_model_kwargs		,     num_madeup_words=__snake_case		,     **__snake_case		,     )

      a		:   str	  =	vocab_file
      a		:   List[str]	  =	load_json(__snake_case      )
      a		:   List[str]	  =	{v: k for k, v in self.encoder.items()}
      a		:   Union[str, Any]	  =	spm_file
      a		:   Any	  =	load_spm(__snake_case		,     self.sp_model_kwargs      )

      a		:   Tuple	  =	len(self.encoder      )

      a		:   Dict	  =	{
          self.get_lang_token(__snake_case      ): self.encoder_size + i for i, lang_code in enumerate(__snake_case      )
      }
      a		:   Union[str, Any]	  =	{lang_code: self.encoder_size + i for i, lang_code in enumerate(__snake_case      )}
      a		:   str	  =	{v: k for k, v in self.lang_token_to_id.items()}

      a		:   List[Any]	  =	src_lang if src_lang is not None else 'en'
      a		:   Optional[int]	  =	tgt_lang
      a		:   int	  =	self.get_lang_id(self._src_lang      )
      self.set_src_lang_special_tokens(self._src_lang      )

      a		:   Optional[Any]	  =	num_madeup_words







    @property
    def      lowercase_			( self					:    Dict      ):
      return len(self.encoder      ) + len(self.lang_token_to_id      )







    @property
    def      lowercase_			( self					:    List[str]      ):
      return self._src_lang







    @src_lang.setter
    def      lowercase_			( self					:    Any		,     __snake_case					:    str      ):
      a		:   List[Any]	  =	new_src_lang
      self.set_src_lang_special_tokens(self._src_lang      )







    def      lowercase_			( self					:    int		,     __snake_case					:    str      ):
      return self.sp_model.encode(__snake_case		,     out_type=__snake_case      )







    def      lowercase_			( self					:    Union[str, Any]		,     __snake_case					:    Dict      ):
      if token in self.lang_token_to_id:
        return self.lang_token_to_id[token]
      return self.encoder.get(__snake_case		,     self.encoder[self.unk_token]      )







    def      lowercase_			( self					:    Any		,     __snake_case					:    int      ):
      if index in self.id_to_lang_token:
        return self.id_to_lang_token[index]
      return self.decoder.get(__snake_case		,     self.unk_token      )







    def      lowercase_			( self					:    List[str]		,     __snake_case					:    Optional[int]      ):
      a		:   Dict	  =	[]
      a		:   Dict	  =	''
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          out_string += self.sp_model.decode(__snake_case      ) + token
          a		:   Optional[int]	  =	[]
        else:
          current_sub_tokens.append(__snake_case      )
      out_string += self.sp_model.decode(__snake_case      )
      return out_string.strip()







    def      lowercase_			( self					:    List[Any]		,     __snake_case					:    List[int]		,     __snake_case					:    Optional[List[int]] = None		,     __snake_case					:    bool = False      ):

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__snake_case		,     token_ids_a=__snake_case		,     already_has_special_tokens=__snake_case      )

      a		:   Tuple	  =	[1] * len(self.prefix_tokens      )
      a		:   Dict	  =	[1] * len(self.suffix_tokens      )
      if token_ids_a is None:
        return prefix_ones + ([0] * len(__snake_case      )) + suffix_ones
      return prefix_ones + ([0] * len(__snake_case      )) + ([0] * len(__snake_case      )) + suffix_ones







    def      lowercase_			( self					:    Optional[Any]		,     __snake_case					:    List[int]		,     __snake_case					:    Optional[List[int]] = None      ):
      if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







    def      lowercase_			( self					:    Tuple      ):
      a		:   Union[str, Any]	  =	{self.convert_ids_to_tokens(__snake_case      ): i for i in range(self.vocab_size      )}
      vocab.update(self.added_tokens_encoder      )
      return vocab







    def __getstate__( self					:    str      ):
      a		:   List[str]	  =	self.__dict__.copy()
      a		:   int	  =	None
      return state







    def __setstate__( self					:    Union[str, Any]		,     __snake_case					:    Dict      ):
      a		:   List[Any]	  =	d

      # for backward compatibility
      if not hasattr(self		,     'sp_model_kwargs'      ):
        a		:   Tuple	  =	{}

      a		:   List[str]	  =	load_spm(self.spm_file		,     self.sp_model_kwargs      )







    def      lowercase_			( self					:    Any		,     __snake_case					:    str		,     __snake_case					:    Optional[str] = None      ):
      a		:   int	  =	Path(__snake_case      )
      if not save_dir.is_dir():
        raise OSError(F"""{save_directory} should be a directory"""      )
      a		:   List[Any]	  =	save_dir / (
          (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
      )
      a		:   Any	  =	save_dir / (
          (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
      )

      save_json(self.encoder		,     __snake_case      )

      if os.path.abspath(self.spm_file      ) != os.path.abspath(__snake_case      ) and os.path.isfile(self.spm_file      ):
        copyfile(self.spm_file		,     __snake_case      )
      elif not os.path.isfile(self.spm_file      ):
        with open(__snake_case		,     'wb'      ) as fi:
          a		:   int	  =	self.sp_model.serialized_model_proto()
          fi.write(__snake_case      )

      return (str(__snake_case      ), str(__snake_case      ))







    def      lowercase_			( self					:    str		,     __snake_case					:    List[str]		,     __snake_case					:    str = "en"		,     __snake_case					:    Optional[List[str]] = None		,     __snake_case					:    str = "ro"		,     **__snake_case					:    int		,     ):
      a		:   str	  =	src_lang
      a		:   Tuple	  =	tgt_lang
      self.set_src_lang_special_tokens(self.src_lang      )
      return super().prepare_seqaseq_batch(__snake_case		,     __snake_case		,     **__snake_case      )







    def      lowercase_			( self					:    Optional[Any]		,     __snake_case					:    Optional[Any]		,     __snake_case					:    Optional[str]		,     __snake_case					:    Optional[str]		,     **__snake_case					:    List[Any]      ):
      if src_lang is None or tgt_lang is None:
        raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'      )
      a		:   str	  =	src_lang
      a		:   int	  =	self(__snake_case		,     add_special_tokens=__snake_case		,     **__snake_case      )
      a		:   List[Any]	  =	self.get_lang_id(__snake_case      )
      a		:   Any	  =	tgt_lang_id
      return inputs







    def      lowercase_			( self					:    int      ):
      self.set_src_lang_special_tokens(self.src_lang      )







    def      lowercase_			( self					:    List[str]      ):
      self.set_tgt_lang_special_tokens(self.tgt_lang      )







    def      lowercase_			( self					:    List[Any]		,     __snake_case					:    str      ):
      a		:   Any	  =	self.get_lang_token(__snake_case      )
      a		:   List[str]	  =	self.lang_token_to_id[lang_token]
      a		:   List[Any]	  =	[self.cur_lang_id]
      a		:   List[str]	  =	[self.eos_token_id]







    def      lowercase_			( self					:    Optional[Any]		,     __snake_case					:    str      ):
      a		:   str	  =	self.get_lang_token(__snake_case      )
      a		:   Tuple	  =	self.lang_token_to_id[lang_token]
      a		:   Union[str, Any]	  =	[self.cur_lang_id]
      a		:   Optional[int]	  =	[self.eos_token_id]







    def      lowercase_			( self					:    Any		,     __snake_case					:    str      ):
      return self.lang_code_to_token[lang]







    def      lowercase_			( self					:    Optional[int]		,     __snake_case					:    str      ):
      a		:   Union[str, Any]	  =	self.get_lang_token(__snake_case      )
      return self.lang_token_to_id[lang_token]
def  lowerCamelCase__			(		_A					,     _A							):
  a		:   List[str]	  =	sentencepiece.SentencePieceProcessor(**_A							)
  spm.Load(str(_A							)							)
  return spm
def  lowerCamelCase__			(		_A							):
  with open(_A					,     'r'							) as f:
    return json.load(_A							)

def  lowerCamelCase__			(		_A					,     _A							):
  with open(_A					,     'w'							) as f:
    json.dump(_A					,     _A					,     indent=2							)




'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase:   Optional[int]					=		logging.get_logger(__name__)

lowerCAmelCase:   int					=		{'ctrl': 'https://huggingface.co/ctrl/resolve/main/config.json'}

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       """ctrl"""
    lowercase__					=       ["""past_key_values"""]
    lowercase__					=       {
        """max_position_embeddings""": """n_positions""",
        """hidden_size""": """n_embd""",
        """num_attention_heads""": """n_head""",
        """num_hidden_layers""": """n_layer""",
    }







    def __init__( self					:    Optional[int]		,     __snake_case					:    Union[str, Any]=24_65_34		,     __snake_case					:    Dict=2_56		,     __snake_case					:    Optional[int]=12_80		,     __snake_case					:    Optional[int]=81_92		,     __snake_case					:    Union[str, Any]=48		,     __snake_case					:    str=16		,     __snake_case					:    Any=0.1		,     __snake_case					:    List[Any]=0.1		,     __snake_case					:    str=1e-6		,     __snake_case					:    List[str]=0.02		,     __snake_case					:    int=True		,     **__snake_case					:    Any		,     ):
      a		:   int	  =	vocab_size
      a		:   Union[str, Any]	  =	n_positions
      a		:   Optional[int]	  =	n_embd
      a		:   Dict	  =	n_layer
      a		:   Union[str, Any]	  =	n_head
      a		:   Tuple	  =	dff
      a		:   Union[str, Any]	  =	resid_pdrop
      a		:   Tuple	  =	embd_pdrop
      a		:   int	  =	layer_norm_epsilon
      a		:   str	  =	initializer_range

      a		:   Any	  =	use_cache

      super().__init__(**__snake_case      )