





"""simple docstring"""





import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase__				:List[str]				   =       logging.get_logger(__name__)

lowercase__				:List[Any]				   =       '▁'

lowercase__				:List[str]				   =       {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

lowercase__				:Any				   =       {
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

lowercase__				:Optional[Any]				   =       {
    'facebook/m2m100_418M': 1_0_2_4,
}

# fmt: off
lowercase__				:Optional[Any]				   =       {
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       Tuple         =       VOCAB_FILES_NAMES
      _A     :       Optional[Any]         =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _A     :       Optional[Any]         =       PRETRAINED_VOCAB_FILES_MAP
      _A     :       Union[str, Any]         =       ['input_ids', 'attention_mask']

      _A     :       List[int]         =       []
      _A     :       List[int]         =       []

      def __init__(							self       :      Any						,				__lowercase       :      Any						,				__lowercase       :      List[str]						,				__lowercase       :      List[Any]=None						,				__lowercase       :      Tuple=None						,				__lowercase       :      Dict="<s>"						,				__lowercase       :      Tuple="</s>"						,				__lowercase       :      Union[str, Any]="</s>"						,				__lowercase       :      List[Any]="<pad>"						,				__lowercase       :      List[str]="<unk>"						,				__lowercase       :      Any="m2m100"						,				__lowercase       :      Optional[Dict[str, Any]] = None						,				__lowercase       :      str=8						,				**__lowercase       :      str						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						{} if sp_model_kwargs is None else sp_model_kwargs

        __UpperCAmelCase    :						int       =						language_codes
        __UpperCAmelCase    :						Dict       =						FAIRSEQ_LANGUAGE_CODES[language_codes]
        __UpperCAmelCase    :						Tuple       =						{lang_code: f'''__{lang_code}__''' for lang_code in fairseq_language_code}

        __UpperCAmelCase    :						List[Any]       =						kwargs.get('''additional_special_tokens'''						,				[] )
        kwargs["additional_special_tokens"] += [
            self.get_lang_token(__lowercase )
            for lang_code in fairseq_language_code
            if self.get_lang_token(__lowercase ) not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            src_lang=__lowercase						,				tgt_lang=__lowercase						,				bos_token=__lowercase						,				eos_token=__lowercase						,				sep_token=__lowercase						,				unk_token=__lowercase						,				pad_token=__lowercase						,				language_codes=__lowercase						,				sp_model_kwargs=self.sp_model_kwargs						,				num_madeup_words=__lowercase						,				**__lowercase						,				)

        __UpperCAmelCase    :						Any       =						vocab_file
        __UpperCAmelCase    :						Any       =						load_json(__lowercase )
        __UpperCAmelCase    :						Any       =						{v: k for k, v in self.encoder.items()}
        __UpperCAmelCase    :						Optional[int]       =						spm_file
        __UpperCAmelCase    :						Optional[Any]       =						load_spm(__lowercase						,				self.sp_model_kwargs )

        __UpperCAmelCase    :						List[str]       =						len(self.encoder )

        __UpperCAmelCase    :						int       =						{
            self.get_lang_token(__lowercase ): self.encoder_size + i for i, lang_code in enumerate(__lowercase )
        }
        __UpperCAmelCase    :						Optional[int]       =						{lang_code: self.encoder_size + i for i, lang_code in enumerate(__lowercase )}
        __UpperCAmelCase    :						Dict       =						{v: k for k, v in self.lang_token_to_id.items()}

        __UpperCAmelCase    :						Union[str, Any]       =						src_lang if src_lang is not None else '''en'''
        __UpperCAmelCase    :						Tuple       =						tgt_lang
        __UpperCAmelCase    :						Optional[int]       =						self.get_lang_id(self._src_lang )
        self.set_src_lang_special_tokens(self._src_lang )

        __UpperCAmelCase    :						List[Any]       =						num_madeup_words

      @property
      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        return len(self.encoder ) + len(self.lang_token_to_id )

      @property
      def       A_					(							self       :      str ):


        '''simple docstring'''




        return self._src_lang

      @src_lang.setter
      def       A_					(							self       :      int						,				__lowercase       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						new_src_lang
        self.set_src_lang_special_tokens(self._src_lang )

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      str ):


        '''simple docstring'''




        return self.sp_model.encode(__lowercase						,				out_type=__lowercase )

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      List[Any] ):


        '''simple docstring'''




        if token in self.lang_token_to_id:
          return self.lang_token_to_id[token]
        return self.encoder.get(__lowercase						,				self.encoder[self.unk_token] )

      def       A_					(							self       :      Any						,				__lowercase       :      int ):


        '''simple docstring'''




        if index in self.id_to_lang_token:
          return self.id_to_lang_token[index]
        return self.decoder.get(__lowercase						,				self.unk_token )

      def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						[]
        __UpperCAmelCase    :						List[Any]       =						''''''
        for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
            out_string += self.sp_model.decode(__lowercase ) + token
            __UpperCAmelCase    :						Any       =						[]
          else:
            current_sub_tokens.append(__lowercase )
        out_string += self.sp_model.decode(__lowercase )
        return out_string.strip()

      def       A_					(							self       :      List[Any]						,				__lowercase       :      List[int]						,				__lowercase       :      Optional[List[int]] = None						,				__lowercase       :      bool = False ):


        '''simple docstring'''




        if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__lowercase						,				token_ids_a=__lowercase						,				already_has_special_tokens=__lowercase )

        __UpperCAmelCase    :						Union[str, Any]       =						[1] * len(self.prefix_tokens )
        __UpperCAmelCase    :						Union[str, Any]       =						[1] * len(self.suffix_tokens )
        if token_ids_a is None:
          return prefix_ones + ([0] * len(__lowercase )) + suffix_ones
        return prefix_ones + ([0] * len(__lowercase )) + ([0] * len(__lowercase )) + suffix_ones

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      List[int]						,				__lowercase       :      Optional[List[int]] = None ):


        '''simple docstring'''




        if token_ids_a is None:
          return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						{self.convert_ids_to_tokens(__lowercase ): i for i in range(self.vocab_size )}
        vocab.update(self.added_tokens_encoder )
        return vocab

      def __getstate__(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						self.__dict__.copy()
        __UpperCAmelCase    :						Any       =						None
        return state

      def __setstate__(							self       :      Union[str, Any]						,				__lowercase       :      Dict ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						d

        # for backward compatibility
        if not hasattr(self						,				'''sp_model_kwargs''' ):
          __UpperCAmelCase    :						Union[str, Any]       =						{}

        __UpperCAmelCase    :						int       =						load_spm(self.spm_file						,				self.sp_model_kwargs )

      def       A_					(							self       :      str						,				__lowercase       :      str						,				__lowercase       :      Optional[str] = None ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						Path(__lowercase )
        if not save_dir.is_dir():
          raise OSError(f'''{save_directory} should be a directory''' )
        __UpperCAmelCase    :						Any       =						save_dir / (
            (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
        )
        __UpperCAmelCase    :						int       =						save_dir / (
            (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
        )

        save_json(self.encoder						,				__lowercase )

        if os.path.abspath(self.spm_file ) != os.path.abspath(__lowercase ) and os.path.isfile(self.spm_file ):
          copyfile(self.spm_file						,				__lowercase )
        elif not os.path.isfile(self.spm_file ):
          with open(__lowercase						,				'''wb''' ) as fi:
            __UpperCAmelCase    :						List[Any]       =						self.sp_model.serialized_model_proto()
            fi.write(__lowercase )

        return (str(__lowercase ), str(__lowercase ))

      def       A_					(							self       :      Any						,				__lowercase       :      List[str]						,				__lowercase       :      str = "en"						,				__lowercase       :      Optional[List[str]] = None						,				__lowercase       :      str = "ro"						,				**__lowercase       :      Union[str, Any]						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						src_lang
        __UpperCAmelCase    :						List[Any]       =						tgt_lang
        self.set_src_lang_special_tokens(self.src_lang )
        return super().prepare_seqaseq_batch(__lowercase						,				__lowercase						,				**__lowercase )

      def       A_					(							self       :      List[str]						,				__lowercase       :      Any						,				__lowercase       :      Optional[str]						,				__lowercase       :      Optional[str]						,				**__lowercase       :      List[Any] ):


        '''simple docstring'''




        if src_lang is None or tgt_lang is None:
          raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model''' )
        __UpperCAmelCase    :						Optional[int]       =						src_lang
        __UpperCAmelCase    :						Dict       =						self(__lowercase						,				add_special_tokens=__lowercase						,				**__lowercase )
        __UpperCAmelCase    :						str       =						self.get_lang_id(__lowercase )
        __UpperCAmelCase    :						Tuple       =						tgt_lang_id
        return inputs

      def       A_					(							self       :      int ):


        '''simple docstring'''




        self.set_src_lang_special_tokens(self.src_lang )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        self.set_tgt_lang_special_tokens(self.tgt_lang )

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						self.get_lang_token(__lowercase )
        __UpperCAmelCase    :						str       =						self.lang_token_to_id[lang_token]
        __UpperCAmelCase    :						Dict       =						[self.cur_lang_id]
        __UpperCAmelCase    :						int       =						[self.eos_token_id]

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						self.get_lang_token(__lowercase )
        __UpperCAmelCase    :						Union[str, Any]       =						self.lang_token_to_id[lang_token]
        __UpperCAmelCase    :						List[str]       =						[self.cur_lang_id]
        __UpperCAmelCase    :						str       =						[self.eos_token_id]

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      str ):


        '''simple docstring'''




        return self.lang_code_to_token[lang]







      def       A_					(							self       :      List[str]						,				__lowercase       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						self.get_lang_token(__lowercase )
        return self.lang_token_to_id[lang_token]


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  )						->sentencepiece.SentencePieceProcessor:





  """simple docstring"""


  __UpperCAmelCase    :						Tuple       =						sentencepiece.SentencePieceProcessor(**UpperCAmelCase_  )
  spm.Load(str(UpperCAmelCase_  )  )
  return spm


def       lowerCamelCase_       (   UpperCAmelCase_  )						->Union[Dict, List]:





  """simple docstring"""


  with open(UpperCAmelCase_  ,			'''r'''  ) as f:
    return json.load(UpperCAmelCase_  )







def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  )						->None:





  """simple docstring"""


  with open(UpperCAmelCase_  ,			'''w'''  ) as f:
    json.dump(UpperCAmelCase_  ,			UpperCAmelCase_  ,			indent=2  )






"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__				:int				   =       logging.get_logger(__name__)

lowercase__				:List[str]				   =       {
    'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json',
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       List[Any]         =       'gpt_neox'

      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      Any=50_432						,				__lowercase       :      int=6_144						,				__lowercase       :      List[str]=44						,				__lowercase       :      List[Any]=64						,				__lowercase       :      int=24_576						,				__lowercase       :      List[str]="gelu"						,				__lowercase       :      Optional[int]=0.2_5						,				__lowercase       :      Any=10_000						,				__lowercase       :      Union[str, Any]=0.0						,				__lowercase       :      List[str]=0.0						,				__lowercase       :      Union[str, Any]=0.1						,				__lowercase       :      Optional[int]=2_048						,				__lowercase       :      Union[str, Any]=0.0_2						,				__lowercase       :      Optional[int]=1e-5						,				__lowercase       :      str=True						,				__lowercase       :      str=0						,				__lowercase       :      Tuple=2						,				__lowercase       :      Tuple=False						,				__lowercase       :      int=True						,				__lowercase       :      Any=None						,				**__lowercase       :      List[str]						,				):


        '''simple docstring'''




        super().__init__(bos_token_id=__lowercase						,				eos_token_id=__lowercase						,				**__lowercase )
        __UpperCAmelCase    :						int       =						vocab_size
        __UpperCAmelCase    :						Union[str, Any]       =						max_position_embeddings
        __UpperCAmelCase    :						List[str]       =						hidden_size
        __UpperCAmelCase    :						Tuple       =						num_hidden_layers
        __UpperCAmelCase    :						Any       =						num_attention_heads
        __UpperCAmelCase    :						Union[str, Any]       =						intermediate_size
        __UpperCAmelCase    :						str       =						hidden_act
        __UpperCAmelCase    :						str       =						rotary_pct
        __UpperCAmelCase    :						str       =						rotary_emb_base
        __UpperCAmelCase    :						Dict       =						attention_dropout
        __UpperCAmelCase    :						str       =						hidden_dropout
        __UpperCAmelCase    :						Any       =						classifier_dropout
        __UpperCAmelCase    :						Tuple       =						initializer_range
        __UpperCAmelCase    :						Any       =						layer_norm_eps
        __UpperCAmelCase    :						Optional[int]       =						use_cache
        __UpperCAmelCase    :						List[Any]       =						tie_word_embeddings
        __UpperCAmelCase    :						Any       =						use_parallel_residual
        __UpperCAmelCase    :						List[str]       =						rope_scaling
        self._rope_scaling_validation()

        if self.hidden_size % self.num_attention_heads != 0:
          raise ValueError(
              '''The hidden size is not divisble by the number of attention heads! Make sure to update them!''' )







      def       A_					(							self       :      int ):


        '''simple docstring'''




        if self.rope_scaling is None:
          return

        if not isinstance(self.rope_scaling						,				__lowercase ) or len(self.rope_scaling ) != 2:
          raise ValueError(
              '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
              f'''got {self.rope_scaling}''' )
        __UpperCAmelCase    :						Union[str, Any]       =						self.rope_scaling.get('''type'''						,				__lowercase )
        __UpperCAmelCase    :						List[Any]       =						self.rope_scaling.get('''factor'''						,				__lowercase )
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
          raise ValueError(
              f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}''' )
        if rope_scaling_factor is None or not isinstance(__lowercase						,				__lowercase ) or rope_scaling_factor <= 1.0:
          raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}''' )