
"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__SCREAMING_SNAKE_CASE							=     {
    'configuration_electra': ['ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ElectraConfig', 'ElectraOnnxConfig'],
    'tokenization_electra': ['ElectraTokenizer'],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     ['ElectraTokenizerFast']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     [
     'ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST',
     'ElectraForCausalLM',
     'ElectraForMaskedLM',
     'ElectraForMultipleChoice',
     'ElectraForPreTraining',
     'ElectraForQuestionAnswering',
     'ElectraForSequenceClassification',
     'ElectraForTokenClassification',
     'ElectraModel',
     'ElectraPreTrainedModel',
     'load_tf_weights_in_electra',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     [
     'TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST',
     'TFElectraForMaskedLM',
     'TFElectraForMultipleChoice',
     'TFElectraForPreTraining',
     'TFElectraForQuestionAnswering',
     'TFElectraForSequenceClassification',
     'TFElectraForTokenClassification',
     'TFElectraModel',
     'TFElectraPreTrainedModel',
 ]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     [
     'FlaxElectraForCausalLM',
     'FlaxElectraForMaskedLM',
     'FlaxElectraForMultipleChoice',
     'FlaxElectraForPreTraining',
     'FlaxElectraForQuestionAnswering',
     'FlaxElectraForSequenceClassification',
     'FlaxElectraForTokenClassification',
     'FlaxElectraModel',
     'FlaxElectraPreTrainedModel',
 ]


if TYPE_CHECKING:
 from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraOnnxConfig
 from .tokenization_electra import ElectraTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_electra_fast import ElectraTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_electra import (
      ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
      ElectraForCausalLM,
      ElectraForMaskedLM,
      ElectraForMultipleChoice,
      ElectraForPreTraining,
      ElectraForQuestionAnswering,
      ElectraForSequenceClassification,
      ElectraForTokenClassification,
      ElectraModel,
      ElectraPreTrainedModel,
      load_tf_weights_in_electra,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_electra import (
      TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFElectraForMaskedLM,
      TFElectraForMultipleChoice,
      TFElectraForPreTraining,
      TFElectraForQuestionAnswering,
      TFElectraForSequenceClassification,
      TFElectraForTokenClassification,
      TFElectraModel,
      TFElectraPreTrainedModel,
  )

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_electra import (
      FlaxElectraForCausalLM,
      FlaxElectraForMaskedLM,
      FlaxElectraForMultipleChoice,
      FlaxElectraForPreTraining,
      FlaxElectraForQuestionAnswering,
      FlaxElectraForSequenceClassification,
      FlaxElectraForTokenClassification,
      FlaxElectraModel,
      FlaxElectraPreTrainedModel,
  )

else:
 import sys

 __SCREAMING_SNAKE_CASE							=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'sentencepiece.bpe.model'}

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'camembert-base': 'https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model',
    }
}

__SCREAMING_SNAKE_CASE							=     {
    'camembert-base': 512,
}

__SCREAMING_SNAKE_CASE							=     '▁'





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      ['''input_ids''', '''attention_mask''']



     def __init__(							self :Optional[Any]      ,  _lowerCamelCase :int      ,  _lowerCamelCase :List[Any]="<s>"      ,  _lowerCamelCase :List[Any]="</s>"      ,  _lowerCamelCase :Optional[int]="</s>"      ,  _lowerCamelCase :List[Any]="<s>"      ,  _lowerCamelCase :List[Any]="<unk>"      ,  _lowerCamelCase :Union[str, Any]="<pad>"      ,  _lowerCamelCase :int="<mask>"      ,  _lowerCamelCase :Union[str, Any]=["<s>NOTUSED", "</s>NOTUSED"]      ,  _lowerCamelCase :Optional[Dict[str, Any]] = None      ,  **_lowerCamelCase :Optional[int]      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=AddedToken(_lowerCamelCase      ,  lstrip=_lowerCamelCase      ,  rstrip=_lowerCamelCase   ) if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ) else mask_token

      UpperCamelCase_     :					List[str]   	={} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=_lowerCamelCase      ,  eos_token=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  additional_special_tokens=_lowerCamelCase      ,  sp_model_kwargs=self.sp_model_kwargs      ,  **_lowerCamelCase      ,  )
      UpperCamelCase_     :					Optional[int]   	=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(str(_lowerCamelCase   )   )
      UpperCamelCase_     :					str   	=vocab_file
      # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
      # sentencepiece vocabulary (this is the case for <s> and </s>
      UpperCamelCase_     :					Tuple   	={'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}
      UpperCamelCase_     :					List[str]   	=len(self.fairseq_tokens_to_ids   )
      UpperCamelCase_     :					List[str]   	=len(self.sp_model   ) + len(self.fairseq_tokens_to_ids   )
      UpperCamelCase_     :					Any   	={v: k for k, v in self.fairseq_tokens_to_ids.items()}



     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      UpperCamelCase_     :					Any   	=[self.cls_token_id]
      UpperCamelCase_     :					str   	=[self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep



     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None      ,  _lowerCamelCase :bool = False   ):







      '''simple docstring'''



      if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=_lowerCamelCase      ,  token_ids_a=_lowerCamelCase      ,  already_has_special_tokens=_lowerCamelCase   )

      if token_ids_a is None:
       return [1] + ([0] * len(_lowerCamelCase   )) + [1]
      return [1] + ([0] * len(_lowerCamelCase   )) + [1, 1] + ([0] * len(_lowerCamelCase   )) + [1]



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Union[str, Any]   	=[self.sep_token_id]
      UpperCamelCase_     :					Dict   	=[self.cls_token_id]

      if token_ids_a is None:
       return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]



     @property
     def  lowerCamelCase_						(							self :str   ):







      '''simple docstring'''



      return len(self.fairseq_tokens_to_ids   ) + len(self.sp_model   )



     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	={self.convert_ids_to_tokens(_lowerCamelCase   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab



     def  lowerCamelCase_						(							self :List[str]      ,  _lowerCamelCase :str   ):







      '''simple docstring'''



      return self.sp_model.encode(_lowerCamelCase      ,  out_type=_lowerCamelCase   )



     def  lowerCamelCase_						(							self :str      ,  _lowerCamelCase :Optional[Any]   ):







      '''simple docstring'''



      if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
      elif self.sp_model.PieceToId(_lowerCamelCase   ) == 0:
       # Convert sentence piece unk token to fairseq unk token index
       return self.unk_token_id
      return self.fairseq_offset + self.sp_model.PieceToId(_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :List[Any]   ):







      '''simple docstring'''



      if index in self.fairseq_ids_to_tokens:
       return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset   )



     def  lowerCamelCase_						(							self :int      ,  _lowerCamelCase :List[str]   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=[]
      UpperCamelCase_     :					Optional[Any]   	=''
      UpperCamelCase_     :					Optional[Any]   	=False
      for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
        if not prev_is_special:
         out_string += " "
        out_string += self.sp_model.decode(_lowerCamelCase   ) + token
        UpperCamelCase_     :					Union[str, Any]   	=True
        UpperCamelCase_     :					Optional[Any]   	=[]
       else:
        current_sub_tokens.append(_lowerCamelCase   )
        UpperCamelCase_     :					Any   	=False
      out_string += self.sp_model.decode(_lowerCamelCase   )
      return out_string.strip()



     def __getstate__(							self :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=self.__dict__.copy()
      UpperCamelCase_     :					str   	=None
      return state



     def __setstate__(							self :Optional[Any]      ,  _lowerCamelCase :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=d

      # for backward compatibility
      if not hasattr(self      ,  'sp_model_kwargs'   ):
       UpperCamelCase_     :					Any   	={}

      UpperCamelCase_     :					List[Any]   	=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(self.vocab_file   )




     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      if not os.path.isdir(_lowerCamelCase   ):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
       return
      UpperCamelCase_     :					Any   	=os.path.join(
          _lowerCamelCase      ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowerCamelCase   ) and os.path.isfile(self.vocab_file   ):
       copyfile(self.vocab_file      ,  _lowerCamelCase   )
      elif not os.path.isfile(self.vocab_file   ):
       with open(_lowerCamelCase      ,  'wb'   ) as fi:
        UpperCamelCase_     :					int   	=self.sp_model.serialized_model_proto()
        fi.write(_lowerCamelCase   )

      return (out_vocab_file,)
