


'''simple docstring'''







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_   =      logging.get_logger(__name__)
SCREAMING_SNAKE_CASE_   =      {'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE_   =      {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

SCREAMING_SNAKE_CASE_   =      {
    'albert-base-v1': 5_12,
    'albert-large-v1': 5_12,
    'albert-xlarge-v1': 5_12,
    'albert-xxlarge-v1': 5_12,
    'albert-base-v2': 5_12,
    'albert-large-v2': 5_12,
    'albert-xlarge-v2': 5_12,
    'albert-xxlarge-v2': 5_12,
}

SCREAMING_SNAKE_CASE_   =      '▁'


class 			lowerCAmelCase_    (							snake_case__   ):





  """simple docstring"""



  a_							:List[Any]				       =VOCAB_FILES_NAMES
  a_							:str				       =PRETRAINED_VOCAB_FILES_MAP
  a_							:List[str]				       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

  def __init__(       self  :				List[Any]					,				SCREAMING_SNAKE_CASE__  :				Tuple					,				SCREAMING_SNAKE_CASE__  :				Optional[Any]=True					,				SCREAMING_SNAKE_CASE__  :				Optional[int]=True					,				SCREAMING_SNAKE_CASE__  :				Dict=False					,				SCREAMING_SNAKE_CASE__  :				List[Any]="[CLS]"					,				SCREAMING_SNAKE_CASE__  :				Tuple="[SEP]"					,				SCREAMING_SNAKE_CASE__  :				str="<unk>"					,				SCREAMING_SNAKE_CASE__  :				Dict="[SEP]"					,				SCREAMING_SNAKE_CASE__  :				Optional[int]="<pad>"					,				SCREAMING_SNAKE_CASE__  :				List[Any]="[CLS]"					,				SCREAMING_SNAKE_CASE__  :				Tuple="[MASK]"					,				SCREAMING_SNAKE_CASE__  :				Any = None					,				**SCREAMING_SNAKE_CASE__  :				Optional[Any]					,				):





       '''simple docstring'''

       __a   			= (
           AddedToken(SCREAMING_SNAKE_CASE__					,				lstrip=SCREAMING_SNAKE_CASE__					,				rstrip=SCREAMING_SNAKE_CASE__					,				normalized=SCREAMING_SNAKE_CASE__    )
           if isinstance(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__    )
           else mask_token
       )

       __a   			= {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=SCREAMING_SNAKE_CASE__					,				remove_space=SCREAMING_SNAKE_CASE__					,				keep_accents=SCREAMING_SNAKE_CASE__					,				bos_token=SCREAMING_SNAKE_CASE__					,				eos_token=SCREAMING_SNAKE_CASE__					,				unk_token=SCREAMING_SNAKE_CASE__					,				sep_token=SCREAMING_SNAKE_CASE__					,				pad_token=SCREAMING_SNAKE_CASE__					,				cls_token=SCREAMING_SNAKE_CASE__					,				mask_token=SCREAMING_SNAKE_CASE__					,				sp_model_kwargs=self.sp_model_kwargs					,				**SCREAMING_SNAKE_CASE__					,				)

       __a   			= do_lower_case
       __a   			= remove_space
       __a   			= keep_accents
       __a   			= vocab_file

       __a   			= spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(SCREAMING_SNAKE_CASE__    )

  @property
  def __a    (       self  :				List[str]    ):





       '''simple docstring'''

       return len(self.sp_model    )

  def __a    (       self  :				List[str]    ):





       '''simple docstring'''

       __a   			= {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__    ): i for i in range(self.vocab_size    )}
       vocab.update(self.added_tokens_encoder    )
       return vocab

  def __getstate__(       self  :				Dict    ):





       '''simple docstring'''

       __a   			= self.__dict__.copy()
       __a   			= None
       return state

  def __setstate__(       self  :				str					,				SCREAMING_SNAKE_CASE__  :				Optional[int]    ):





       '''simple docstring'''

       __a   			= d

       # for backward compatibility
       if not hasattr(self					,				"""sp_model_kwargs"""    ):
            __a   			= {}

       __a   			= spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(self.vocab_file    )

  def __a    (       self  :				Tuple					,				SCREAMING_SNAKE_CASE__  :				str    ):





       '''simple docstring'''

       if self.remove_space:
            __a   			= """ """.join(inputs.strip().split()    )
       else:
            __a   			= inputs
       __a   			= outputs.replace("""``"""					,				"""\""""    ).replace("""\'\'"""					,				"""\""""    )

       if not self.keep_accents:
            __a   			= unicodedata.normalize("""NFKD"""					,				SCREAMING_SNAKE_CASE__    )
            __a   			= """""".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE__    )]    )
       if self.do_lower_case:
            __a   			= outputs.lower()

       return outputs

  def __a    (       self  :				str					,				SCREAMING_SNAKE_CASE__  :				Any    ):





       '''simple docstring'''

       __a   			= self.preprocess_text(SCREAMING_SNAKE_CASE__    )
       __a   			= self.sp_model.encode(SCREAMING_SNAKE_CASE__					,				out_type=SCREAMING_SNAKE_CASE__    )
       __a   			= []
       for piece in pieces:
            if len(SCREAMING_SNAKE_CASE__    ) > 1 and piece[-1] == str(""","""    ) and piece[-2].isdigit():
                 __a   			= self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE__					,				""""""    )    )
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                      if len(cur_pieces[0]    ) == 1:
                           __a   			= cur_pieces[1:]
                      else:
                           __a   			= cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]    )
                 new_pieces.extend(SCREAMING_SNAKE_CASE__    )
            else:
                 new_pieces.append(SCREAMING_SNAKE_CASE__    )

       return new_pieces

  def __a    (       self  :				int					,				SCREAMING_SNAKE_CASE__  :				Optional[int]    ):





       '''simple docstring'''

       return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE__    )

  def __a    (       self  :				Dict					,				SCREAMING_SNAKE_CASE__  :				Optional[int]    ):





       '''simple docstring'''

       return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE__    )

  def __a    (       self  :				Dict					,				SCREAMING_SNAKE_CASE__  :				int    ):





       '''simple docstring'''

       __a   			= []
       __a   			= """"""
       __a   			= False
       for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                 if not prev_is_special:
                      out_string += " "
                 out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE__    ) + token
                 __a   			= True
                 __a   			= []
            else:
                 current_sub_tokens.append(SCREAMING_SNAKE_CASE__    )
                 __a   			= False
       out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE__    )
       return out_string.strip()

  def __a    (       self  :				str					,				SCREAMING_SNAKE_CASE__  :				Dict					,				SCREAMING_SNAKE_CASE__  :				List[str] = None    ):





       '''simple docstring'''

       __a   			= [self.sep_token_id]
       __a   			= [self.cls_token_id]
       if token_ids_a is None:
            return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep

  def __a    (       self  :				List[str]					,				SCREAMING_SNAKE_CASE__  :				Optional[int]					,				SCREAMING_SNAKE_CASE__  :				str = None					,				SCREAMING_SNAKE_CASE__  :				Union[str, Any] = False    ):





       '''simple docstring'''

       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=SCREAMING_SNAKE_CASE__					,				token_ids_a=SCREAMING_SNAKE_CASE__					,				already_has_special_tokens=SCREAMING_SNAKE_CASE__    )

       if token_ids_a is not None:
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]
       return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]

  def __a    (       self  :				str					,				SCREAMING_SNAKE_CASE__  :				List[Any]					,				SCREAMING_SNAKE_CASE__  :				Optional[Any] = None    ):





       '''simple docstring'''

       __a   			= [self.sep_token_id]
       __a   			= [self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]


  def __a    (       self  :				int					,				SCREAMING_SNAKE_CASE__  :				Optional[Any]					,				SCREAMING_SNAKE_CASE__  :				List[str] = None    ):





       '''simple docstring'''

       if not os.path.isdir(SCREAMING_SNAKE_CASE__    ):
            logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
            return
       __a   			= os.path.join(
           SCREAMING_SNAKE_CASE__					,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(SCREAMING_SNAKE_CASE__    ) and os.path.isfile(self.vocab_file    ):
            copyfile(self.vocab_file					,				SCREAMING_SNAKE_CASE__    )
       elif not os.path.isfile(self.vocab_file    ):
            with open(SCREAMING_SNAKE_CASE__					,				"""wb"""    ) as fi:
                 __a   			= self.sp_model.serialized_model_proto()
                 fi.write(SCREAMING_SNAKE_CASE__    )

       return (out_vocab_file,)











'''simple docstring'''





class __A     :



 '''simple docstring'''







 def __init__(self					,  A			)       ->	None:


      """simple docstring"""



      _a           =    len(A			)
      _a           =    [0] * len_array

      if len_array > 0:
           _a           =    array[0]

      for i in range(1					,  A			):
           _a           =    self.prefix_sum[i - 1] + array[i]



 def    a__    (self					,  A					,  A			)       ->	int:


      """simple docstring"""



      if start == 0:
           return self.prefix_sum[end]

      return self.prefix_sum[end] - self.prefix_sum[start - 1]



 def    a__    (self					,  A			)       ->	bool:


      """simple docstring"""



      _a           =    {0}
      for sum_item in self.prefix_sum:
           if sum_item - target_sum in sums:
                return True

           sums.add(A			)

      return False


if __name__ == "__main__":
  import doctest

  doctest.testmod()



