

"""simple docstring"""




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase			=					logging.get_logger(__name__)
__UpperCAmelCase			=					{'vocab_file': 'spiece.model'}

__UpperCAmelCase			=					{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__UpperCAmelCase			=					{
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

__UpperCAmelCase			=					'▁'







class   __lowercase      (     __lowerCamelCase       ):
  snake_case_												=							VOCAB_FILES_NAMES
  snake_case_												=							PRETRAINED_VOCAB_FILES_MAP
  snake_case_												=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

  def __init__(	self		:      Union[str, Any]       ,A		:      Dict       ,A		:      Union[str, Any]=True       ,A		:      Optional[Any]=True       ,A		:      Any=False       ,A		:      Optional[Any]="[CLS]"       ,A		:      Union[str, Any]="[SEP]"       ,A		:      Dict="<unk>"       ,A		:      Optional[Any]="[SEP]"       ,A		:      Union[str, Any]="<pad>"       ,A		:      int="[CLS]"       ,A		:      Optional[int]="[MASK]"       ,A		:      Optional[Dict[str, Any]] = None       ,**A		:      Any       ,):






   '''simple docstring'''
   # Mask token behave like a normal word, i.e. include the space before it and
   # is included in the raw text, there should be a match in a non-normalized sentence.
   UpperCAmelCase__       :	List[Any]        =       (
       AddedToken(A       ,lstrip=A       ,rstrip=A       ,normalized=A     )
       if isinstance(A       ,A     )
       else mask_token
   )

   UpperCAmelCase__       :	List[Any]        =       {} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=A       ,remove_space=A       ,keep_accents=A       ,bos_token=A       ,eos_token=A       ,unk_token=A       ,sep_token=A       ,pad_token=A       ,cls_token=A       ,mask_token=A       ,sp_model_kwargs=self.sp_model_kwargs       ,**A       ,)

   UpperCAmelCase__       :	List[Any]        =       do_lower_case
   UpperCAmelCase__       :	List[Any]        =       remove_space
   UpperCAmelCase__       :	str        =       keep_accents
   UpperCAmelCase__       :	List[Any]        =       vocab_file

   UpperCAmelCase__       :	List[Any]        =       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(A     )

  @property
  def  __lowercase							(	self		:      Optional[int]     ):






   '''simple docstring'''
   return len(self.sp_model     )

  def  __lowercase							(	self		:      List[str]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[Any]        =       {self.convert_ids_to_tokens(A     ): i for i in range(self.vocab_size     )}
   vocab.update(self.added_tokens_encoder     )
   return vocab

  def __getstate__(	self		:      List[str]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       self.__dict__.copy()
   UpperCAmelCase__       :	Any        =       None
   return state

  def __setstate__(	self		:      List[Any]       ,A		:      Tuple     ):






   '''simple docstring'''
   UpperCAmelCase__       :	str        =       d

   # for backward compatibility
   if not hasattr(self       ,"""sp_model_kwargs"""     ):
    UpperCAmelCase__       :	List[str]        =       {}

   UpperCAmelCase__       :	str        =       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(self.vocab_file     )

  def  __lowercase							(	self		:      Any       ,A		:      List[str]     ):






   '''simple docstring'''
   if self.remove_space:
    UpperCAmelCase__       :	Any        =       """ """.join(inputs.strip().split()     )
   else:
    UpperCAmelCase__       :	Union[str, Any]        =       inputs
   UpperCAmelCase__       :	Optional[int]        =       outputs.replace("""``"""       ,"""\""""     ).replace("""''"""       ,"""\""""     )

   if not self.keep_accents:
    UpperCAmelCase__       :	Optional[Any]        =       unicodedata.normalize("""NFKD"""       ,A     )
    UpperCAmelCase__       :	Optional[Any]        =       """""".join([c for c in outputs if not unicodedata.combining(A     )]     )
   if self.do_lower_case:
    UpperCAmelCase__       :	int        =       outputs.lower()

   return outputs

  def  __lowercase							(	self		:      Union[str, Any]       ,A		:      str     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Any        =       self.preprocess_text(A     )
   UpperCAmelCase__       :	str        =       self.sp_model.encode(A       ,out_type=A     )
   UpperCAmelCase__       :	List[Any]        =       []
   for piece in pieces:
    if len(A     ) > 1 and piece[-1] == str(""","""     ) and piece[-2].isdigit():
     UpperCAmelCase__       :	Any        =       self.sp_model.EncodeAsPieces(piece[:-1].replace(A       ,""""""     )     )
     if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
      if len(cur_pieces[0]     ) == 1:
       UpperCAmelCase__       :	Union[str, Any]        =       cur_pieces[1:]
      else:
       UpperCAmelCase__       :	int        =       cur_pieces[0][1:]
     cur_pieces.append(piece[-1]     )
     new_pieces.extend(A     )
    else:
     new_pieces.append(A     )

   return new_pieces

  def  __lowercase							(	self		:      Union[str, Any]       ,A		:      int     ):






   '''simple docstring'''
   return self.sp_model.PieceToId(A     )

  def  __lowercase							(	self		:      Optional[int]       ,A		:      str     ):






   '''simple docstring'''
   return self.sp_model.IdToPiece(A     )

  def  __lowercase							(	self		:      Tuple       ,A		:      str     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       []
   UpperCAmelCase__       :	Any        =       """"""
   UpperCAmelCase__       :	Optional[Any]        =       False
   for token in tokens:
    # make sure that special tokens are not decoded using sentencepiece model
    if token in self.all_special_tokens:
     if not prev_is_special:
      out_string += " "
     out_string += self.sp_model.decode(A     ) + token
     UpperCAmelCase__       :	Any        =       True
     UpperCAmelCase__       :	str        =       []
    else:
     current_sub_tokens.append(A     )
     UpperCAmelCase__       :	str        =       False
   out_string += self.sp_model.decode(A     )
   return out_string.strip()

  def  __lowercase							(	self		:      Any       ,A		:      List[int]       ,A		:      Optional[List[int]] = None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       [self.sep_token_id]
   UpperCAmelCase__       :	Optional[Any]        =       [self.cls_token_id]
   if token_ids_a is None:
    return cls + token_ids_a + sep
   return cls + token_ids_a + sep + token_ids_a + sep

  def  __lowercase							(	self		:      str       ,A		:      List[int]       ,A		:      Optional[List[int]] = None       ,A		:      bool = False     ):






   '''simple docstring'''

   if already_has_special_tokens:
    return super().get_special_tokens_mask(
        token_ids_a=A       ,token_ids_a=A       ,already_has_special_tokens=A     )

   if token_ids_a is not None:
    return [1] + ([0] * len(A     )) + [1] + ([0] * len(A     )) + [1]
   return [1] + ([0] * len(A     )) + [1]

  def  __lowercase							(	self		:      Tuple       ,A		:      List[int]       ,A		:      Optional[List[int]] = None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[Any]        =       [self.sep_token_id]
   UpperCAmelCase__       :	Tuple        =       [self.cls_token_id]

   if token_ids_a is None:
    return len(cls + token_ids_a + sep     ) * [0]
   return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]




  def  __lowercase							(	self		:      Union[str, Any]       ,A		:      str       ,A		:      Optional[str] = None     ):






   '''simple docstring'''
   if not os.path.isdir(A     ):
    logger.error(f"Vocabulary path ({save_directory}) should be a directory"     )
    return
   UpperCAmelCase__       :	List[str]        =       os.path.join(
       A       ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

   if os.path.abspath(self.vocab_file     ) != os.path.abspath(A     ) and os.path.isfile(self.vocab_file     ):
    copyfile(self.vocab_file       ,A     )
   elif not os.path.isfile(self.vocab_file     ):
    with open(A       ,"""wb"""     ) as fi:
     UpperCAmelCase__       :	int        =       self.sp_model.serialized_model_proto()
     fi.write(A     )

   return (out_vocab_file,)









'''simple docstring'''


import os


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_doctest_list.py
UpperCAmelCase_						:  List[Any]				  =						'.'


if __name__ == "__main__":
				UpperCAmelCase_						:  Any				  =						os.path.join(REPO_PATH, 'utils/documentation_tests.txt')
				UpperCAmelCase_						:  Dict				  =						[]
				UpperCAmelCase_						:  Union[str, Any]				  =						[]
				with open(doctest_file_path) as fp:
								for line in fp:
												UpperCAmelCase_						:  int				  =						line.strip()
												UpperCAmelCase_						:  str				  =						os.path.join(REPO_PATH, line)
												if not (os.path.isfile(path) or os.path.isdir(path)):
																non_existent_paths.append(line)
												all_paths.append(path)
				if len(non_existent_paths) > 0:
								UpperCAmelCase_						:  int				  =						'\n'.join(non_existent_paths)
								raise ValueError(F"`utils/documentation_tests.txt` contains non-existent paths:\n{non_existent_paths}")
				if all_paths != sorted(all_paths):
								raise ValueError('Files in `utils/documentation_tests.txt` are not in alphabetical order.')




