

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase     						=      logging.get_logger(__name__)
_UpperCAmelCase     						=      {"""vocab_file""": """spiece.model"""}

_UpperCAmelCase     						=      {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

_UpperCAmelCase     						=      {
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

_UpperCAmelCase     						=      """▁"""



class 							_UpperCAmelCase			( _A      ):
  '''simple docstring'''







  SCREAMING_SNAKE_CASE				:					Dict          =					VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE				:					str          =					PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE				:					Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







  def __init__(			self						:			Dict		,    UpperCamelCase__						:			Union[str, Any]		,    UpperCamelCase__						:			Dict=True		,    UpperCamelCase__						:			int=True		,    UpperCamelCase__						:			Optional[Any]=False		,    UpperCamelCase__						:			Dict="[CLS]"		,    UpperCamelCase__						:			Optional[int]="[SEP]"		,    UpperCamelCase__						:			Any="<unk>"		,    UpperCamelCase__						:			Tuple="[SEP]"		,    UpperCamelCase__						:			Optional[int]="<pad>"		,    UpperCamelCase__						:			Union[str, Any]="[CLS]"		,    UpperCamelCase__						:			Any="[MASK]"		,    UpperCamelCase__						:			Optional[Dict[str, Any]] = None		,    **UpperCamelCase__						:			List[Any]		,    ):
        # Mask token behave like a normal word, i.e. include the space before it and
        # is included in the raw text, there should be a match in a non-normalized sentence.
        A 			=					(
            AddedToken(snake_case_		,    lstrip=snake_case_		,    rstrip=snake_case_		,    normalized=snake_case_			)
            if isinstance(snake_case_		,    snake_case_			)
            else mask_token
        )

        A 			=					{} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            do_lower_case=snake_case_		,    remove_space=snake_case_		,    keep_accents=snake_case_		,    bos_token=snake_case_		,    eos_token=snake_case_		,    unk_token=snake_case_		,    sep_token=snake_case_		,    pad_token=snake_case_		,    cls_token=snake_case_		,    mask_token=snake_case_		,    sp_model_kwargs=self.sp_model_kwargs		,    **snake_case_		,    )

        A 			=					do_lower_case
        A 			=					remove_space
        A 			=					keep_accents
        A 			=					vocab_file

        A 			=					spm.SentencePieceProcessor(**self.sp_model_kwargs			)
        self.sp_model.Load(snake_case_			)







  @property
  def 						UpperCamelCase							(			self						:			Union[str, Any]			):
        return len(self.sp_model			)







  def 						UpperCamelCase							(			self						:			int			):
        A 			=					{self.convert_ids_to_tokens(snake_case_			): i for i in range(self.vocab_size			)}
        vocab.update(self.added_tokens_encoder			)
        return vocab







  def __getstate__(			self						:			Union[str, Any]			):
        A 			=					self.__dict__.copy()
        A 			=					None
        return state







  def __setstate__(			self						:			Dict		,    UpperCamelCase__						:			int			):
        A 			=					d

        # for backward compatibility
        if not hasattr(self		,    'sp_model_kwargs'			):
              A 			=					{}

        A 			=					spm.SentencePieceProcessor(**self.sp_model_kwargs			)
        self.sp_model.Load(self.vocab_file			)







  def 						UpperCamelCase							(			self						:			Union[str, Any]		,    UpperCamelCase__						:			Union[str, Any]			):
        if self.remove_space:
              A 			=					""" """.join(inputs.strip().split()			)
        else:
              A 			=					inputs
        A 			=					outputs.replace('``'		,    '\"'			).replace('\'\''		,    '\"'			)

        if not self.keep_accents:
              A 			=					unicodedata.normalize('NFKD'		,    snake_case_			)
              A 			=					"""""".join([c for c in outputs if not unicodedata.combining(snake_case_			)]			)
        if self.do_lower_case:
              A 			=					outputs.lower()

        return outputs







  def 						UpperCamelCase							(			self						:			str		,    UpperCamelCase__						:			str			):
        A 			=					self.preprocess_text(snake_case_			)
        A 			=					self.sp_model.encode(snake_case_		,    out_type=snake_case_			)
        A 			=					[]
        for piece in pieces:
              if len(snake_case_			) > 1 and piece[-1] == str(','			) and piece[-2].isdigit():
                    A 			=					self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case_		,    ''			)			)
                    if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                          if len(cur_pieces[0]			) == 1:
                                A 			=					cur_pieces[1:]
                          else:
                                A 			=					cur_pieces[0][1:]
                    cur_pieces.append(piece[-1]			)
                    new_pieces.extend(snake_case_			)
              else:
                    new_pieces.append(snake_case_			)

        return new_pieces







  def 						UpperCamelCase							(			self						:			Tuple		,    UpperCamelCase__						:			Dict			):
        return self.sp_model.PieceToId(snake_case_			)







  def 						UpperCamelCase							(			self						:			Tuple		,    UpperCamelCase__						:			Any			):
        return self.sp_model.IdToPiece(snake_case_			)







  def 						UpperCamelCase							(			self						:			Tuple		,    UpperCamelCase__						:			Union[str, Any]			):
        A 			=					[]
        A 			=					""""""
        A 			=					False
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    if not prev_is_special:
                          out_string += " "
                    out_string += self.sp_model.decode(snake_case_			) + token
                    A 			=					True
                    A 			=					[]
              else:
                    current_sub_tokens.append(snake_case_			)
                    A 			=					False
        out_string += self.sp_model.decode(snake_case_			)
        return out_string.strip()







  def 						UpperCamelCase							(			self						:			Optional[int]		,    UpperCamelCase__						:			List[int]		,    UpperCamelCase__						:			Optional[List[int]] = None			):
        A 			=					[self.sep_token_id]
        A 			=					[self.cls_token_id]
        if token_ids_a is None:
              return cls + token_ids_a + sep
        return cls + token_ids_a + sep + token_ids_a + sep







  def 						UpperCamelCase							(			self						:			List[str]		,    UpperCamelCase__						:			List[int]		,    UpperCamelCase__						:			Optional[List[int]] = None		,    UpperCamelCase__						:			bool = False			):

        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=snake_case_		,    token_ids_a=snake_case_		,    already_has_special_tokens=snake_case_			)

        if token_ids_a is not None:
              return [1] + ([0] * len(snake_case_			)) + [1] + ([0] * len(snake_case_			)) + [1]
        return [1] + ([0] * len(snake_case_			)) + [1]







  def 						UpperCamelCase							(			self						:			int		,    UpperCamelCase__						:			List[int]		,    UpperCamelCase__						:			Optional[List[int]] = None			):
        A 			=					[self.sep_token_id]
        A 			=					[self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep			) * [0]
        return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







  def 						UpperCamelCase							(			self						:			Optional[int]		,    UpperCamelCase__						:			str		,    UpperCamelCase__						:			Optional[str] = None			):
        if not os.path.isdir(snake_case_			):
              logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''			)
              return
        A 			=					os.path.join(
            snake_case_		,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

        if os.path.abspath(self.vocab_file			) != os.path.abspath(snake_case_			) and os.path.isfile(self.vocab_file			):
              copyfile(self.vocab_file		,    snake_case_			)
        elif not os.path.isfile(self.vocab_file			):
              with open(snake_case_		,    'wb'			) as fi:
                    A 			=					self.sp_model.serialized_model_proto()
                    fi.write(snake_case_			)

        return (out_vocab_file,)





lowerCamelCase_    :			Tuple    =	{
    """meter""": """m""",
    """kilometer""": """km""",
    """megametre""": """Mm""",
    """gigametre""": """Gm""",
    """terametre""": """Tm""",
    """petametre""": """Pm""",
    """exametre""": """Em""",
    """zettametre""": """Zm""",
    """yottametre""": """Ym""",
}
# Exponent of the factor(meter)
lowerCamelCase_    :			Union[str, Any]    =	{
    """m""": 0,
    """km""": 3,
    """Mm""": 6,
    """Gm""": 9,
    """Tm""": 12,
    """Pm""": 15,
    """Em""": 18,
    """Zm""": 21,
    """Ym""": 24,
}







def 							A__       (    lowerCamelCase  ,     lowerCamelCase  ,     lowerCamelCase    )	->						float:
				UpperCamelCase_:				Any						=    from_type.lower().strip("""s"""    )
				UpperCamelCase_:				int						=    to_type.lower().strip("""s"""    )

				UpperCamelCase_:				Any						=    UNIT_SYMBOL.get(lowerCamelCase  ,     lowerCamelCase    )
				UpperCamelCase_:				str						=    UNIT_SYMBOL.get(lowerCamelCase  ,     lowerCamelCase    )

				if from_sanitized not in METRIC_CONVERSION:
								UpperCamelCase_:				Optional[int]						=    (
								    F'''Invalid \'from_type\' value: {from_type!r}.\n'''
								    F'''Conversion abbreviations are: {", ".join(lowerCamelCase    )}'''
								)
								raise ValueError(lowerCamelCase    )
				if to_sanitized not in METRIC_CONVERSION:
								UpperCamelCase_:				Dict						=    (
								    F'''Invalid \'to_type\' value: {to_type!r}.\n'''
								    F'''Conversion abbreviations are: {", ".join(lowerCamelCase    )}'''
								)
								raise ValueError(lowerCamelCase    )
				UpperCamelCase_:				Union[str, Any]						=    METRIC_CONVERSION[from_sanitized]
				UpperCamelCase_:				str						=    METRIC_CONVERSION[to_sanitized]
				UpperCamelCase_:				Tuple						=    1

				if from_exponent > to_exponent:
								UpperCamelCase_:				Union[str, Any]						=    from_exponent - to_exponent
				else:
								UpperCamelCase_:				Dict						=    -(to_exponent - from_exponent)

				return value * pow(10  ,     lowerCamelCase    )


if __name__ == "__main__":
							from doctest import testmod

							testmod()






