






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__A					=	logging.get_logger(__name__)

__A					=	{"vocab_file": "spiece.model"}

__A					=	{
    "vocab_file": {
        "TsinghuaAI/CPM-Generate": "https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model",
    }
}



class 				_A			(     UpperCamelCase				):
    """simple docstring"""

    def __init__(				self      :     List[str] ,     __SCREAMING_SNAKE_CASE      :     Optional[Any] ,     __SCREAMING_SNAKE_CASE      :     Dict=False ,     __SCREAMING_SNAKE_CASE      :     List[str]=True ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=False ,     __SCREAMING_SNAKE_CASE      :     Dict="<s>" ,     __SCREAMING_SNAKE_CASE      :     Optional[Any]="</s>" ,     __SCREAMING_SNAKE_CASE      :     str="<unk>" ,     __SCREAMING_SNAKE_CASE      :     str="<sep>" ,     __SCREAMING_SNAKE_CASE      :     Any="<pad>" ,     __SCREAMING_SNAKE_CASE      :     Tuple="<cls>" ,     __SCREAMING_SNAKE_CASE      :     Optional[int]="<mask>" ,     __SCREAMING_SNAKE_CASE      :     Tuple=["<eop>", "<eod>"] ,     __SCREAMING_SNAKE_CASE      :     Optional[Dict[str, Any]] = None ,     **__SCREAMING_SNAKE_CASE      :     Optional[int] ,     )      ->							None:
           __UpperCAmelCase     				=AddedToken(__SCREAMING_SNAKE_CASE ,     lstrip=__SCREAMING_SNAKE_CASE ,     rstrip=__SCREAMING_SNAKE_CASE      ) if isinstance(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      ) else mask_token

           __UpperCAmelCase     				={} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=__SCREAMING_SNAKE_CASE ,     remove_space=__SCREAMING_SNAKE_CASE ,     keep_accents=__SCREAMING_SNAKE_CASE ,     bos_token=__SCREAMING_SNAKE_CASE ,     eos_token=__SCREAMING_SNAKE_CASE ,     unk_token=__SCREAMING_SNAKE_CASE ,     sep_token=__SCREAMING_SNAKE_CASE ,     pad_token=__SCREAMING_SNAKE_CASE ,     cls_token=__SCREAMING_SNAKE_CASE ,     mask_token=__SCREAMING_SNAKE_CASE ,     additional_special_tokens=__SCREAMING_SNAKE_CASE ,     sp_model_kwargs=self.sp_model_kwargs ,     **__SCREAMING_SNAKE_CASE ,     )

           __UpperCAmelCase     				=3

           __UpperCAmelCase     				=do_lower_case
           __UpperCAmelCase     				=remove_space
           __UpperCAmelCase     				=keep_accents
           __UpperCAmelCase     				=vocab_file

           __UpperCAmelCase     				=spm.SentencePieceProcessor(**self.sp_model_kwargs      )
           self.sp_model.Load(__SCREAMING_SNAKE_CASE      )

           try:
                  import jieba
           except ModuleNotFoundError as error:
                  raise error.__class__(
                      """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
                      """See https://pypi.org/project/jieba/ for installation."""      )
           __UpperCAmelCase     				=jieba
           __UpperCAmelCase     				=str.maketrans(""" \n""" ,     """\u2582\u2583"""      )



    @property
    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
    def _a							(				self      :     Optional[Any]      )      ->							List[Any]:
           return len(self.sp_model      )



    def _a							(				self      :     Optional[int]      )      ->							List[Any]:
           __UpperCAmelCase     				={self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE      ): i for i in range(self.vocab_size      )}
           vocab.update(self.added_tokens_encoder      )
           return vocab



    def __getstate__(				self      :     str      )      ->							List[str]:
           __UpperCAmelCase     				=self.__dict__.copy()
           __UpperCAmelCase     				=None
           return state



    def __setstate__(				self      :     str ,     __SCREAMING_SNAKE_CASE      :     Tuple      )      ->							str:
           __UpperCAmelCase     				=d

           # for backward compatibility
           if not hasattr(self ,     """sp_model_kwargs"""      ):
                  __UpperCAmelCase     				={}

           __UpperCAmelCase     				=spm.SentencePieceProcessor(**self.sp_model_kwargs      )
           self.sp_model.Load(self.vocab_file      )



    def _a							(				self      :     Optional[int] ,     __SCREAMING_SNAKE_CASE      :     Any      )      ->							Tuple:
           if self.remove_space:
                  __UpperCAmelCase     				=""" """.join(inputs.strip().split()      )
           else:
                  __UpperCAmelCase     				=inputs
           __UpperCAmelCase     				=outputs.replace("""``""" ,     """\""""      ).replace("""''""" ,     """\""""      )

           if not self.keep_accents:
                  __UpperCAmelCase     				=unicodedata.normalize("""NFKD""" ,     __SCREAMING_SNAKE_CASE      )
                  __UpperCAmelCase     				="""""".join([c for c in outputs if not unicodedata.combining(__SCREAMING_SNAKE_CASE      )]      )
           if self.do_lower_case:
                  __UpperCAmelCase     				=outputs.lower()

           return outputs



    def _a							(				self      :     List[str] ,     __SCREAMING_SNAKE_CASE      :     str      )      ->							List[str]:
           __UpperCAmelCase     				=self.preprocess_text(__SCREAMING_SNAKE_CASE      )
           __UpperCAmelCase     				=self.sp_model.encode(__SCREAMING_SNAKE_CASE ,     out_type=__SCREAMING_SNAKE_CASE      )
           __UpperCAmelCase     				=[]
           for piece in pieces:
                  if len(__SCREAMING_SNAKE_CASE      ) > 1 and piece[-1] == str(""","""      ) and piece[-2].isdigit():
                         __UpperCAmelCase     				=self.sp_model.EncodeAsPieces(piece[:-1].replace(__SCREAMING_SNAKE_CASE ,     """"""      )      )
                         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                if len(cur_pieces[0]      ) == 1:
                                       __UpperCAmelCase     				=cur_pieces[1:]
                                else:
                                       __UpperCAmelCase     				=cur_pieces[0][1:]
                         cur_pieces.append(piece[-1]      )
                         new_pieces.extend(__SCREAMING_SNAKE_CASE      )
                  else:
                         new_pieces.append(__SCREAMING_SNAKE_CASE      )

           return new_pieces



    def _a							(				self      :     Optional[Any] ,     __SCREAMING_SNAKE_CASE      :     Optional[Any]      )      ->							str:
           return self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE      )



    def _a							(				self      :     str ,     __SCREAMING_SNAKE_CASE      :     int      )      ->							Any:
           return self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE      )



    def _a							(				self      :     Optional[Any] ,     __SCREAMING_SNAKE_CASE      :     Tuple      )      ->							List[Any]:
           __UpperCAmelCase     				="""""".join(__SCREAMING_SNAKE_CASE      ).replace(__SCREAMING_SNAKE_CASE ,     """ """      ).strip()
           return out_string



    def _a							(				self      :     Union[str, Any] ,     __SCREAMING_SNAKE_CASE      :     List[int] ,     __SCREAMING_SNAKE_CASE      :     Optional[List[int]] = None      )      ->							List[int]:
           __UpperCAmelCase     				=[self.sep_token_id]
           __UpperCAmelCase     				=[self.cls_token_id]
           if token_ids_a is None:
                  return token_ids_a + sep + cls
           return token_ids_a + sep + token_ids_a + sep + cls



    def _a							(				self      :     str ,     __SCREAMING_SNAKE_CASE      :     List[int] ,     __SCREAMING_SNAKE_CASE      :     Optional[List[int]] = None ,     __SCREAMING_SNAKE_CASE      :     bool = False      )      ->							List[int]:

           if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=__SCREAMING_SNAKE_CASE ,     token_ids_a=__SCREAMING_SNAKE_CASE ,     already_has_special_tokens=__SCREAMING_SNAKE_CASE      )

           if token_ids_a is not None:
                  return ([0] * len(__SCREAMING_SNAKE_CASE      )) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE      )) + [1, 1]
           return ([0] * len(__SCREAMING_SNAKE_CASE      )) + [1, 1]



    def _a							(				self      :     Optional[int] ,     __SCREAMING_SNAKE_CASE      :     List[int] ,     __SCREAMING_SNAKE_CASE      :     Optional[List[int]] = None      )      ->							List[int]:
           __UpperCAmelCase     				=[self.sep_token_id]
           __UpperCAmelCase     				=[2]

           if token_ids_a is None:
                  return len(token_ids_a + sep      ) * [0] + cls_segment_id
           return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1] + cls_segment_id



    def _a							(				self      :     List[str] ,     __SCREAMING_SNAKE_CASE      :     str ,     __SCREAMING_SNAKE_CASE      :     Optional[str] = None      )      ->							Tuple[str]:
           if not os.path.isdir(__SCREAMING_SNAKE_CASE      ):
                  logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
                  return
           __UpperCAmelCase     				=os.path.join(
               __SCREAMING_SNAKE_CASE ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

           if os.path.abspath(self.vocab_file      ) != os.path.abspath(__SCREAMING_SNAKE_CASE      ) and os.path.isfile(self.vocab_file      ):
                  copyfile(self.vocab_file ,     __SCREAMING_SNAKE_CASE      )
           elif not os.path.isfile(self.vocab_file      ):
                  with open(__SCREAMING_SNAKE_CASE ,     """wb"""      ) as fi:
                         __UpperCAmelCase     				=self.sp_model.serialized_model_proto()
                         fi.write(__SCREAMING_SNAKE_CASE      )

           return (out_vocab_file,)



    def _a							(				self      :     List[Any] ,     *__SCREAMING_SNAKE_CASE      :     Any ,     **__SCREAMING_SNAKE_CASE      :     Dict      )      ->							Dict:
           __UpperCAmelCase     				=super()._decode(*__SCREAMING_SNAKE_CASE ,     **__SCREAMING_SNAKE_CASE      )
           __UpperCAmelCase     				=text.replace(""" """ ,     """"""      ).replace("""\u2582""" ,     """ """      ).replace("""\u2583""" ,     """\n"""      )
           return text






# Imports
import numpy as np





class   UpperCamelCase_      :







						'''simple docstring'''



						def __init__(			self							:Optional[int]		,    lowerCAmelCase__							:int=None		,    lowerCAmelCase__							:List[str]=None		,    lowerCAmelCase__							:int=None		,    lowerCAmelCase__							:Optional[int]=None		,    lowerCAmelCase__							:List[str]=None     )  ->int:
										self.set_matricies(red=lowerCAmelCase__		,    green=lowerCAmelCase__		,    blue=lowerCAmelCase__		,    red_edge=lowerCAmelCase__		,    nir=lowerCAmelCase__     )





						def        SCREAMING_SNAKE_CASE(			self							:Optional[Any]		,    lowerCAmelCase__							:List[Any]=None		,    lowerCAmelCase__							:Any=None		,    lowerCAmelCase__							:Tuple=None		,    lowerCAmelCase__							:Optional[Any]=None		,    lowerCAmelCase__							:Optional[Any]=None     )  ->List[Any]:
										if red is not None:
														lowercase         =      red
										if green is not None:
														lowercase         =      green
										if blue is not None:
														lowercase         =      blue
										if red_edge is not None:
														lowercase         =      red_edge
										if nir is not None:
														lowercase         =      nir
										return True





						def        SCREAMING_SNAKE_CASE(			self							:Optional[Any]		,    lowerCAmelCase__							:Dict=""		,    lowerCAmelCase__							:Optional[Any]=None		,    lowerCAmelCase__							:Union[str, Any]=None		,    lowerCAmelCase__							:int=None		,    lowerCAmelCase__							:int=None		,    lowerCAmelCase__							:Union[str, Any]=None     )  ->Optional[Any]:
										self.set_matricies(red=lowerCAmelCase__		,    green=lowerCAmelCase__		,    blue=lowerCAmelCase__		,    red_edge=lowerCAmelCase__		,    nir=lowerCAmelCase__     )
										lowercase         =      {
										    "ARVI2": self.arvaa,
										    "CCCI": self.ccci,
										    "CVI": self.cvi,
										    "GLI": self.gli,
										    "NDVI": self.ndvi,
										    "BNDVI": self.bndvi,
										    "redEdgeNDVI": self.red_edge_ndvi,
										    "GNDVI": self.gndvi,
										    "GBNDVI": self.gbndvi,
										    "GRNDVI": self.grndvi,
										    "RBNDVI": self.rbndvi,
										    "PNDVI": self.pndvi,
										    "ATSAVI": self.atsavi,
										    "BWDRVI": self.bwdrvi,
										    "CIgreen": self.ci_green,
										    "CIrededge": self.ci_rededge,
										    "CI": self.ci,
										    "CTVI": self.ctvi,
										    "GDVI": self.gdvi,
										    "EVI": self.evi,
										    "GEMI": self.gemi,
										    "GOSAVI": self.gosavi,
										    "GSAVI": self.gsavi,
										    "Hue": self.hue,
										    "IVI": self.ivi,
										    "IPVI": self.ipvi,
										    "I": self.i,
										    "RVI": self.rvi,
										    "MRVI": self.mrvi,
										    "MSAVI": self.m_savi,
										    "NormG": self.norm_g,
										    "NormNIR": self.norm_nir,
										    "NormR": self.norm_r,
										    "NGRDI": self.ngrdi,
										    "RI": self.ri,
										    "S": self.s,
										    "IF": self._if,
										    "DVI": self.dvi,
										    "TVI": self.tvi,
										    "NDRE": self.ndre,
										}

										try:
														return funcs[index]()
										except KeyError:
														print("Index not in the list!"     )
														return False





						def        SCREAMING_SNAKE_CASE(			self							:Any     )  ->int:
										return -0.18 + (1.17 * ((self.nir - self.red) / (self.nir + self.red)))





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->int:
										return ((self.nir - self.redEdge) / (self.nir + self.redEdge)) / (
										    (self.nir - self.red) / (self.nir + self.red)
										)





						def        SCREAMING_SNAKE_CASE(			self							:int     )  ->Tuple:
										return self.nir * (self.red / (self.green**2))





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->Dict:
										return (2 * self.green - self.red - self.blue) / (
										    2 * self.green + self.red + self.blue
										)





						def        SCREAMING_SNAKE_CASE(			self							:str     )  ->Optional[int]:
										return (self.nir - self.red) / (self.nir + self.red)





						def        SCREAMING_SNAKE_CASE(			self							:str     )  ->int:
										return (self.nir - self.blue) / (self.nir + self.blue)





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->List[str]:
										return (self.redEdge - self.red) / (self.redEdge + self.red)





						def        SCREAMING_SNAKE_CASE(			self							:Optional[int]     )  ->str:
										return (self.nir - self.green) / (self.nir + self.green)





						def        SCREAMING_SNAKE_CASE(			self							:int     )  ->Any:
										return (self.nir - (self.green + self.blue)) / (
										    self.nir + (self.green + self.blue)
										)





						def        SCREAMING_SNAKE_CASE(			self							:Union[str, Any]     )  ->List[Any]:
										return (self.nir - (self.green + self.red)) / (
										    self.nir + (self.green + self.red)
										)





						def        SCREAMING_SNAKE_CASE(			self							:Optional[Any]     )  ->Union[str, Any]:
										return (self.nir - (self.blue + self.red)) / (self.nir + (self.blue + self.red))





						def        SCREAMING_SNAKE_CASE(			self							:Any     )  ->Union[str, Any]:
										return (self.nir - (self.green + self.red + self.blue)) / (
										    self.nir + (self.green + self.red + self.blue)
										)





						def        SCREAMING_SNAKE_CASE(			self							:Tuple		,    lowerCAmelCase__							:Union[str, Any]=0.08		,    lowerCAmelCase__							:Dict=1.22		,    lowerCAmelCase__							:Tuple=0.03     )  ->Tuple:
										return a * (
										    (self.nir - a * self.red - b)
										    / (a * self.nir + self.red - a * b + x * (1 + a**2))
										)





						def        SCREAMING_SNAKE_CASE(			self							:Dict     )  ->Any:
										return (0.1 * self.nir - self.blue) / (0.1 * self.nir + self.blue)





						def        SCREAMING_SNAKE_CASE(			self							:Optional[Any]     )  ->int:
										return (self.nir / self.green) - 1





						def        SCREAMING_SNAKE_CASE(			self							:int     )  ->List[str]:
										return (self.nir / self.redEdge) - 1





						def        SCREAMING_SNAKE_CASE(			self							:Union[str, Any]     )  ->List[Any]:
										return (self.red - self.blue) / self.red





						def        SCREAMING_SNAKE_CASE(			self							:Any     )  ->int:
										lowercase         =      self.ndvi()
										return ((ndvi + 0.5) / (abs(ndvi + 0.5     ))) * (abs(ndvi + 0.5     ) ** (1 / 2))





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->Optional[Any]:
										return self.nir - self.green





						def        SCREAMING_SNAKE_CASE(			self							:str     )  ->int:
										return 2.5 * (
										    (self.nir - self.red) / (self.nir + 6 * self.red - 7.5 * self.blue + 1)
										)





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->Optional[Any]:
										lowercase         =      (2 * (self.nir**2 - self.red**2) + 1.5 * self.nir + 0.5 * self.red) / (
										    self.nir + self.red + 0.5
										)
										return n * (1 - 0.25 * n) - (self.red - 0.1_25) / (1 - self.red)





						def        SCREAMING_SNAKE_CASE(			self							:Union[str, Any]		,    lowerCAmelCase__							:Tuple=0.16     )  ->Any:
										return (self.nir - self.green) / (self.nir + self.green + y)





						def        SCREAMING_SNAKE_CASE(			self							:Dict		,    lowerCAmelCase__							:Any=0.5     )  ->str:
										return ((self.nir - self.green) / (self.nir + self.green + n)) * (1 + n)





						def        SCREAMING_SNAKE_CASE(			self							:int     )  ->List[str]:
										return np.arctan(
										    ((2 * self.red - self.green - self.blue) / 30.5) * (self.green - self.blue)     )





						def        SCREAMING_SNAKE_CASE(			self							:Any		,    lowerCAmelCase__							:str=None		,    lowerCAmelCase__							:Tuple=None     )  ->Dict:
										return (self.nir - b) / (a * self.red)





						def        SCREAMING_SNAKE_CASE(			self							:Optional[Any]     )  ->Dict:
										return (self.nir / ((self.nir + self.red) / 2)) * (self.ndvi() + 1)





						def        SCREAMING_SNAKE_CASE(			self							:str     )  ->List[Any]:
										return (self.red + self.green + self.blue) / 30.5





						def        SCREAMING_SNAKE_CASE(			self							:Tuple     )  ->Optional[Any]:
										return self.nir / self.red





						def        SCREAMING_SNAKE_CASE(			self							:Optional[int]     )  ->Dict:
										return (self.rvi() - 1) / (self.rvi() + 1)





						def        SCREAMING_SNAKE_CASE(			self							:Optional[Any]     )  ->Dict:
										return (
										    (2 * self.nir + 1)
										    - ((2 * self.nir + 1) ** 2 - 8 * (self.nir - self.red)) ** (1 / 2)
										) / 2





						def        SCREAMING_SNAKE_CASE(			self							:str     )  ->int:
										return self.green / (self.nir + self.red + self.green)





						def        SCREAMING_SNAKE_CASE(			self							:Any     )  ->Any:
										return self.nir / (self.nir + self.red + self.green)





						def        SCREAMING_SNAKE_CASE(			self							:int     )  ->Tuple:
										return self.red / (self.nir + self.red + self.green)





						def        SCREAMING_SNAKE_CASE(			self							:List[Any]     )  ->List[Any]:
										return (self.green - self.red) / (self.green + self.red)





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->str:
										return (self.red - self.green) / (self.red + self.green)





						def        SCREAMING_SNAKE_CASE(			self							:Union[str, Any]     )  ->List[str]:
										lowercase         =      np.max([np.max(self.red     ), np.max(self.green     ), np.max(self.blue     )]     )
										lowercase         =      np.min([np.min(self.red     ), np.min(self.green     ), np.min(self.blue     )]     )
										return (max_value - min_value) / max_value





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->Any:
										return (2 * self.red - self.green - self.blue) / (self.green - self.blue)





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->List[Any]:
										return self.nir / self.red





						def        SCREAMING_SNAKE_CASE(			self							:Any     )  ->List[str]:
										return (self.ndvi() + 0.5) ** (1 / 2)





						def        SCREAMING_SNAKE_CASE(			self							:Optional[int]     )  ->Union[str, Any]:
										return (self.nir - self.redEdge) / (self.nir + self.redEdge)
