
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_A						      =				logging.get_logger(__name__)

_A						      =				{"vocab_file": "spiece.model"}

_A						      =				{
    "vocab_file": {
        "TsinghuaAI/CPM-Generate": "https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model",
    }
}







class   lowerCamelCase	(						A_						):





     def __init__(self   :					Optional[Any]       ,					_A   :					List[Any]       ,					_A   :					int=False       ,					_A   :					Optional[Any]=True       ,					_A   :					List[str]=False       ,					_A   :					Any="<s>"       ,					_A   :					Optional[int]="</s>"       ,					_A   :					Union[str, Any]="<unk>"       ,					_A   :					int="<sep>"       ,					_A   :					str="<pad>"       ,					_A   :					Union[str, Any]="<cls>"       ,					_A   :					str="<mask>"       ,					_A   :					List[Any]=["<eop>", "<eod>"]       ,					_A   :					Optional[Dict[str, Any]] = None       ,					**_A   :					Union[str, Any]       ,					)		->		None:
         snake_case				  =      AddedToken(_A       ,					lstrip=_A       ,					rstrip=_A     ) if isinstance(_A       ,					_A     ) else mask_token

         snake_case				  =      {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=_A       ,					remove_space=_A       ,					keep_accents=_A       ,					bos_token=_A       ,					eos_token=_A       ,					unk_token=_A       ,					sep_token=_A       ,					pad_token=_A       ,					cls_token=_A       ,					mask_token=_A       ,					additional_special_tokens=_A       ,					sp_model_kwargs=self.sp_model_kwargs       ,					**_A       ,					)

         snake_case				  =      3

         snake_case				  =      do_lower_case
         snake_case				  =      remove_space
         snake_case				  =      keep_accents
         snake_case				  =      vocab_file

         snake_case				  =      spm.SentencePieceProcessor(**self.sp_model_kwargs     )
         self.sp_model.Load(_A     )

         try:
             import jieba
         except ModuleNotFoundError as error:
             raise error.__class__(
                 "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
                 "See https://pypi.org/project/jieba/ for installation."     )
         snake_case				  =      jieba
         snake_case				  =      str.maketrans(" \n"       ,					"\u2582\u2583"     )





     @property
     # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
     def 	UpperCAmelCase(self   :					int     )		->		Tuple:
         return len(self.sp_model     )





     def 	UpperCAmelCase(self   :					Any     )		->		Optional[int]:
         snake_case				  =      {self.convert_ids_to_tokens(_A     ): i for i in range(self.vocab_size     )}
         vocab.update(self.added_tokens_encoder     )
         return vocab





     def __getstate__(self   :					Any     )		->		List[Any]:
         snake_case				  =      self.__dict__.copy()
         snake_case				  =      None
         return state





     def __setstate__(self   :					Optional[int]       ,					_A   :					Optional[int]     )		->		Tuple:
         snake_case				  =      d

         # for backward compatibility
         if not hasattr(self       ,					"sp_model_kwargs"     ):
             snake_case				  =      {}

         snake_case				  =      spm.SentencePieceProcessor(**self.sp_model_kwargs     )
         self.sp_model.Load(self.vocab_file     )





     def 	UpperCAmelCase(self   :					Optional[int]       ,					_A   :					Optional[int]     )		->		List[Any]:
         if self.remove_space:
             snake_case				  =      " ".join(inputs.strip().split()     )
         else:
             snake_case				  =      inputs
         snake_case				  =      outputs.replace("``"       ,					"\""     ).replace("''"       ,					"\""     )

         if not self.keep_accents:
             snake_case				  =      unicodedata.normalize("NFKD"       ,					_A     )
             snake_case				  =      "".join([c for c in outputs if not unicodedata.combining(_A     )]     )
         if self.do_lower_case:
             snake_case				  =      outputs.lower()

         return outputs





     def 	UpperCAmelCase(self   :					Optional[int]       ,					_A   :					str     )		->		List[str]:
         snake_case				  =      self.preprocess_text(_A     )
         snake_case				  =      self.sp_model.encode(_A       ,					out_type=_A     )
         snake_case				  =      []
         for piece in pieces:
             if len(_A     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
                 snake_case				  =      self.sp_model.EncodeAsPieces(piece[:-1].replace(_A       ,					""     )     )
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                     if len(cur_pieces[0]     ) == 1:
                         snake_case				  =      cur_pieces[1:]
                     else:
                         snake_case				  =      cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]     )
                 new_pieces.extend(_A     )
             else:
                 new_pieces.append(_A     )

         return new_pieces





     def 	UpperCAmelCase(self   :					Optional[Any]       ,					_A   :					Optional[Any]     )		->		List[Any]:
         return self.sp_model.PieceToId(_A     )





     def 	UpperCAmelCase(self   :					Optional[int]       ,					_A   :					Dict     )		->		Optional[int]:
         return self.sp_model.IdToPiece(_A     )





     def 	UpperCAmelCase(self   :					str       ,					_A   :					Dict     )		->		int:
         snake_case				  =      "".join(_A     ).replace(_A       ,					" "     ).strip()
         return out_string





     def 	UpperCAmelCase(self   :					Any       ,					_A   :					List[int]       ,					_A   :					Optional[List[int]] = None     )		->		List[int]:
         snake_case				  =      [self.sep_token_id]
         snake_case				  =      [self.cls_token_id]
         if token_ids_a is None:
             return token_ids_a + sep + cls
         return token_ids_a + sep + token_ids_a + sep + cls





     def 	UpperCAmelCase(self   :					Optional[Any]       ,					_A   :					List[int]       ,					_A   :					Optional[List[int]] = None       ,					_A   :					bool = False     )		->		List[int]:

         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_A       ,					token_ids_a=_A       ,					already_has_special_tokens=_A     )

         if token_ids_a is not None:
             return ([0] * len(_A     )) + [1] + ([0] * len(_A     )) + [1, 1]
         return ([0] * len(_A     )) + [1, 1]





     def 	UpperCAmelCase(self   :					str       ,					_A   :					List[int]       ,					_A   :					Optional[List[int]] = None     )		->		List[int]:
         snake_case				  =      [self.sep_token_id]
         snake_case				  =      [2]

         if token_ids_a is None:
             return len(token_ids_a + sep     ) * [0] + cls_segment_id
         return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id





     def 	UpperCAmelCase(self   :					Any       ,					_A   :					str       ,					_A   :					Optional[str] = None     )		->		Tuple[str]:
         if not os.path.isdir(_A     ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'     )
             return
         snake_case				  =      os.path.join(
             _A       ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

         if os.path.abspath(self.vocab_file     ) != os.path.abspath(_A     ) and os.path.isfile(self.vocab_file     ):
             copyfile(self.vocab_file       ,					_A     )
         elif not os.path.isfile(self.vocab_file     ):
             with open(_A       ,					"wb"     ) as fi:
                 snake_case				  =      self.sp_model.serialized_model_proto()
                 fi.write(_A     )

         return (out_vocab_file,)





     def 	UpperCAmelCase(self   :					Tuple       ,					*_A   :					List[Any]       ,					**_A   :					Dict     )		->		int:
         snake_case				  =      super()._decode(*_A       ,					**_A     )
         snake_case				  =      text.replace(" "       ,					""     ).replace("\u2582"       ,					" "     ).replace("\u2583"       ,					"\n"     )
         return text



from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_A						      =				logging.get_logger(__name__)

_A						      =				{
    "sail/poolformer_s12": "https://huggingface.co/sail/poolformer_s12/resolve/main/config.json",
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
}







class   lowerCamelCase	(						A_						):
     UpperCAmelCase__       :							Any     =	"poolformer"





     def __init__(self   :					Optional[int]       ,					_A   :					Optional[Any]=3       ,					_A   :					Optional[int]=1_6       ,					_A   :					Dict=1_6       ,					_A   :					Tuple=3       ,					_A   :					Tuple=4.0       ,					_A   :					int=[2, 2, 6, 2]       ,					_A   :					Dict=[6_4, 1_2_8, 3_2_0, 5_1_2]       ,					_A   :					int=[7, 3, 3, 3]       ,					_A   :					List[str]=[4, 2, 2, 2]       ,					_A   :					str=[2, 1, 1, 1]       ,					_A   :					List[Any]=4       ,					_A   :					Any=0.0       ,					_A   :					Optional[Any]="gelu"       ,					_A   :					Optional[Any]=True       ,					_A   :					List[str]=1E-5       ,					_A   :					List[str]=0.02       ,					**_A   :					str       ,					)		->		Tuple:
         snake_case				  =      num_channels
         snake_case				  =      patch_size
         snake_case				  =      stride
         snake_case				  =      padding
         snake_case				  =      pool_size
         snake_case				  =      hidden_sizes
         snake_case				  =      mlp_ratio
         snake_case				  =      depths
         snake_case				  =      patch_sizes
         snake_case				  =      strides
         snake_case				  =      num_encoder_blocks
         snake_case				  =      drop_path_rate
         snake_case				  =      hidden_act
         snake_case				  =      use_layer_scale
         snake_case				  =      layer_scale_init_value
         snake_case				  =      initializer_range
         super().__init__(**_A     )







class   lowerCamelCase	(						A_						):
     UpperCAmelCase__       :							Tuple     =	version.parse("1.11"						)





     @property
     def 	UpperCAmelCase(self   :					int     )		->		Mapping[str, Mapping[int, str]]:
         return OrderedDict(
             [
                 ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
             ]     )





     @property
     def 	UpperCAmelCase(self   :					int     )		->		float:
         return 2E-3

