

import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase      =   logging.get_logger(__name__)

UpperCAmelCase      =   '''▁'''

UpperCAmelCase      =   {'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

UpperCAmelCase      =   {
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

UpperCAmelCase      =   {
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

UpperCAmelCase      =   {
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

UpperCAmelCase      =   {
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}







class lowerCAmelCase_						(			A__	):

 '''simple docstring'''

 __snake_case												=  ["input_ids"]

 __snake_case												=  VOCAB_FILES_NAMES
 __snake_case												=  PRETRAINED_INIT_CONFIGURATION
 __snake_case												=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case												=  PRETRAINED_VOCAB_FILES_MAP
 __snake_case												=  RESOURCE_FILES_NAMES



 def __init__(     self		,				_UpperCAmelCase		,				_UpperCAmelCase=None		,				_UpperCAmelCase=False		,				_UpperCAmelCase="utf8"		,				_UpperCAmelCase="[UNK]"		,				_UpperCAmelCase="[SEP]"		,				_UpperCAmelCase="[PAD]"		,				_UpperCAmelCase="[CLS]"		,				_UpperCAmelCase="[MASK]"		,				_UpperCAmelCase = None		,				**_UpperCAmelCase		,				):
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.

       snake_case_	=				{} if sp_model_kwargs is None else sp_model_kwargs
       super().__init__(
           do_lower_case=lowerCamelCase__		,				unk_token=lowerCamelCase__		,				sep_token=lowerCamelCase__		,				pad_token=lowerCamelCase__		,				cls_token=lowerCamelCase__		,				mask_token=lowerCamelCase__		,				vocab_file=lowerCamelCase__		,				encoding=lowerCamelCase__		,				sp_model_kwargs=self.sp_model_kwargs		,				**lowerCamelCase__		,				)
       snake_case_	=				do_lower_case
       snake_case_	=				sentencepiece_model_ckpt
       snake_case_	=				spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(lowerCamelCase__ )

       # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
       if vocab_file is not None:
             snake_case_	=				self.load_vocab(filepath=lowerCamelCase__ )
       else:
             snake_case_	=				{self.sp_model.id_to_piece(lowerCamelCase__ ): id for id in range(self.sp_model.get_piece_size() )}
       snake_case_	=				{v: k for k, v in self.vocab.items()}



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       if text is None:
             return None

       snake_case_	=				self.tokenize(lowerCamelCase__ )
       snake_case_	,   snake_case_	=				'''''', []

       for i, ch in enumerate(lowerCamelCase__ ):
             if ch in self.SP_CHAR_MAPPING:
                   snake_case_	=				self.SP_CHAR_MAPPING.get(lowerCamelCase__ )
             else:
                   snake_case_	=				unicodedata.normalize('''NFKC'''		,				lowerCamelCase__ )
             if self.is_whitespace(lowerCamelCase__ ):
                   continue
             normalized_text += ch
             char_mapping.extend([i] * len(lowerCamelCase__ ) )

       snake_case_	,   snake_case_	,   snake_case_	=				normalized_text, [], 0

       if self.do_lower_case:
             snake_case_	=				text.lower()

       for token in split_tokens:
             if token[:1] == "▁":
                   snake_case_	=				token[1:]
             snake_case_	=				text[offset:].index(lowerCamelCase__ ) + offset
             snake_case_	=				start + len(lowerCamelCase__ )

             token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1) )
             snake_case_	=				end
       return token_mapping



 @property
 def 						UpperCamelCase__					(     self ):
       return len(self.vocab )



 def 						UpperCamelCase__					(     self ):
       return dict(self.vocab		,				**self.added_tokens_encoder )



 def __getstate__(     self ):
       snake_case_	=				self.__dict__.copy()
       snake_case_	=				None
       return state



 def __setstate__(     self		,				_UpperCAmelCase ):
       snake_case_	=				d

       # for backward compatibility
       if not hasattr(self		,				'''sp_model_kwargs''' ):
             snake_case_	=				{}

       snake_case_	=				spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(self.sentencepiece_model_ckpt )



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       return "".join((self.SP_CHAR_MAPPING.get(lowerCamelCase__		,				lowerCamelCase__ ) for c in text) )



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase=False		,				_UpperCAmelCase=64		,				_UpperCAmelCase=0.1 ):

       if self.sp_model_kwargs.get('''enable_sampling''' ) is True:
             snake_case_	=				True
       if self.sp_model_kwargs.get('''alpha''' ) is not None:
             snake_case_	=				self.sp_model_kwargs.get('''alpha''' )
       if self.sp_model_kwargs.get('''nbest_size''' ) is not None:
             snake_case_	=				self.sp_model_kwargs.get('''nbest_size''' )

       if not enable_sampling:
             snake_case_	=				self.sp_model.EncodeAsPieces(lowerCamelCase__ )
       else:
             snake_case_	=				self.sp_model.SampleEncodeAsPieces(lowerCamelCase__		,				lowerCamelCase__		,				lowerCamelCase__ )
       snake_case_	=				[]
       for pi, piece in enumerate(lowerCamelCase__ ):
             if piece == SPIECE_UNDERLINE:
                   if not pieces[pi + 1].startswith(lowerCamelCase__ ) and pi != 0:
                         new_pieces.append(lowerCamelCase__ )
                         continue
                   else:
                         continue
             snake_case_	=				0
             for i, chunk in enumerate(lowerCamelCase__ ):
                   if chunk == SPIECE_UNDERLINE:
                         continue
                   if self.is_ch_char(lowerCamelCase__ ) or self.is_punct(lowerCamelCase__ ):
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                               new_pieces.append(piece[lst_i:i] )
                         new_pieces.append(lowerCamelCase__ )
                         snake_case_	=				i + 1
                   elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                               new_pieces.append(piece[lst_i:i] )
                         snake_case_	=				i
                   elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                               new_pieces.append(piece[lst_i:i] )
                         snake_case_	=				i
             if len(lowerCamelCase__ ) > lst_i:
                   new_pieces.append(piece[lst_i:] )
       return new_pieces



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       snake_case_	=				''''''.join(lowerCamelCase__ ).replace(lowerCamelCase__		,				''' ''' ).strip()
       return out_string



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       snake_case_	=				self.convert_ids_to_tokens(lowerCamelCase__ )
       snake_case_	=				''''''.join(lowerCamelCase__ ).replace(lowerCamelCase__		,				''' ''' ).strip()
       return out_string



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       return self.vocab.get(lowerCamelCase__		,				self.vocab.get(self.unk_token ) )



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       return self.reverse_vocab.get(lowerCamelCase__		,				self.unk_token )



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase=None ):
       if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       snake_case_	=				[self.cls_token_id]
       snake_case_	=				[self.sep_token_id]
       return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase=None ):
       if offset_mapping_a is None:
             return [(0, 0)] + offset_mapping_a + [(0, 0)]

       return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase=None		,				_UpperCAmelCase=False ):

       if already_has_special_tokens:
             if token_ids_a is not None:
                   raise ValueError(
                       '''You should not supply a second sequence if the provided sequence of '''
                       '''ids is already formatted with special tokens for the model.''' )
             return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
             return [1] + ([0] * len(lowerCamelCase__ )) + [1, 1] + ([0] * len(lowerCamelCase__ )) + [1]
       return [1] + ([0] * len(lowerCamelCase__ )) + [1]



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase = None ):
       # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
       if token_ids_a is None:
             # [CLS] X [SEP]
             return (len(lowerCamelCase__ ) + 2) * [0]

       # [CLS] A [SEP] [SEP] B [SEP]
       return [0] * (len(lowerCamelCase__ ) + 1) + [1] * (len(lowerCamelCase__ ) + 3)



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       if "\u4e00" <= char <= "\u9fff":
             return True
       return False



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       if ("a" <= char <= "z") or ("A" <= char <= "Z"):
             return True
       return False



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       if char in ",;:.?!~，；：。？！《》【】":
             return True
       return False



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       if char == " " or char == "\t" or char == "\n" or char == "\r":
             return True
       if len(lowerCamelCase__ ) == 1:
             snake_case_	=				unicodedata.category(lowerCamelCase__ )
             if cat == "Zs":
                   return True
       return False



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       snake_case_	=				{}
       with io.open(lowerCamelCase__		,				'''r'''		,				encoding='''utf-8''' ) as f:
             for index, line in enumerate(lowerCamelCase__ ):
                   snake_case_	=				line.rstrip('''\n''' )
                   snake_case_	=				int(lowerCamelCase__ )

       return token_to_idx





 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase = None ):
       snake_case_	=				0
       if os.path.isdir(lowerCamelCase__ ):
             snake_case_	=				os.path.join(
                 lowerCamelCase__		,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
       else:
             snake_case_	=				(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
       with open(lowerCamelCase__		,				'''w'''		,				encoding='''utf-8''' ) as writer:
             for token, token_index in sorted(self.vocab.items()		,				key=lambda _UpperCAmelCase : kv[1] ):
                   if index != token_index:
                         logger.warning(
                             F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                             ''' Please check that the vocabulary is not corrupted!''' )
                         snake_case_	=				token_index
                   writer.write(token + '''\n''' )
                   index += 1

       snake_case_	=				os.path.join(lowerCamelCase__		,				'''sentencepiece.bpe.model''' )
       with open(lowerCamelCase__		,				'''wb''' ) as fi:
             snake_case_	=				self.sp_model.serialized_model_proto()
             fi.write(lowerCamelCase__ )

       return (vocab_file,)



from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


UpperCAmelCase      =   logging.get_logger(__name__)

UpperCAmelCase      =   {
    """facebook/convnextv2-tiny-1k-224""": """https://huggingface.co/facebook/convnextv2-tiny-1k-224/resolve/main/config.json""",
}







class lowerCAmelCase_						(			lowerCamelCase__ ,						lowerCamelCase__	):

 '''simple docstring'''

 __snake_case												=  "convnextv2"



 def __init__(     self		,				_UpperCAmelCase=3		,				_UpperCAmelCase=4		,				_UpperCAmelCase=4		,				_UpperCAmelCase=None		,				_UpperCAmelCase=None		,				_UpperCAmelCase="gelu"		,				_UpperCAmelCase=0.02		,				_UpperCAmelCase=1E-12		,				_UpperCAmelCase=0.0		,				_UpperCAmelCase=2_24		,				_UpperCAmelCase=None		,				_UpperCAmelCase=None		,				**_UpperCAmelCase		,				):
       super().__init__(**_UpperCAmelCase )

       snake_case_	=				num_channels
       snake_case_	=				patch_size
       snake_case_	=				num_stages
       snake_case_	=				[96, 1_92, 3_84, 7_68] if hidden_sizes is None else hidden_sizes
       snake_case_	=				[3, 3, 9, 3] if depths is None else depths
       snake_case_	=				hidden_act
       snake_case_	=				initializer_range
       snake_case_	=				layer_norm_eps
       snake_case_	=				drop_path_rate
       snake_case_	=				image_size
       snake_case_	=				['''stem'''] + [F'''stage{idx}''' for idx in range(1		,				len(self.depths ) + 1 )]
       snake_case_	,   snake_case_	=				get_aligned_output_features_output_indices(
           out_features=_UpperCAmelCase		,				out_indices=_UpperCAmelCase		,				stage_names=self.stage_names )