
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_a      :			Tuple					=							logging.get_logger(__name__)

_a      :			Dict					=							{'vocab_file': 'spiece.model'}

_a      :			Dict					=							{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

_a      :			Optional[Any]					=							{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
_a      :			List[str]					=							0
_a      :			Tuple					=							1
_a      :			Dict					=							2
_a      :			Optional[Any]					=							3
_a      :			List[Any]					=							4






class      _UpperCAmelCase				(					UpperCamelCase_):
    __lowercase							: Any  =					VOCAB_FILES_NAMES
    __lowercase							: str  =					PRETRAINED_VOCAB_FILES_MAP
    __lowercase							: Tuple  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowercase							: List[Any]  =					"""left"""
    def __init__(    self						,    snake_case_						,    snake_case_=False						,    snake_case_=True						,    snake_case_=False						,    snake_case_="<s>"						,    snake_case_="</s>"						,    snake_case_="<unk>"						,    snake_case_="<sep>"						,    snake_case_="<pad>"						,    snake_case_="<cls>"						,    snake_case_="<mask>"						,    snake_case_=["<eop>", "<eod>"]						,    snake_case_ = None						,    **snake_case_						,    ):
      _snake_case     :						Union[str, Any]						=   AddedToken(UpperCamelCase__						,    lstrip=UpperCamelCase__						,    rstrip=UpperCamelCase__     ) if isinstance(UpperCamelCase__						,    UpperCamelCase__     ) else mask_token

      _snake_case     :						str						=   {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=UpperCamelCase__						,    remove_space=UpperCamelCase__						,    keep_accents=UpperCamelCase__						,    bos_token=UpperCamelCase__						,    eos_token=UpperCamelCase__						,    unk_token=UpperCamelCase__						,    sep_token=UpperCamelCase__						,    pad_token=UpperCamelCase__						,    cls_token=UpperCamelCase__						,    mask_token=UpperCamelCase__						,    additional_special_tokens=UpperCamelCase__						,    sp_model_kwargs=self.sp_model_kwargs						,    **UpperCamelCase__						,    )

      _snake_case     :						List[str]						=   3

      _snake_case     :						Tuple						=   do_lower_case
      _snake_case     :						Dict						=   remove_space
      _snake_case     :						Optional[int]						=   keep_accents
      _snake_case     :						List[Any]						=   vocab_file

      _snake_case     :						Any						=   spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(UpperCamelCase__     )
    @property
    def 					lowerCamelCase__	(    self     ):
      return len(self.sp_model     )
    def 					lowerCamelCase__	(    self     ):
      _snake_case     :						int						=   {self.convert_ids_to_tokens(UpperCamelCase__     ): i for i in range(self.vocab_size     )}
      vocab.update(self.added_tokens_encoder     )
      return vocab
    def __getstate__(    self     ):
      _snake_case     :						int						=   self.__dict__.copy()
      _snake_case     :						Tuple						=   None
      return state
    def __setstate__(    self						,    snake_case_     ):
      _snake_case     :						Union[str, Any]						=   d

      # for backward compatibility
      if not hasattr(self						,    "sp_model_kwargs"     ):
        _snake_case     :						Optional[Any]						=   {}

      _snake_case     :						Dict						=   spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(self.vocab_file     )
    def 					lowerCamelCase__	(    self						,    snake_case_     ):





      if self.remove_space:
        _snake_case     :						List[Any]						=   ''' '''.join(inputs.strip().split()     )
      else:
        _snake_case     :						List[str]						=   inputs
      _snake_case     :						List[str]						=   outputs.replace("``"						,    "\""     ).replace("\'\'"						,    "\""     )

      if not self.keep_accents:
        _snake_case     :						Any						=   unicodedata.normalize("NFKD"						,    UpperCamelCase__     )
        _snake_case     :						List[str]						=   ''''''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase__     )]     )
      if self.do_lower_case:
        _snake_case     :						Union[str, Any]						=   outputs.lower()

      return outputs
    def 					lowerCamelCase__	(    self						,    snake_case_     ):
      _snake_case     :						Optional[Any]						=   self.preprocess_text(UpperCamelCase__     )
      _snake_case     :						Optional[Any]						=   self.sp_model.encode(UpperCamelCase__						,    out_type=UpperCamelCase__     )
      _snake_case     :						Optional[int]						=   []
      for piece in pieces:
        if len(UpperCamelCase__     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
          _snake_case     :						str						=   self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase__						,    ""     )     )
          if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
            if len(cur_pieces[0]     ) == 1:
              _snake_case     :						Tuple						=   cur_pieces[1:]
            else:
              _snake_case     :						Union[str, Any]						=   cur_pieces[0][1:]
          cur_pieces.append(piece[-1]     )
          new_pieces.extend(UpperCamelCase__     )
        else:
          new_pieces.append(UpperCamelCase__     )

      return new_pieces
    def 					lowerCamelCase__	(    self						,    snake_case_     ):
      return self.sp_model.PieceToId(UpperCamelCase__     )
    def 					lowerCamelCase__	(    self						,    snake_case_     ):
      return self.sp_model.IdToPiece(UpperCamelCase__     )
    def 					lowerCamelCase__	(    self						,    snake_case_     ):
      _snake_case     :						Any						=   ''''''.join(UpperCamelCase__     ).replace(UpperCamelCase__						,    " "     ).strip()
      return out_string
    def 					lowerCamelCase__	(    self						,    snake_case_						,    snake_case_ = False						,    snake_case_ = None						,    snake_case_ = True						,    **snake_case_						,    ):
      _snake_case     :						Optional[int]						=   kwargs.pop("use_source_tokenizer"						,    UpperCamelCase__     )

      _snake_case     :						Tuple						=   self.convert_ids_to_tokens(UpperCamelCase__						,    skip_special_tokens=UpperCamelCase__     )

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      _snake_case     :						Tuple						=   []
      _snake_case     :						Optional[int]						=   []
      for token in filtered_tokens:
        if skip_special_tokens and token in self.all_special_ids:
          continue
        if token in self.added_tokens_encoder:
          if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(UpperCamelCase__     )     )
            _snake_case     :						List[str]						=   []
          sub_texts.append(UpperCamelCase__     )
        else:
          current_sub_text.append(UpperCamelCase__     )
      if current_sub_text:
        sub_texts.append(self.convert_tokens_to_string(UpperCamelCase__     )     )

      # Mimic the behavior of the Rust tokenizer:
      # By default, there are no spaces between special tokens
      _snake_case     :						Any						=   ''''''.join(UpperCamelCase__     )

      _snake_case     :						Optional[int]						=   (
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
        _snake_case     :						Dict						=   self.clean_up_tokenization(UpperCamelCase__     )
        return clean_text
      else:
        return text
    def 					lowerCamelCase__	(    self						,    snake_case_						,    snake_case_ = None     ):
      _snake_case     :						int						=   [self.sep_token_id]
      _snake_case     :						List[Any]						=   [self.cls_token_id]
      if token_ids_a is None:
        return token_ids_a + sep + cls
      return token_ids_a + sep + token_ids_a + sep + cls
    def 					lowerCamelCase__	(    self						,    snake_case_						,    snake_case_ = None						,    snake_case_ = False     ):





      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=UpperCamelCase__						,    token_ids_a=UpperCamelCase__						,    already_has_special_tokens=UpperCamelCase__     )

      if token_ids_a is not None:
        return ([0] * len(UpperCamelCase__     )) + [1] + ([0] * len(UpperCamelCase__     )) + [1, 1]
      return ([0] * len(UpperCamelCase__     )) + [1, 1]
    def 					lowerCamelCase__	(    self						,    snake_case_						,    snake_case_ = None     ):
      _snake_case     :						List[str]						=   [self.sep_token_id]
      _snake_case     :						Dict						=   [2]

      if token_ids_a is None:
        return len(token_ids_a + sep     ) * [0] + cls_segment_id
      return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id


    def 					lowerCamelCase__	(    self						,    snake_case_						,    snake_case_ = None     ):





      if not os.path.isdir(UpperCamelCase__     ):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'     )
        return
      _snake_case     :						List[Any]						=   os.path.join(
          UpperCamelCase__						,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

      if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase__     ) and os.path.isfile(self.vocab_file     ):
        copyfile(self.vocab_file						,    UpperCamelCase__     )
      elif not os.path.isfile(self.vocab_file     ):
        with open(UpperCamelCase__						,    "wb"     ) as fi:
          _snake_case     :						Union[str, Any]						=   self.sp_model.serialized_model_proto()
          fi.write(UpperCamelCase__     )

      return (out_vocab_file,)



"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


_a      :			Optional[int]					=							logging.get_logger(__name__)

_a      :			List[str]					=							{
    """facebook/convnextv2-tiny-1k-224""": """https://huggingface.co/facebook/convnextv2-tiny-1k-224/resolve/main/config.json""",
}






class      _UpperCAmelCase				(					_snake_case		,					_snake_case):
    __lowercase							: List[Any]  =					"""convnextv2"""
    def __init__(    self						,    snake_case_=3						,    snake_case_=4						,    snake_case_=4						,    snake_case_=None						,    snake_case_=None						,    snake_case_="gelu"						,    snake_case_=0.02						,    snake_case_=1E-12						,    snake_case_=0.0						,    snake_case_=2_24						,    snake_case_=None						,    snake_case_=None						,    **snake_case_						,    ):
      super().__init__(**snake_case_     )

      _snake_case     :						Tuple						=   num_channels
      _snake_case     :						Optional[int]						=   patch_size
      _snake_case     :						Tuple						=   num_stages
      _snake_case     :						int						=   [96, 1_92, 3_84, 7_68] if hidden_sizes is None else hidden_sizes
      _snake_case     :						str						=   [3, 3, 9, 3] if depths is None else depths
      _snake_case     :						int						=   hidden_act
      _snake_case     :						Tuple						=   initializer_range
      _snake_case     :						Union[str, Any]						=   layer_norm_eps
      _snake_case     :						Optional[int]						=   drop_path_rate
      _snake_case     :						Union[str, Any]						=   image_size
      _snake_case     :						List[Any]						=   ["stem"] + [F'stage{idx}' for idx in range(1						,    len(self.depths     ) + 1     )]
      _snake_case			,					_snake_case     :						Dict						=   get_aligned_output_features_output_indices(
          out_features=snake_case_						,    out_indices=snake_case_						,    stage_names=self.stage_names     )

