"""simple docstring"""


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCAmelCase__			=    logging.get_logger(__name__)

UpperCAmelCase__			=    {"""vocab_file""": """spiece.model"""}

UpperCAmelCase__			=    {
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

UpperCAmelCase__			=    {
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
UpperCAmelCase__			=    0
UpperCAmelCase__			=    1
UpperCAmelCase__			=    2
UpperCAmelCase__			=    3
UpperCAmelCase__			=    4


class 							a					(		lowerCAmelCase_ ):
    _snake_case  :	List[Any]  				=				VOCAB_FILES_NAMES
    _snake_case  :	str  				=				PRETRAINED_VOCAB_FILES_MAP
    _snake_case  :	List[Any]  				=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _snake_case  :	Tuple  				=				'left'


    def __init__(       self						:	Any ,					__lowerCAmelCase						:	int ,					__lowerCAmelCase						:	List[Any]=False ,					__lowerCAmelCase						:	List[Any]=True ,					__lowerCAmelCase						:	List[str]=False ,					__lowerCAmelCase						:	str="<s>" ,					__lowerCAmelCase						:	Tuple="</s>" ,					__lowerCAmelCase						:	Tuple="<unk>" ,					__lowerCAmelCase						:	Tuple="<sep>" ,					__lowerCAmelCase						:	List[str]="<pad>" ,					__lowerCAmelCase						:	List[str]="<cls>" ,					__lowerCAmelCase						:	List[str]="<mask>" ,					__lowerCAmelCase						:	Optional[int]=["<eop>", "<eod>"] ,					__lowerCAmelCase						:	Optional[Dict[str, Any]] = None ,					**__lowerCAmelCase						:	Dict ,					):
      # Mask token behave like a normal word, i.e. include the space before it
      _UpperCAmelCase    			=   AddedToken(__lowerCAmelCase ,					lstrip=__lowerCAmelCase ,					rstrip=__lowerCAmelCase       ) if isinstance(__lowerCAmelCase ,					__lowerCAmelCase       ) else mask_token

      _UpperCAmelCase    			=   {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=__lowerCAmelCase ,					remove_space=__lowerCAmelCase ,					keep_accents=__lowerCAmelCase ,					bos_token=__lowerCAmelCase ,					eos_token=__lowerCAmelCase ,					unk_token=__lowerCAmelCase ,					sep_token=__lowerCAmelCase ,					pad_token=__lowerCAmelCase ,					cls_token=__lowerCAmelCase ,					mask_token=__lowerCAmelCase ,					additional_special_tokens=__lowerCAmelCase ,					sp_model_kwargs=self.sp_model_kwargs ,					**__lowerCAmelCase ,					)

      _UpperCAmelCase    			=   3

      _UpperCAmelCase    			=   do_lower_case
      _UpperCAmelCase    			=   remove_space
      _UpperCAmelCase    			=   keep_accents
      _UpperCAmelCase    			=   vocab_file

      _UpperCAmelCase    			=   spm.SentencePieceProcessor(**self.sp_model_kwargs       )
      self.sp_model.Load(__lowerCAmelCase       )


    @property
    def   lowerCAmelCase_	(       self						:	Dict       ):
      return len(self.sp_model       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   {self.convert_ids_to_tokens(__lowerCAmelCase       ): i for i in range(self.vocab_size       )}
      vocab.update(self.added_tokens_encoder       )
      return vocab


    def __getstate__(       self						:	Dict       ):
      _UpperCAmelCase    			=   self.__dict__.copy()
      _UpperCAmelCase    			=   None
      return state


    def __setstate__(       self						:	Dict ,					__lowerCAmelCase						:	str       ):
      _UpperCAmelCase    			=   d

      # for backward compatibility
      if not hasattr(self ,					"""sp_model_kwargs"""       ):
        _UpperCAmelCase    			=   {}

      _UpperCAmelCase    			=   spm.SentencePieceProcessor(**self.sp_model_kwargs       )
      self.sp_model.Load(self.vocab_file       )


    def   lowerCAmelCase_	(       self						:	str ,					__lowerCAmelCase						:	Optional[Any]       ):
      if self.remove_space:
        _UpperCAmelCase    			=   """ """.join(inputs.strip().split()       )
      else:
        _UpperCAmelCase    			=   inputs
      _UpperCAmelCase    			=   outputs.replace("""``""" ,					"""\""""       ).replace("""''""" ,					"""\""""       )

      if not self.keep_accents:
        _UpperCAmelCase    			=   unicodedata.normalize("""NFKD""" ,					__lowerCAmelCase       )
        _UpperCAmelCase    			=   """""".join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase       )]       )
      if self.do_lower_case:
        _UpperCAmelCase    			=   outputs.lower()

      return outputs


    def   lowerCAmelCase_	(       self						:	Dict ,					__lowerCAmelCase						:	str       ):
      _UpperCAmelCase    			=   self.preprocess_text(__lowerCAmelCase       )
      _UpperCAmelCase    			=   self.sp_model.encode(__lowerCAmelCase ,					out_type=__lowerCAmelCase       )
      _UpperCAmelCase    			=   []
      for piece in pieces:
        if len(__lowerCAmelCase       ) > 1 and piece[-1] == str(""","""       ) and piece[-2].isdigit():
          _UpperCAmelCase    			=   self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase ,					""""""       )       )
          if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
            if len(cur_pieces[0]       ) == 1:
              _UpperCAmelCase    			=   cur_pieces[1:]
            else:
              _UpperCAmelCase    			=   cur_pieces[0][1:]
          cur_pieces.append(piece[-1]       )
          new_pieces.extend(__lowerCAmelCase       )
        else:
          new_pieces.append(__lowerCAmelCase       )

      return new_pieces


    def   lowerCAmelCase_	(       self						:	Any ,					__lowerCAmelCase						:	Optional[int]       ):
      return self.sp_model.PieceToId(__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	int ,					__lowerCAmelCase						:	str       ):
      return self.sp_model.IdToPiece(__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	str ,					__lowerCAmelCase						:	List[Any]       ):
      _UpperCAmelCase    			=   """""".join(__lowerCAmelCase       ).replace(__lowerCAmelCase ,					""" """       ).strip()
      return out_string


    def   lowerCAmelCase_	(       self						:	Any ,					__lowerCAmelCase						:	List[int] ,					__lowerCAmelCase						:	bool = False ,					__lowerCAmelCase						:	bool = None ,					__lowerCAmelCase						:	bool = True ,					**__lowerCAmelCase						:	Tuple ,					):
      _UpperCAmelCase    			=   kwargs.pop("""use_source_tokenizer""" ,					__lowerCAmelCase       )

      _UpperCAmelCase    			=   self.convert_ids_to_tokens(__lowerCAmelCase ,					skip_special_tokens=__lowerCAmelCase       )

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      _UpperCAmelCase    			=   []
      _UpperCAmelCase    			=   []
      for token in filtered_tokens:
        if skip_special_tokens and token in self.all_special_ids:
          continue
        if token in self.added_tokens_encoder:
          if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(__lowerCAmelCase       )       )
            _UpperCAmelCase    			=   []
          sub_texts.append(__lowerCAmelCase       )
        else:
          current_sub_text.append(__lowerCAmelCase       )
      if current_sub_text:
        sub_texts.append(self.convert_tokens_to_string(__lowerCAmelCase       )       )

      # Mimic the behavior of the Rust tokenizer:
      # By default, there are no spaces between special tokens
      _UpperCAmelCase    			=   """""".join(__lowerCAmelCase       )

      _UpperCAmelCase    			=   (
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
        _UpperCAmelCase    			=   self.clean_up_tokenization(__lowerCAmelCase       )
        return clean_text
      else:
        return text


    def   lowerCAmelCase_	(       self						:	List[Any] ,					__lowerCAmelCase						:	List[int] ,					__lowerCAmelCase						:	Optional[List[int]] = None       ):
      _UpperCAmelCase    			=   [self.sep_token_id]
      _UpperCAmelCase    			=   [self.cls_token_id]
      if token_ids_a is None:
        return token_ids_a + sep + cls
      return token_ids_a + sep + token_ids_a + sep + cls


    def   lowerCAmelCase_	(       self						:	List[str] ,					__lowerCAmelCase						:	List[int] ,					__lowerCAmelCase						:	Optional[List[int]] = None ,					__lowerCAmelCase						:	bool = False       ):

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__lowerCAmelCase ,					token_ids_a=__lowerCAmelCase ,					already_has_special_tokens=__lowerCAmelCase       )

      if token_ids_a is not None:
        return ([0] * len(__lowerCAmelCase       )) + [1] + ([0] * len(__lowerCAmelCase       )) + [1, 1]
      return ([0] * len(__lowerCAmelCase       )) + [1, 1]


    def   lowerCAmelCase_	(       self						:	int ,					__lowerCAmelCase						:	List[int] ,					__lowerCAmelCase						:	Optional[List[int]] = None       ):
      _UpperCAmelCase    			=   [self.sep_token_id]
      _UpperCAmelCase    			=   [2]

      if token_ids_a is None:
        return len(token_ids_a + sep       ) * [0] + cls_segment_id
      return len(token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1] + cls_segment_id


    def   lowerCAmelCase_	(       self						:	Any ,					__lowerCAmelCase						:	str ,					__lowerCAmelCase						:	Optional[str] = None       ):
      if not os.path.isdir(__lowerCAmelCase       ):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
        return
      _UpperCAmelCase    			=   os.path.join(
          __lowerCAmelCase ,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

      if os.path.abspath(self.vocab_file       ) != os.path.abspath(__lowerCAmelCase       ) and os.path.isfile(self.vocab_file       ):
        copyfile(self.vocab_file ,					__lowerCAmelCase       )
      elif not os.path.isfile(self.vocab_file       ):
        with open(__lowerCAmelCase ,					"""wb"""       ) as fi:
          _UpperCAmelCase    			=   self.sp_model.serialized_model_proto()
          fi.write(__lowerCAmelCase       )

      return (out_vocab_file,)

"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__			=    logging.get_logger(__name__)


class 							a					(		lowerCAmelCase_ ):
    _snake_case  :	List[Any]  				=				'timm_backbone'


    def __init__(       self						:	List[Any] ,					__lowerCAmelCase						:	str=None ,					__lowerCAmelCase						:	Optional[int]=3 ,					__lowerCAmelCase						:	Any=True ,					__lowerCAmelCase						:	Optional[int]=True ,					__lowerCAmelCase						:	Optional[Any]=None ,					**__lowerCAmelCase						:	Optional[Any] ,					):
      super().__init__(**__lowerCAmelCase       )
      _UpperCAmelCase    			=   backbone
      _UpperCAmelCase    			=   num_channels
      _UpperCAmelCase    			=   features_only
      _UpperCAmelCase    			=   use_pretrained_backbone
      _UpperCAmelCase    			=   True
      _UpperCAmelCase    			=   out_indices if out_indices is not None else (-1,)
