





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase__  			=logging.get_logger(__name__)

UpperCamelCase__  			={'vocab_file': 'spiece.model'}

UpperCamelCase__  			={
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

UpperCamelCase__  			={
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
UpperCamelCase__  			=0
UpperCamelCase__  			=1
UpperCamelCase__  			=2
UpperCamelCase__  			=3
UpperCamelCase__  			=4





class       lowerCAmelCase__(			lowerCAmelCase_     ):






       '''simple docstring'''


       __snake_case			      =     VOCAB_FILES_NAMES
       __snake_case			      =     PRETRAINED_VOCAB_FILES_MAP
       __snake_case			      =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __snake_case			      =     'left'




       def __init__(   self     ,    __lowerCamelCase     ,    __lowerCamelCase=False     ,    __lowerCamelCase=True     ,    __lowerCamelCase=False     ,    __lowerCamelCase="<s>"     ,    __lowerCamelCase="</s>"     ,    __lowerCamelCase="<unk>"     ,    __lowerCamelCase="<sep>"     ,    __lowerCamelCase="<pad>"     ,    __lowerCamelCase="<cls>"     ,    __lowerCamelCase="<mask>"     ,    __lowerCamelCase=["<eop>", "<eod>"]     ,    __lowerCamelCase = None     ,    **__lowerCamelCase     ,    )      ->  Union[str, Any]:
            # Mask token behave like a normal word, i.e. include the space before it
            _SCREAMING_SNAKE_CASE							:						Optional[int]         =				AddedToken(__lowerCAmelCase     ,    lstrip=__lowerCAmelCase     ,    rstrip=__lowerCAmelCase				) if isinstance(__lowerCAmelCase     ,    __lowerCAmelCase				) else mask_token

            _SCREAMING_SNAKE_CASE							:						int         =				{} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                do_lower_case=__lowerCAmelCase     ,    remove_space=__lowerCAmelCase     ,    keep_accents=__lowerCAmelCase     ,    bos_token=__lowerCAmelCase     ,    eos_token=__lowerCAmelCase     ,    unk_token=__lowerCAmelCase     ,    sep_token=__lowerCAmelCase     ,    pad_token=__lowerCAmelCase     ,    cls_token=__lowerCAmelCase     ,    mask_token=__lowerCAmelCase     ,    additional_special_tokens=__lowerCAmelCase     ,    sp_model_kwargs=self.sp_model_kwargs     ,    **__lowerCAmelCase     ,    )

            _SCREAMING_SNAKE_CASE							:						Any         =				3

            _SCREAMING_SNAKE_CASE							:						Optional[int]         =				do_lower_case
            _SCREAMING_SNAKE_CASE							:						Tuple         =				remove_space
            _SCREAMING_SNAKE_CASE							:						List[str]         =				keep_accents
            _SCREAMING_SNAKE_CASE							:						str         =				vocab_file

            _SCREAMING_SNAKE_CASE							:						Dict         =				spm.SentencePieceProcessor(**self.sp_model_kwargs				)
            self.sp_model.Load(__lowerCAmelCase				)




       @property
       def      UpperCamelCase_      (   self				)      ->  List[str]:
            return len(self.sp_model				)




       def      UpperCamelCase_      (   self				)      ->  Optional[Any]:
            _SCREAMING_SNAKE_CASE							:						Optional[Any]         =				{self.convert_ids_to_tokens(__lowerCAmelCase				): i for i in range(self.vocab_size				)}
            vocab.update(self.added_tokens_encoder				)
            return vocab




       def __getstate__(   self				)      ->  Dict:
            _SCREAMING_SNAKE_CASE							:						int         =				self.__dict__.copy()
            _SCREAMING_SNAKE_CASE							:						Tuple         =				None
            return state




       def __setstate__(   self     ,    __lowerCamelCase				)      ->  List[str]:
            _SCREAMING_SNAKE_CASE							:						List[Any]         =				d

            # for backward compatibility
            if not hasattr(self     ,    "sp_model_kwargs"				):
                 _SCREAMING_SNAKE_CASE							:						List[str]         =				{}

            _SCREAMING_SNAKE_CASE							:						int         =				spm.SentencePieceProcessor(**self.sp_model_kwargs				)
            self.sp_model.Load(self.vocab_file				)




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase				)      ->  int:
            if self.remove_space:
                 _SCREAMING_SNAKE_CASE							:						int         =				" ".join(inputs.strip().split()				)
            else:
                 _SCREAMING_SNAKE_CASE							:						Any         =				inputs
            _SCREAMING_SNAKE_CASE							:						Tuple         =				outputs.replace("``"     ,    "\""				).replace("''"     ,    "\""				)

            if not self.keep_accents:
                 _SCREAMING_SNAKE_CASE							:						Optional[int]         =				unicodedata.normalize("NFKD"     ,    __lowerCAmelCase				)
                 _SCREAMING_SNAKE_CASE							:						Any         =				"".join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase				)]				)
            if self.do_lower_case:
                 _SCREAMING_SNAKE_CASE							:						List[str]         =				outputs.lower()

            return outputs




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase				)      ->  str:
            _SCREAMING_SNAKE_CASE							:						Any         =				self.preprocess_text(__lowerCAmelCase				)
            _SCREAMING_SNAKE_CASE							:						Tuple         =				self.sp_model.encode(__lowerCAmelCase     ,    out_type=__lowerCAmelCase				)
            _SCREAMING_SNAKE_CASE							:						List[Any]         =				[]
            for piece in pieces:
                 if len(__lowerCAmelCase				) > 1 and piece[-1] == str(","				) and piece[-2].isdigit():
                      _SCREAMING_SNAKE_CASE							:						Optional[Any]         =				self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase     ,    ""				)				)
                      if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                           if len(cur_pieces[0]				) == 1:
                                _SCREAMING_SNAKE_CASE							:						Any         =				cur_pieces[1:]
                           else:
                                _SCREAMING_SNAKE_CASE							:						Optional[int]         =				cur_pieces[0][1:]
                      cur_pieces.append(piece[-1]				)
                      new_pieces.extend(__lowerCAmelCase				)
                 else:
                      new_pieces.append(__lowerCAmelCase				)

            return new_pieces




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase				)      ->  int:
            return self.sp_model.PieceToId(__lowerCAmelCase				)




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase				)      ->  int:
            return self.sp_model.IdToPiece(__lowerCAmelCase				)




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase				)      ->  int:
            _SCREAMING_SNAKE_CASE							:						List[Any]         =				"".join(__lowerCAmelCase				).replace(__lowerCAmelCase     ,    " "				).strip()
            return out_string




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase     ,    __lowerCamelCase = False     ,    __lowerCamelCase = None     ,    __lowerCamelCase = True     ,    **__lowerCamelCase     ,    )      ->  Dict:
            _SCREAMING_SNAKE_CASE							:						List[Any]         =				kwargs.pop("use_source_tokenizer"     ,    __lowerCAmelCase				)

            _SCREAMING_SNAKE_CASE							:						int         =				self.convert_ids_to_tokens(__lowerCAmelCase     ,    skip_special_tokens=__lowerCAmelCase				)

            # To avoid mixing byte-level and unicode for byte-level BPT
            # we need to build string separately for added tokens and byte-level tokens
            # cf. https://github.com/huggingface/transformers/issues/1133
            _SCREAMING_SNAKE_CASE							:						Optional[int]         =				[]
            _SCREAMING_SNAKE_CASE							:						List[str]         =				[]
            for token in filtered_tokens:
                 if skip_special_tokens and token in self.all_special_ids:
                      continue
                 if token in self.added_tokens_encoder:
                      if current_sub_text:
                           sub_texts.append(self.convert_tokens_to_string(__lowerCAmelCase				)				)
                           _SCREAMING_SNAKE_CASE							:						int         =				[]
                      sub_texts.append(__lowerCAmelCase				)
                 else:
                      current_sub_text.append(__lowerCAmelCase				)
            if current_sub_text:
                 sub_texts.append(self.convert_tokens_to_string(__lowerCAmelCase				)				)

            # Mimic the behavior of the Rust tokenizer:
            # By default, there are no spaces between special tokens
            _SCREAMING_SNAKE_CASE							:						int         =				"".join(__lowerCAmelCase				)

            _SCREAMING_SNAKE_CASE							:						Tuple         =				(
                clean_up_tokenization_spaces
                if clean_up_tokenization_spaces is not None
                else self.clean_up_tokenization_spaces
            )
            if clean_up_tokenization_spaces:
                 _SCREAMING_SNAKE_CASE							:						Any         =				self.clean_up_tokenization(__lowerCAmelCase				)
                 return clean_text
            else:
                 return text




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase     ,    __lowerCamelCase = None				)      ->  Optional[int]:
            _SCREAMING_SNAKE_CASE							:						int         =				[self.sep_token_id]
            _SCREAMING_SNAKE_CASE							:						int         =				[self.cls_token_id]
            if token_ids_a is None:
                 return token_ids_a + sep + cls
            return token_ids_a + sep + token_ids_a + sep + cls




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase     ,    __lowerCamelCase = None     ,    __lowerCamelCase = False				)      ->  Dict:

            if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__lowerCAmelCase     ,    token_ids_a=__lowerCAmelCase     ,    already_has_special_tokens=__lowerCAmelCase				)

            if token_ids_a is not None:
                 return ([0] * len(__lowerCAmelCase				)) + [1] + ([0] * len(__lowerCAmelCase				)) + [1, 1]
            return ([0] * len(__lowerCAmelCase				)) + [1, 1]




       def      UpperCamelCase_      (   self     ,    __lowerCamelCase     ,    __lowerCamelCase = None				)      ->  List[Any]:
            _SCREAMING_SNAKE_CASE							:						List[Any]         =				[self.sep_token_id]
            _SCREAMING_SNAKE_CASE							:						Optional[int]         =				[2]

            if token_ids_a is None:
                 return len(token_ids_a + sep				) * [0] + cls_segment_id
            return len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1] + cls_segment_id







       def      UpperCamelCase_      (   self     ,    __lowerCamelCase     ,    __lowerCamelCase = None				)      ->  Dict:
            if not os.path.isdir(__lowerCAmelCase				):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
                 return
            _SCREAMING_SNAKE_CASE							:						Tuple         =				os.path.join(
                __lowerCAmelCase     ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

            if os.path.abspath(self.vocab_file				) != os.path.abspath(__lowerCAmelCase				) and os.path.isfile(self.vocab_file				):
                 copyfile(self.vocab_file     ,    __lowerCAmelCase				)
            elif not os.path.isfile(self.vocab_file				):
                 with open(__lowerCAmelCase     ,    "wb"				) as fi:
                      _SCREAMING_SNAKE_CASE							:						Dict         =				self.sp_model.serialized_model_proto()
                      fi.write(__lowerCAmelCase				)

            return (out_vocab_file,)
"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__			=    logging.get_logger(__name__)


class 							a					(		lowerCAmelCase_ ):
    _snake_case  :	List[Any]  				=				'timm_backbone'


    def __init__(       self						:	List[Any] ,					__lowerCAmelCase						:	str=None ,					__lowerCAmelCase						:	Optional[int]=3 ,					__lowerCAmelCase						:	Any=True ,					__lowerCAmelCase						:	Optional[int]=True ,					__lowerCAmelCase						:	Optional[Any]=None ,					**__lowerCAmelCase						:	Optional[Any] ,					):
      super().__init__(**__lowerCAmelCase       )
      _UpperCAmelCase    			=   backbone
      _UpperCAmelCase    			=   num_channels
      _UpperCAmelCase    			=   features_only
      _UpperCAmelCase    			=   use_pretrained_backbone
      _UpperCAmelCase    			=   True
      _UpperCAmelCase    			=   out_indices if out_indices is not None else (-1,)
