


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


snake_case    :   Dict	    =					logging.get_logger(__name__)

snake_case    :   List[str]	    =					{'''vocab_file''': '''spiece.model'''}

snake_case    :   Any	    =					{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

snake_case    :   List[Any]	    =					{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
snake_case    :   Dict	    =					0
snake_case    :   Union[str, Any]	    =					1
snake_case    :   Any	    =					2
snake_case    :   Optional[Any]	    =					3
snake_case    :   Tuple	    =					4







class 		_snake_case  (				_snake_case							):
 SCREAMING_SNAKE_CASE__				=	VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE__				=	PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE__				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE__				=	'left'




 def __init__(		self ,  _lowerCamelCase ,  _lowerCamelCase=False ,  _lowerCamelCase=True ,  _lowerCamelCase=False ,  _lowerCamelCase="<s>" ,  _lowerCamelCase="</s>" ,  _lowerCamelCase="<unk>" ,  _lowerCamelCase="<sep>" ,  _lowerCamelCase="<pad>" ,  _lowerCamelCase="<cls>" ,  _lowerCamelCase="<mask>" ,  _lowerCamelCase=["<eop>", "<eod>"] ,  _lowerCamelCase = None ,  **_lowerCamelCase ,  ):
       # Mask token behave like a normal word, i.e. include the space before it
       a						:List[Any]										=	AddedToken(_lowerCamelCase ,  lstrip=_lowerCamelCase ,  rstrip=_lowerCamelCase   ) if isinstance(_lowerCamelCase ,  _lowerCamelCase   ) else mask_token

       a						:Union[str, Any]										=	{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=_lowerCamelCase ,  remove_space=_lowerCamelCase ,  keep_accents=_lowerCamelCase ,  bos_token=_lowerCamelCase ,  eos_token=_lowerCamelCase ,  unk_token=_lowerCamelCase ,  sep_token=_lowerCamelCase ,  pad_token=_lowerCamelCase ,  cls_token=_lowerCamelCase ,  mask_token=_lowerCamelCase ,  additional_special_tokens=_lowerCamelCase ,  sp_model_kwargs=self.sp_model_kwargs ,  **_lowerCamelCase ,  )

       a						:Dict										=	3

       a						:Optional[Any]										=	do_lower_case
       a						:List[Any]										=	remove_space
       a						:Tuple										=	keep_accents
       a						:Dict										=	vocab_file

       a						:Optional[int]										=	spm.SentencePieceProcessor(**self.sp_model_kwargs   )
       self.sp_model.Load(_lowerCamelCase   )




 @property
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return len(self.sp_model   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:Optional[int]										=	{self.convert_ids_to_tokens(_lowerCamelCase   ): i for i in range(self.vocab_size   )}
       vocab.update(self.added_tokens_encoder   )
       return vocab




 def __getstate__(		self   ):
       a						:List[str]										=	self.__dict__.copy()
       a						:str										=	None
       return state




 def __setstate__(		self ,  _lowerCamelCase   ):
       a						:List[str]										=	d

       # for backward compatibility
       if not hasattr(self ,  '''sp_model_kwargs'''   ):
             a						:Union[str, Any]										=	{}

       a						:Union[str, Any]										=	spm.SentencePieceProcessor(**self.sp_model_kwargs   )
       self.sp_model.Load(self.vocab_file   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       if self.remove_space:
             a						:Union[str, Any]										=	''' '''.join(inputs.strip().split()   )
       else:
             a						:Union[str, Any]										=	inputs
       a						:List[Any]										=	outputs.replace('''``''' ,  '''"'''   ).replace('''\'\'''' ,  '''"'''   )

       if not self.keep_accents:
             a						:Tuple										=	unicodedata.normalize('''NFKD''' ,  _lowerCamelCase   )
             a						:Dict										=	''''''.join([c for c in outputs if not unicodedata.combining(_lowerCamelCase   )]   )
       if self.do_lower_case:
             a						:Dict										=	outputs.lower()

       return outputs




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       a						:List[str]										=	self.preprocess_text(_lowerCamelCase   )
       a						:Optional[Any]										=	self.sp_model.encode(_lowerCamelCase ,  out_type=_lowerCamelCase   )
       a						:Union[str, Any]										=	[]
       for piece in pieces:
             if len(_lowerCamelCase   ) > 1 and piece[-1] == str(''','''   ) and piece[-2].isdigit():
                   a						:List[Any]										=	self.sp_model.EncodeAsPieces(piece[:-1].replace(_lowerCamelCase ,  ''''''   )   )
                   if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                         if len(cur_pieces[0]   ) == 1:
                               a						:int										=	cur_pieces[1:]
                         else:
                               a						:int										=	cur_pieces[0][1:]
                   cur_pieces.append(piece[-1]   )
                   new_pieces.extend(_lowerCamelCase   )
             else:
                   new_pieces.append(_lowerCamelCase   )

       return new_pieces




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       return self.sp_model.PieceToId(_lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       return self.sp_model.IdToPiece(_lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       a						:Tuple										=	''''''.join(_lowerCamelCase   ).replace(_lowerCamelCase ,  ''' '''   ).strip()
       return out_string




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = False ,  _lowerCamelCase = None ,  _lowerCamelCase = True ,  **_lowerCamelCase ,  ):
       a						:Union[str, Any]										=	kwargs.pop('''use_source_tokenizer''' ,  _lowerCamelCase   )

       a						:Union[str, Any]										=	self.convert_ids_to_tokens(_lowerCamelCase ,  skip_special_tokens=_lowerCamelCase   )

       # To avoid mixing byte-level and unicode for byte-level BPT
       # we need to build string separately for added tokens and byte-level tokens
       # cf. https://github.com/huggingface/transformers/issues/1133
       a						:Dict										=	[]
       a						:Dict										=	[]
       for token in filtered_tokens:
             if skip_special_tokens and token in self.all_special_ids:
                   continue
             if token in self.added_tokens_encoder:
                   if current_sub_text:
                         sub_texts.append(self.convert_tokens_to_string(_lowerCamelCase   )   )
                         a						:Optional[int]										=	[]
                   sub_texts.append(_lowerCamelCase   )
             else:
                   current_sub_text.append(_lowerCamelCase   )
       if current_sub_text:
             sub_texts.append(self.convert_tokens_to_string(_lowerCamelCase   )   )

       # Mimic the behavior of the Rust tokenizer:
       # By default, there are no spaces between special tokens
       a						:Any										=	''''''.join(_lowerCamelCase   )

       a						:Union[str, Any]										=	(
           clean_up_tokenization_spaces
           if clean_up_tokenization_spaces is not None
           else self.clean_up_tokenization_spaces
       )
       if clean_up_tokenization_spaces:
             a						:Optional[Any]										=	self.clean_up_tokenization(_lowerCamelCase   )
             return clean_text
       else:
             return text




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = None   ):
       a						:Any										=	[self.sep_token_id]
       a						:Optional[Any]										=	[self.cls_token_id]
       if token_ids_a is None:
             return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = None ,  _lowerCamelCase = False   ):

       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_lowerCamelCase ,  token_ids_a=_lowerCamelCase ,  already_has_special_tokens=_lowerCamelCase   )

       if token_ids_a is not None:
             return ([0] * len(_lowerCamelCase   )) + [1] + ([0] * len(_lowerCamelCase   )) + [1, 1]
       return ([0] * len(_lowerCamelCase   )) + [1, 1]




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = None   ):
       a						:str										=	[self.sep_token_id]
       a						:List[str]										=	[2]

       if token_ids_a is None:
             return len(token_ids_a + sep   ) * [0] + cls_segment_id
       return len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1] + cls_segment_id




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = None   ):
       if not os.path.isdir(_lowerCamelCase   ):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
             return
       a						:Optional[int]										=	os.path.join(
           _lowerCamelCase ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

       if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowerCamelCase   ) and os.path.isfile(self.vocab_file   ):
             copyfile(self.vocab_file ,  _lowerCamelCase   )
       elif not os.path.isfile(self.vocab_file   ):
             with open(_lowerCamelCase ,  '''wb'''   ) as fi:
                   a						:List[str]										=	self.sp_model.serialized_model_proto()
                   fi.write(_lowerCamelCase   )

       return (out_vocab_file,)









from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


snake_case    :   Optional[Any]	    =					logging.get_logger(__name__)

snake_case    :   Dict	    =					{
    '''facebook/data2vec-vision-base-ft''': (
        '''https://huggingface.co/facebook/data2vec-vision-base-ft/resolve/main/config.json'''
    ),
}







class 		_snake_case  (				_snake_case							):
 SCREAMING_SNAKE_CASE__				=	'data2vec-vision'




 def __init__(		self ,  _lowerCamelCase=768 ,  _lowerCamelCase=12 ,  _lowerCamelCase=12 ,  _lowerCamelCase=3072 ,  _lowerCamelCase="gelu" ,  _lowerCamelCase=0.0 ,  _lowerCamelCase=0.0 ,  _lowerCamelCase=0.02 ,  _lowerCamelCase=1e-12 ,  _lowerCamelCase=224 ,  _lowerCamelCase=16 ,  _lowerCamelCase=3 ,  _lowerCamelCase=False ,  _lowerCamelCase=False ,  _lowerCamelCase=False ,  _lowerCamelCase=False ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=True ,  _lowerCamelCase=[3, 5, 7, 11] ,  _lowerCamelCase=[1, 2, 3, 6] ,  _lowerCamelCase=True ,  _lowerCamelCase=0.4 ,  _lowerCamelCase=256 ,  _lowerCamelCase=1 ,  _lowerCamelCase=False ,  _lowerCamelCase=255 ,  **_lowerCamelCase ,  ):
       super().__init__(**_lowerCamelCase   )

       a						:Tuple										=	hidden_size
       a						:Any										=	num_hidden_layers
       a						:Optional[int]										=	num_attention_heads
       a						:Dict										=	intermediate_size
       a						:List[Any]										=	hidden_act
       a						:List[str]										=	hidden_dropout_prob
       a						:Union[str, Any]										=	attention_probs_dropout_prob
       a						:Any										=	initializer_range
       a						:Any										=	layer_norm_eps

       a						:Union[str, Any]										=	image_size
       a						:int										=	patch_size
       a						:Optional[int]										=	num_channels
       a						:Union[str, Any]										=	use_mask_token
       a						:Optional[Any]										=	use_absolute_position_embeddings
       a						:Tuple										=	use_relative_position_bias
       a						:List[Any]										=	use_shared_relative_position_bias
       a						:Dict										=	layer_scale_init_value
       a						:Optional[int]										=	drop_path_rate
       a						:List[str]										=	use_mean_pooling
       # decode head attributes (semantic segmentation)
       a						:str										=	out_indices
       a						:Tuple										=	pool_scales
       # auxiliary head attributes (semantic segmentation)
       a						:List[Any]										=	use_auxiliary_head
       a						:List[Any]										=	auxiliary_loss_weight
       a						:Optional[int]										=	auxiliary_channels
       a						:List[str]										=	auxiliary_num_convs
       a						:str										=	auxiliary_concat_input
       a						:Union[str, Any]										=	semantic_loss_ignore_index







class 		_snake_case  (				_snake_case							):
 SCREAMING_SNAKE_CASE__				=	version.parse('1.11'							)




 @property
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return OrderedDict(
           [
               ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
           ]   )




 @property
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return 1e-4





