



'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case_			=     logging.get_logger(__name__)


snake_case_			=     '''▁'''

snake_case_			=     {'''vocab_file''': '''spiece.model'''}

snake_case_			=     {
    '''vocab_file''': {
        '''google/reformer-crime-and-punishment''': (
            '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'''
        )
    }
}

snake_case_			=     {
    '''google/reformer-crime-and-punishment''': 524288,
}


class 	a__     (				__UpperCAmelCase	):
	__magic_name__		:    List[Any]	    =     VOCAB_FILES_NAMES
	__magic_name__		:    Optional[Any]	    =     PRETRAINED_VOCAB_FILES_MAP
	__magic_name__		:    Tuple	    =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__magic_name__		:    Tuple	    =     ['''input_ids''', '''attention_mask''']


	def __init__(self : int,					__UpperCAmelCase : List[Any],					__UpperCAmelCase : Any="</s>",					__UpperCAmelCase : Optional[int]="<unk>",					__UpperCAmelCase : List[str]=[],					__UpperCAmelCase : Optional[Dict[str, Any]] = None,					**__UpperCAmelCase : Tuple,					)     ->			Optional[Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[str]      					= {} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    eos_token=__SCREAMING_SNAKE_CASE,					unk_token=__SCREAMING_SNAKE_CASE,					additional_special_tokens=__SCREAMING_SNAKE_CASE,					sp_model_kwargs=self.sp_model_kwargs,					**__SCREAMING_SNAKE_CASE,					)

					SCREAMING_SNAKE_CASE  :						List[str]      					= vocab_file
					SCREAMING_SNAKE_CASE  :						str      					= spm.SentencePieceProcessor(**self.sp_model_kwargs      )
					self.sp_model.Load(__SCREAMING_SNAKE_CASE      )


	@property
	def 				lowercase__  (self : Optional[int]      )     ->			Tuple:

					"""simple docstring"""

					return self.sp_model.get_piece_size()


	def 				lowercase__  (self : Dict      )     ->			Union[str, Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Any      					= {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE      ): i for i in range(self.vocab_size      )}
					vocab.update(self.added_tokens_encoder      )
					return vocab


	def __getstate__(self : Union[str, Any]      )     ->			Union[str, Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						str      					= self.__dict__.copy()
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= None
					return state


	def __setstate__(self : str,					__UpperCAmelCase : Optional[Any]      )     ->			List[Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[str]      					= d

					# for backward compatibility
					if not hasattr(self,					'''sp_model_kwargs'''      ):
									SCREAMING_SNAKE_CASE  :						Dict      					= {}

					SCREAMING_SNAKE_CASE  :						str      					= spm.SentencePieceProcessor(**self.sp_model_kwargs      )
					self.sp_model.Load(self.vocab_file      )


	def 				lowercase__  (self : Optional[Any],					__UpperCAmelCase : str      )     ->			Dict:

					"""simple docstring"""

					return self.sp_model.encode(__SCREAMING_SNAKE_CASE,					out_type=__SCREAMING_SNAKE_CASE      )


	def 				lowercase__  (self : Union[str, Any],					__UpperCAmelCase : Union[str, Any]      )     ->			str:

					"""simple docstring"""

					return self.sp_model.piece_to_id(__SCREAMING_SNAKE_CASE      )


	def 				lowercase__  (self : str,					__UpperCAmelCase : Dict      )     ->			str:

					"""simple docstring"""

					if index < self.sp_model.get_piece_size():
									SCREAMING_SNAKE_CASE  :						Dict      					= self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE      )
					return token


	def 				lowercase__  (self : List[Any],					__UpperCAmelCase : int      )     ->			Optional[int]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= []
					SCREAMING_SNAKE_CASE  :						str      					= ''''''
					for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
													out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE      ) + token
													SCREAMING_SNAKE_CASE  :						str      					= []
									else:
													current_sub_tokens.append(__SCREAMING_SNAKE_CASE      )
					out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE      )
					return out_string.strip()






	def 				lowercase__  (self : Tuple,					__UpperCAmelCase : str,					__UpperCAmelCase : Optional[str] = None      )     ->			Any:

					"""simple docstring"""

					if not os.path.isdir(__SCREAMING_SNAKE_CASE      ):
									logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
									return
					SCREAMING_SNAKE_CASE  :						List[Any]      					= os.path.join(
					    __SCREAMING_SNAKE_CASE,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

					if os.path.abspath(self.vocab_file      ) != os.path.abspath(__SCREAMING_SNAKE_CASE      ) and os.path.isfile(self.vocab_file      ):
									copyfile(self.vocab_file,					__SCREAMING_SNAKE_CASE      )
					elif not os.path.isfile(self.vocab_file      ):
									with open(__SCREAMING_SNAKE_CASE,					'''wb'''      ) as fi:
													SCREAMING_SNAKE_CASE  :						Any      					= self.sp_model.serialized_model_proto()
													fi.write(__SCREAMING_SNAKE_CASE      )

					return (out_vocab_file,)









'''simple docstring'''

import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_			=     logging.get_logger(__name__)

snake_case_			=     {
    """facebook/wav2vec2-base-960h""": """https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/config.json""",
    # See all Wav2Vec2 models at https://huggingface.co/models?filter=wav2vec2
}


class 	a__     (				_lowercase	):
	__magic_name__		:    Dict	    =     "wav2vec2"


	def __init__(self : Tuple,					__UpperCAmelCase : Tuple=32,					__UpperCAmelCase : int=768,					__UpperCAmelCase : Optional[Any]=12,					__UpperCAmelCase : Any=12,					__UpperCAmelCase : Any=3072,					__UpperCAmelCase : Optional[int]="gelu",					__UpperCAmelCase : Dict=0.1,					__UpperCAmelCase : List[str]=0.1,					__UpperCAmelCase : List[Any]=0.1,					__UpperCAmelCase : Union[str, Any]=0.0,					__UpperCAmelCase : str=0.0,					__UpperCAmelCase : Optional[Any]=0.1,					__UpperCAmelCase : Union[str, Any]=0.1,					__UpperCAmelCase : Union[str, Any]=0.02,					__UpperCAmelCase : Dict=1e-5,					__UpperCAmelCase : List[str]="group",					__UpperCAmelCase : str="gelu",					__UpperCAmelCase : str=(512, 512, 512, 512, 512, 512, 512),					__UpperCAmelCase : str=(5, 2, 2, 2, 2, 2, 2),					__UpperCAmelCase : Union[str, Any]=(10, 3, 3, 3, 3, 2, 2),					__UpperCAmelCase : Any=False,					__UpperCAmelCase : Optional[Any]=128,					__UpperCAmelCase : Tuple=16,					__UpperCAmelCase : str=False,					__UpperCAmelCase : int=True,					__UpperCAmelCase : Tuple=0.05,					__UpperCAmelCase : List[Any]=10,					__UpperCAmelCase : Union[str, Any]=2,					__UpperCAmelCase : Optional[Any]=0.0,					__UpperCAmelCase : List[Any]=10,					__UpperCAmelCase : Dict=0,					__UpperCAmelCase : Any=320,					__UpperCAmelCase : int=2,					__UpperCAmelCase : Dict=0.1,					__UpperCAmelCase : Tuple=100,					__UpperCAmelCase : Any=256,					__UpperCAmelCase : Tuple=256,					__UpperCAmelCase : str=0.1,					__UpperCAmelCase : Any="sum",					__UpperCAmelCase : Union[str, Any]=False,					__UpperCAmelCase : str=False,					__UpperCAmelCase : str=256,					__UpperCAmelCase : Union[str, Any]=(512, 512, 512, 512, 1500),					__UpperCAmelCase : Optional[int]=(5, 3, 3, 1, 1),					__UpperCAmelCase : Dict=(1, 2, 3, 1, 1),					__UpperCAmelCase : List[str]=512,					__UpperCAmelCase : Union[str, Any]=0,					__UpperCAmelCase : List[Any]=1,					__UpperCAmelCase : List[str]=2,					__UpperCAmelCase : List[Any]=False,					__UpperCAmelCase : str=3,					__UpperCAmelCase : Tuple=2,					__UpperCAmelCase : str=3,					__UpperCAmelCase : int=None,					__UpperCAmelCase : List[Any]=None,					**__UpperCAmelCase : int,					)     ->			Tuple:

					"""simple docstring"""

					super().__init__(**__UpperCAmelCase,					pad_token_id=__UpperCAmelCase,					bos_token_id=__UpperCAmelCase,					eos_token_id=__UpperCAmelCase      )
					SCREAMING_SNAKE_CASE  :						Any      					= hidden_size
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= feat_extract_norm
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= feat_extract_activation
					SCREAMING_SNAKE_CASE  :						Dict      					= list(__UpperCAmelCase      )
					SCREAMING_SNAKE_CASE  :						List[str]      					= list(__UpperCAmelCase      )
					SCREAMING_SNAKE_CASE  :						Tuple      					= list(__UpperCAmelCase      )
					SCREAMING_SNAKE_CASE  :						Dict      					= conv_bias
					SCREAMING_SNAKE_CASE  :						Dict      					= num_conv_pos_embeddings
					SCREAMING_SNAKE_CASE  :						List[Any]      					= num_conv_pos_embedding_groups
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= len(self.conv_dim      )
					SCREAMING_SNAKE_CASE  :						Tuple      					= num_hidden_layers
					SCREAMING_SNAKE_CASE  :						str      					= intermediate_size
					SCREAMING_SNAKE_CASE  :						Any      					= hidden_act
					SCREAMING_SNAKE_CASE  :						str      					= num_attention_heads
					SCREAMING_SNAKE_CASE  :						int      					= hidden_dropout
					SCREAMING_SNAKE_CASE  :						Any      					= attention_dropout
					SCREAMING_SNAKE_CASE  :						int      					= activation_dropout
					SCREAMING_SNAKE_CASE  :						int      					= feat_proj_dropout
					SCREAMING_SNAKE_CASE  :						List[str]      					= final_dropout
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= layerdrop
					SCREAMING_SNAKE_CASE  :						Tuple      					= layer_norm_eps
					SCREAMING_SNAKE_CASE  :						Dict      					= initializer_range
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= vocab_size
					SCREAMING_SNAKE_CASE  :						Tuple      					= do_stable_layer_norm
					SCREAMING_SNAKE_CASE  :						Dict      					= use_weighted_layer_sum

					if (
					    (len(self.conv_stride      ) != self.num_feat_extract_layers)
					    or (len(self.conv_kernel      ) != self.num_feat_extract_layers)
					    or (len(self.conv_dim      ) != self.num_feat_extract_layers)
					):
									raise ValueError(
									    '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
									    ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
									    F''' {len(self.conv_dim      )}`, `len(config.conv_stride) = {len(self.conv_stride      )}`,'''
									    F''' `len(config.conv_kernel) = {len(self.conv_kernel      )}`.'''      )

					# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
					SCREAMING_SNAKE_CASE  :						str      					= apply_spec_augment
					SCREAMING_SNAKE_CASE  :						Dict      					= mask_time_prob
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= mask_time_length
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= mask_time_min_masks
					SCREAMING_SNAKE_CASE  :						Dict      					= mask_feature_prob
					SCREAMING_SNAKE_CASE  :						Tuple      					= mask_feature_length
					SCREAMING_SNAKE_CASE  :						Tuple      					= mask_feature_min_masks

					# parameters for pretraining with codevector quantized representations
					SCREAMING_SNAKE_CASE  :						Any      					= num_codevectors_per_group
					SCREAMING_SNAKE_CASE  :						int      					= num_codevector_groups
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= contrastive_logits_temperature
					SCREAMING_SNAKE_CASE  :						List[Any]      					= feat_quantizer_dropout
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= num_negatives
					SCREAMING_SNAKE_CASE  :						Any      					= codevector_dim
					SCREAMING_SNAKE_CASE  :						str      					= proj_codevector_dim
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= diversity_loss_weight

					# ctc loss
					SCREAMING_SNAKE_CASE  :						Tuple      					= ctc_loss_reduction
					SCREAMING_SNAKE_CASE  :						Any      					= ctc_zero_infinity

					# adapter
					SCREAMING_SNAKE_CASE  :						int      					= add_adapter
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= adapter_kernel_size
					SCREAMING_SNAKE_CASE  :						int      					= adapter_stride
					SCREAMING_SNAKE_CASE  :						List[str]      					= num_adapter_layers
					SCREAMING_SNAKE_CASE  :						Tuple      					= output_hidden_size or hidden_size
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= adapter_attn_dim

					# SequenceClassification-specific parameter. Feel free to ignore for other classes.
					SCREAMING_SNAKE_CASE  :						Dict      					= classifier_proj_size

					# XVector-specific parameters. Feel free to ignore for other classes.
					SCREAMING_SNAKE_CASE  :						int      					= list(__UpperCAmelCase      )
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= list(__UpperCAmelCase      )
					SCREAMING_SNAKE_CASE  :						int      					= list(__UpperCAmelCase      )
					SCREAMING_SNAKE_CASE  :						Dict      					= xvector_output_dim






	@property
	def 				lowercase__  (self : Tuple      )     ->			Optional[Any]:

					"""simple docstring"""

					return functools.reduce(operator.mul,					self.conv_stride,					1      )




