



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


_lowercase     							=						logging.get_logger(__name__)

_lowercase     							=						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowercase     							=						{
    '''vocab_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt'''
        ),
        '''squeezebert/squeezebert-mnli''': '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt''',
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json'''
        ),
    },
}

_lowercase     							=						{
    '''squeezebert/squeezebert-uncased''': 5_12,
    '''squeezebert/squeezebert-mnli''': 5_12,
    '''squeezebert/squeezebert-mnli-headless''': 5_12,
}


_lowercase     							=						{
    '''squeezebert/squeezebert-uncased''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli-headless''': {'''do_lower_case''': True},
}
class 				__A					( A_      ):
			UpperCamelCase					:int     =			VOCAB_FILES_NAMES
			UpperCamelCase					:Union[str, Any]     =			PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase					:Union[str, Any]     =			PRETRAINED_INIT_CONFIGURATION
			UpperCamelCase					:List[str]     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase					:Tuple     =			SqueezeBertTokenizer





			def __init__(self							,    __magic_name__=None							,    __magic_name__=None							,    __magic_name__=True							,    __magic_name__="[UNK]"							,    __magic_name__="[SEP]"							,    __magic_name__="[PAD]"							,    __magic_name__="[CLS]"							,    __magic_name__="[MASK]"							,    __magic_name__=True							,    __magic_name__=None							,    **__magic_name__							,    ):
						super().__init__(
						    __magic_name__							,    tokenizer_file=__magic_name__							,    do_lower_case=__magic_name__							,    unk_token=__magic_name__							,    sep_token=__magic_name__							,    pad_token=__magic_name__							,    cls_token=__magic_name__							,    mask_token=__magic_name__							,    tokenize_chinese_chars=__magic_name__							,    strip_accents=__magic_name__							,    **__magic_name__							,    )

						lowerCamelCase__ :      List[Any]								=		json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
						if (
						    normalizer_state.get("""lowercase"""							,    __magic_name__				) != do_lower_case
						    or normalizer_state.get("""strip_accents"""							,    __magic_name__				) != strip_accents
						    or normalizer_state.get("""handle_chinese_chars"""							,    __magic_name__				) != tokenize_chinese_chars
						):
									lowerCamelCase__ :      str								=		getattr(__magic_name__							,    normalizer_state.pop("""type"""				)				)
									lowerCamelCase__ :      Optional[Any]								=		do_lower_case
									lowerCamelCase__ :      Any								=		strip_accents
									lowerCamelCase__ :      Union[str, Any]								=		tokenize_chinese_chars
									lowerCamelCase__ :      Union[str, Any]								=		normalizer_class(**__magic_name__				)

						lowerCamelCase__ :      str								=		do_lower_case





			def        _snake_case    (self							,    __magic_name__							,    __magic_name__=None				):
						lowerCamelCase__ :      Optional[int]								=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output





			def        _snake_case    (self							,    __magic_name__							,    __magic_name__ = None				):
						lowerCamelCase__ :      List[Any]								=		[self.sep_token_id]
						lowerCamelCase__ :      int								=		[self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep				) * [0]
						return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





			def        _snake_case    (self							,    __magic_name__							,    __magic_name__ = None				):
						lowerCamelCase__ :      Optional[int]								=		self._tokenizer.model.save(__magic_name__							,    name=__magic_name__				)
						return tuple(__magic_name__				)








import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, Features, Value
from .base import TaskTemplate
@dataclass(frozen=A_      )
class 				__A					( A_      ):
			UpperCamelCase					:str     =			field(default='''automatic-speech-recognition'''					, metadata={'''include_in_asdict_even_if_is_default''': True}      )
			UpperCamelCase					:ClassVar[Features]     =			Features({'''audio''': Audio()}      )
			UpperCamelCase					:ClassVar[Features]     =			Features({'''transcription''': Value('''string'''      )}      )
			UpperCamelCase					:str     =			"audio"
			UpperCamelCase					:str     =			"transcription"





			def        _snake_case    (self							,    __magic_name__				):
						if self.audio_column not in features:
									raise ValueError(f"Column {self.audio_column} is not present in features."				)
						if not isinstance(features[self.audio_column]							,    __magic_name__				):
									raise ValueError(f"Column {self.audio_column} is not an Audio type."				)
						lowerCamelCase__ :      int								=		copy.deepcopy(self				)
						lowerCamelCase__ :      Tuple								=		self.input_schema.copy()
						lowerCamelCase__ :      Tuple								=		features[self.audio_column]
						lowerCamelCase__ :      List[str]								=		input_schema
						return task_template





			@property
			def        _snake_case    (self				):
						return {self.audio_column: "audio", self.transcription_column: "transcription"}



