



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_lowercase     							=						logging.get_logger(__name__)

_lowercase     							=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_lowercase     							=						{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

_lowercase     							=						{
    'junnyu/roformer_chinese_small': 15_36,
    'junnyu/roformer_chinese_base': 15_36,
    'junnyu/roformer_chinese_char_small': 5_12,
    'junnyu/roformer_chinese_char_base': 5_12,
    'junnyu/roformer_small_discriminator': 1_28,
    'junnyu/roformer_small_generator': 1_28,
}


_lowercase     							=						{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}
class 				__A					( A_      ):
   UpperCamelCase					:Union[str, Any]     =			VOCAB_FILES_NAMES
   UpperCamelCase					:List[Any]     =			PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase					:str     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase					:str     =			PRETRAINED_INIT_CONFIGURATION
   UpperCamelCase					:List[str]     =			RoFormerTokenizer





   def __init__(self							,    __magic_name__=None							,    __magic_name__=None							,    __magic_name__=True							,    __magic_name__="[UNK]"							,    __magic_name__="[SEP]"							,    __magic_name__="[PAD]"							,    __magic_name__="[CLS]"							,    __magic_name__="[MASK]"							,    __magic_name__=True							,    __magic_name__=None							,    **__magic_name__							,    ):
      super().__init__(
          A__							,    tokenizer_file=A__							,    do_lower_case=A__							,    unk_token=A__							,    sep_token=A__							,    pad_token=A__							,    cls_token=A__							,    mask_token=A__							,    tokenize_chinese_chars=A__							,    strip_accents=A__							,    **A__							,    )

      lowerCamelCase__ :      str								=		json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
      if (
          pre_tok_state.get("""lowercase"""							,    A__				) != do_lower_case
          or pre_tok_state.get("""strip_accents"""							,    A__				) != strip_accents
      ):
         lowerCamelCase__ :      Dict								=		getattr(A__							,    pre_tok_state.pop("""type"""				)				)
         lowerCamelCase__ :      int								=		do_lower_case
         lowerCamelCase__ :      str								=		strip_accents
         lowerCamelCase__ :      int								=		pre_tok_class(**A__				)

      lowerCamelCase__ :      List[str]								=		do_lower_case





   def __getstate__(self				):
      lowerCamelCase__ :      Union[str, Any]								=		self.__dict__.copy()
      lowerCamelCase__ :      Tuple								=		BertPreTokenizer()
      return state





   def __setstate__(self							,    __magic_name__				):
      lowerCamelCase__ :      List[Any]								=		d
      lowerCamelCase__ :      str								=		self.__dict__["""_tokenizer"""].get_vocab()
      lowerCamelCase__ :      Optional[int]								=		PreTokenizer.custom(JiebaPreTokenizer(A__				)				)





   def        _snake_case    (self							,    __magic_name__							,    __magic_name__=None				):
      lowerCamelCase__ :      Tuple								=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

      return output





   def        _snake_case    (self							,    __magic_name__							,    __magic_name__ = None				):
      lowerCamelCase__ :      Dict								=		[self.sep_token_id]
      lowerCamelCase__ :      Tuple								=		[self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep				) * [0]
      return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





   def        _snake_case    (self							,    __magic_name__							,    __magic_name__ = None				):
      lowerCamelCase__ :      str								=		self._tokenizer.model.save(A__							,    name=A__				)
      return tuple(A__				)





   def        _snake_case    (self							,    __magic_name__							,    __magic_name__=None							,    __magic_name__=None							,    __magic_name__=False							,    **__magic_name__							,    ):
      lowerCamelCase__ :      str								=		BertPreTokenizer()
      return super().save_pretrained(A__							,    A__							,    A__							,    A__							,    **A__				)








import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, Features, Value
from .base import TaskTemplate
@dataclass(frozen=A_      )
class 				__A					( A_      ):
			UpperCamelCase					:str     =			field(default='''automatic-speech-recognition'''					, metadata={'''include_in_asdict_even_if_is_default''': True}      )
			UpperCamelCase					:ClassVar[Features]     =			Features({'''audio''': Audio()}      )
			UpperCamelCase					:ClassVar[Features]     =			Features({'''transcription''': Value('''string'''      )}      )
			UpperCamelCase					:str     =			"audio"
			UpperCamelCase					:str     =			"transcription"





			def        _snake_case    (self							,    __magic_name__				):
						if self.audio_column not in features:
									raise ValueError(f"Column {self.audio_column} is not present in features."				)
						if not isinstance(features[self.audio_column]							,    __magic_name__				):
									raise ValueError(f"Column {self.audio_column} is not an Audio type."				)
						lowerCamelCase__ :      int								=		copy.deepcopy(self				)
						lowerCamelCase__ :      Tuple								=		self.input_schema.copy()
						lowerCamelCase__ :      Tuple								=		features[self.audio_column]
						lowerCamelCase__ :      List[str]								=		input_schema
						return task_template





			@property
			def        _snake_case    (self				):
						return {self.audio_column: "audio", self.transcription_column: "transcription"}



