



import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 				__A					( __lowerCamelCase					, unittest.TestCase      ):
   UpperCamelCase					:List[str]     =			RoCBertTokenizer
   UpperCamelCase					:Union[str, Any]     =			None
   UpperCamelCase					:Any     =			False
   UpperCamelCase					:Any     =			True
   UpperCamelCase					:Optional[Any]     =			filter_non_english





   def        _snake_case    (self				):
      super().setUp()

      lowerCamelCase__ :      Any								=		["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
      lowerCamelCase__ :      str								=		{}
      lowerCamelCase__ :      Optional[int]								=		{}
      for i, value in enumerate(SCREAMING_SNAKE_CASE_				):
         lowerCamelCase__ :      Optional[Any]								=		i
         lowerCamelCase__ :      List[str]								=		i
      lowerCamelCase__ :      Any								=		os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES["""vocab_file"""]				)
      lowerCamelCase__ :      int								=		os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES["""word_shape_file"""]				)
      lowerCamelCase__ :      Dict								=		os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES["""word_pronunciation_file"""]				)
      with open(self.vocab_file							,    """w"""							,    encoding="""utf-8"""				) as vocab_writer:
         vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]				)				)
      with open(self.word_shape_file							,    """w"""							,    encoding="""utf-8"""				) as word_shape_writer:
         json.dump(SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_							,    ensure_ascii=SCREAMING_SNAKE_CASE_				)
      with open(self.word_pronunciation_file							,    """w"""							,    encoding="""utf-8"""				) as word_pronunciation_writer:
         json.dump(SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_							,    ensure_ascii=SCREAMING_SNAKE_CASE_				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      int								=		self.tokenizer_class(self.vocab_file							,    self.word_shape_file							,    self.word_pronunciation_file				)

      lowerCamelCase__ :      Dict								=		tokenizer.tokenize("""你好[SEP]你是谁"""				)
      self.assertListEqual(SCREAMING_SNAKE_CASE_							,    ["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]				)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_				)							,    [5, 6, 2, 5, 7, 8]				)
      self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_				)							,    [5, 6, 2, 5, 7, 8]				)
      self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_				)							,    [5, 6, 2, 5, 7, 8]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      int								=		RoCBertBasicTokenizer()

      self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""				)							,    ["""ah""", """\u535A""", """\u63A8""", """zz"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Tuple								=		RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_				)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """				)							,    ["""hello""", """!""", """how""", """are""", """you""", """?"""]				)
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""				)							,    ["""hello"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Optional[int]								=		RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_							,    strip_accents=SCREAMING_SNAKE_CASE_				)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """				)							,    ["""hällo""", """!""", """how""", """are""", """you""", """?"""]				)
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""				)							,    ["""h\u00E9llo"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Optional[int]								=		RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_							,    strip_accents=SCREAMING_SNAKE_CASE_				)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """				)							,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]				)
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""				)							,    ["""hello"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Union[str, Any]								=		RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_				)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """				)							,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]				)
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""				)							,    ["""hello"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Optional[Any]								=		RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_				)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """				)							,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Any								=		RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_							,    strip_accents=SCREAMING_SNAKE_CASE_				)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """				)							,    ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Optional[int]								=		RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_							,    strip_accents=SCREAMING_SNAKE_CASE_				)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """				)							,    ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Dict								=		RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_							,    never_split=["""[UNK]"""]				)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""				)							,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      List[str]								=		["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

      lowerCamelCase__ :      List[str]								=		{}
      for i, token in enumerate(SCREAMING_SNAKE_CASE_				):
         lowerCamelCase__ :      Dict								=		i
      lowerCamelCase__ :      Tuple								=		RoCBertWordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_							,    unk_token="""[UNK]"""				)

      self.assertListEqual(tokenizer.tokenize(""""""				)							,    []				)

      self.assertListEqual(tokenizer.tokenize("""unwanted running"""				)							,    ["""un""", """##want""", """##ed""", """runn""", """##ing"""]				)

      self.assertListEqual(tokenizer.tokenize("""unwantedX running"""				)							,    ["""[UNK]""", """runn""", """##ing"""]				)





   def        _snake_case    (self				):
      self.assertTrue(_is_whitespace(""" """				)				)
      self.assertTrue(_is_whitespace("""\t"""				)				)
      self.assertTrue(_is_whitespace("""\r"""				)				)
      self.assertTrue(_is_whitespace("""\n"""				)				)
      self.assertTrue(_is_whitespace("""\u00A0"""				)				)

      self.assertFalse(_is_whitespace("""A"""				)				)
      self.assertFalse(_is_whitespace("""-"""				)				)





   def        _snake_case    (self				):
      self.assertTrue(_is_control("""\u0005"""				)				)

      self.assertFalse(_is_control("""A"""				)				)
      self.assertFalse(_is_control(""" """				)				)
      self.assertFalse(_is_control("""\t"""				)				)
      self.assertFalse(_is_control("""\r"""				)				)





   def        _snake_case    (self				):
      self.assertTrue(_is_punctuation("""-"""				)				)
      self.assertTrue(_is_punctuation("""$"""				)				)
      self.assertTrue(_is_punctuation("""`"""				)				)
      self.assertTrue(_is_punctuation("""."""				)				)

      self.assertFalse(_is_punctuation("""A"""				)				)
      self.assertFalse(_is_punctuation(""" """				)				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Tuple								=		self.get_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE_				) for t in ["""Test""", """\xad""", """test"""]]							,    [["""[UNK]"""], [], ["""[UNK]"""]]				)

      if self.test_rust_tokenizer:
         lowerCamelCase__ :      Any								=		self.get_rust_tokenizer()
         self.assertListEqual(
             [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_				) for t in ["""Test""", """\xad""", """test"""]]							,    [["""[UNK]"""], [], ["""[UNK]"""]]				)





   def        _snake_case    (self				):
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"				):
            lowerCamelCase__ :      Optional[int]								=		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_							,    **SCREAMING_SNAKE_CASE_				)

            lowerCamelCase__ :      List[Any]								=		f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
            lowerCamelCase__ :      Optional[Any]								=		tokenizer_r.encode_plus(
                SCREAMING_SNAKE_CASE_							,    return_attention_mask=SCREAMING_SNAKE_CASE_							,    return_token_type_ids=SCREAMING_SNAKE_CASE_							,    return_offsets_mapping=SCREAMING_SNAKE_CASE_							,    add_special_tokens=SCREAMING_SNAKE_CASE_							,    )

            lowerCamelCase__ :      List[str]								=		tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE_							,    """do_lower_case"""				) else False
            lowerCamelCase__ :      Tuple								=		(
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), """A"""),
                    ((1, 2), ""","""),
                    ((3, 5), """na"""),
                    ((5, 6), """##ï"""),
                    ((6, 8), """##ve"""),
                    ((9, 15), tokenizer_r.mask_token),
                    ((16, 21), """Allen"""),
                    ((21, 23), """##NL"""),
                    ((23, 24), """##P"""),
                    ((25, 33), """sentence"""),
                    ((33, 34), """."""),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), """a"""),
                    ((1, 2), ""","""),
                    ((3, 8), """naive"""),
                    ((9, 15), tokenizer_r.mask_token),
                    ((16, 21), """allen"""),
                    ((21, 23), """##nl"""),
                    ((23, 24), """##p"""),
                    ((25, 33), """sentence"""),
                    ((33, 34), """."""),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]							,    tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]				)				)
            self.assertEqual([e[0] for e in expected_results]							,    tokens["""offset_mapping"""]				)





   def        _snake_case    (self				):
      lowerCamelCase__ :      Optional[int]								=		["""的""", """人""", """有"""]
      lowerCamelCase__ :      Tuple								=		"""""".join(SCREAMING_SNAKE_CASE_				)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"				):
            lowerCamelCase__ :      str								=		True
            lowerCamelCase__ :      Union[str, Any]								=		self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_							,    **SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      str								=		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_							,    **SCREAMING_SNAKE_CASE_				)

            lowerCamelCase__ :      Optional[int]								=		tokenizer_p.encode(SCREAMING_SNAKE_CASE_							,    add_special_tokens=SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      int								=		tokenizer_r.encode(SCREAMING_SNAKE_CASE_							,    add_special_tokens=SCREAMING_SNAKE_CASE_				)

            lowerCamelCase__ :      Dict								=		tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      List[Any]								=		tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_				)

            # it is expected that each Chinese character is not preceded by "##"
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_				)
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_				)

            lowerCamelCase__ :      List[Any]								=		False
            lowerCamelCase__ :      Dict								=		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_							,    **SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      Tuple								=		self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_							,    **SCREAMING_SNAKE_CASE_				)

            lowerCamelCase__ :      Union[str, Any]								=		tokenizer_r.encode(SCREAMING_SNAKE_CASE_							,    add_special_tokens=SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      Tuple								=		tokenizer_p.encode(SCREAMING_SNAKE_CASE_							,    add_special_tokens=SCREAMING_SNAKE_CASE_				)

            lowerCamelCase__ :      Optional[Any]								=		tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      Any								=		tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_				)

            # it is expected that only the first Chinese character is not preceded by "##".
            lowerCamelCase__ :      List[str]								=		[
                f"##{token}" if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE_				)
            ]
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_				)
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_				)





   @slow
   def        _snake_case    (self				):
      lowerCamelCase__ :      List[Any]								=		self.tokenizer_class(self.vocab_file							,    self.word_shape_file							,    self.word_pronunciation_file				)

      lowerCamelCase__ :      Dict								=		tokenizer.encode("""你好"""							,    add_special_tokens=SCREAMING_SNAKE_CASE_				)
      lowerCamelCase__ :      Dict								=		tokenizer.encode("""你是谁"""							,    add_special_tokens=SCREAMING_SNAKE_CASE_				)

      lowerCamelCase__ :      Optional[Any]								=		tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_				)
      lowerCamelCase__ :      Tuple								=		tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_				)

      assert encoded_sentence == [1] + text + [2]
      assert encoded_pair == [1] + text + [2] + text_a + [2]





   def        _snake_case    (self				):
      lowerCamelCase__ :      Optional[Any]								=		self.get_tokenizers(do_lower_case=SCREAMING_SNAKE_CASE_				)
      for tokenizer in tokenizers:
         with self.subTest(f"{tokenizer.__class__.__name__}"				):
            lowerCamelCase__ :      Union[str, Any]								=		"""你好，你是谁"""
            lowerCamelCase__ :      Dict								=		tokenizer.tokenize(SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      Optional[Any]								=		tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      Any								=		tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      Union[str, Any]								=		tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_				)
            lowerCamelCase__ :      List[Any]								=		tokenizer.prepare_for_model(
                SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_							,    add_special_tokens=SCREAMING_SNAKE_CASE_				)

            lowerCamelCase__ :      str								=		tokenizer.encode_plus(SCREAMING_SNAKE_CASE_							,    add_special_tokens=SCREAMING_SNAKE_CASE_				)

            self.assertEqual(SCREAMING_SNAKE_CASE_							,    SCREAMING_SNAKE_CASE_				)








import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, Features, Value
from .base import TaskTemplate
@dataclass(frozen=A_      )
class 				__A					( A_      ):
			UpperCamelCase					:str     =			field(default='''automatic-speech-recognition'''					, metadata={'''include_in_asdict_even_if_is_default''': True}      )
			UpperCamelCase					:ClassVar[Features]     =			Features({'''audio''': Audio()}      )
			UpperCamelCase					:ClassVar[Features]     =			Features({'''transcription''': Value('''string'''      )}      )
			UpperCamelCase					:str     =			"audio"
			UpperCamelCase					:str     =			"transcription"





			def        _snake_case    (self							,    __magic_name__				):
						if self.audio_column not in features:
									raise ValueError(f"Column {self.audio_column} is not present in features."				)
						if not isinstance(features[self.audio_column]							,    __magic_name__				):
									raise ValueError(f"Column {self.audio_column} is not an Audio type."				)
						lowerCamelCase__ :      int								=		copy.deepcopy(self				)
						lowerCamelCase__ :      Tuple								=		self.input_schema.copy()
						lowerCamelCase__ :      Tuple								=		features[self.audio_column]
						lowerCamelCase__ :      List[str]								=		input_schema
						return task_template





			@property
			def        _snake_case    (self				):
						return {self.audio_column: "audio", self.transcription_column: "transcription"}



