






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class       snake_case__ (				lowercase_						,     unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase				:					Optional[Any]								=			RoCBertTokenizer
       lowerCamelCase				:					Dict								=			None
       lowerCamelCase				:					Tuple								=			False
       lowerCamelCase				:					Optional[Any]								=			True
       lowerCamelCase				:					Any								=			filter_non_english



       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            super().setUp()

            __snake_case      :Any 							=    ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
            __snake_case      :Dict 							=    {}
            __snake_case      :Dict 							=    {}
            for i, value in enumerate(a__     ):
                 __snake_case      :int 							=    i
                 __snake_case      :Any 							=    i
            __snake_case      :List[Any] 							=    os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES["""vocab_file"""]     )
            __snake_case      :str 							=    os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES["""word_shape_file"""]     )
            __snake_case      :Optional[Any] 							=    os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES["""word_pronunciation_file"""]     )
            with open(self.vocab_file      ,    """w"""      ,    encoding="""utf-8"""     ) as vocab_writer:
                 vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )
            with open(self.word_shape_file      ,    """w"""      ,    encoding="""utf-8"""     ) as word_shape_writer:
                 json.dump(a__      ,    a__      ,    ensure_ascii=a__     )
            with open(self.word_pronunciation_file      ,    """w"""      ,    encoding="""utf-8"""     ) as word_pronunciation_writer:
                 json.dump(a__      ,    a__      ,    ensure_ascii=a__     )



       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :Optional[Any] 							=    self.tokenizer_class(self.vocab_file      ,    self.word_shape_file      ,    self.word_pronunciation_file     )

            __snake_case      :List[str] 							=    tokenizer.tokenize("""你好[SEP]你是谁"""     )
            self.assertListEqual(a__      ,    ["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]     )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__     )      ,    [5, 6, 2, 5, 7, 8]     )
            self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(a__     )      ,    [5, 6, 2, 5, 7, 8]     )
            self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(a__     )      ,    [5, 6, 2, 5, 7, 8]     )



       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :List[str] 							=    RoCBertBasicTokenizer()

            self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""     )      ,    ["""ah""", """\u535A""", """\u63A8""", """zz"""]     )



       def        __lowercase    (				self     )			->   Dict:






            '''simple docstring'''





            __snake_case      :int 							=    RoCBertBasicTokenizer(do_lower_case=a__     )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )      ,    ["""hello""", """!""", """how""", """are""", """you""", """?"""]     )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )      ,    ["""hello"""]     )



       def        __lowercase    (				self     )			->   Any:






            '''simple docstring'''





            __snake_case      :Tuple 							=    RoCBertBasicTokenizer(do_lower_case=a__      ,    strip_accents=a__     )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )      ,    ["""hällo""", """!""", """how""", """are""", """you""", """?"""]     )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )      ,    ["""h\u00E9llo"""]     )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :int 							=    RoCBertBasicTokenizer(do_lower_case=a__      ,    strip_accents=a__     )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )      ,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )      ,    ["""hello"""]     )



       def        __lowercase    (				self     )			->   Any:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    RoCBertBasicTokenizer(do_lower_case=a__     )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )      ,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )      ,    ["""hello"""]     )



       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    RoCBertBasicTokenizer(do_lower_case=a__     )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )      ,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    RoCBertBasicTokenizer(do_lower_case=a__      ,    strip_accents=a__     )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )      ,    ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )



       def        __lowercase    (				self     )			->   Union[str, Any]:






            '''simple docstring'''





            __snake_case      :Optional[Any] 							=    RoCBertBasicTokenizer(do_lower_case=a__      ,    strip_accents=a__     )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )      ,    ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )



       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            __snake_case      :Tuple 							=    RoCBertBasicTokenizer(do_lower_case=a__      ,    never_split=["""[UNK]"""]     )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""     )      ,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]     )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

            __snake_case      :Optional[int] 							=    {}
            for i, token in enumerate(a__     ):
                 __snake_case      :int 							=    i
            __snake_case      :Optional[int] 							=    RoCBertWordpieceTokenizer(vocab=a__      ,    unk_token="""[UNK]"""     )

            self.assertListEqual(tokenizer.tokenize(""""""     )      ,    []     )

            self.assertListEqual(tokenizer.tokenize("""unwanted running"""     )      ,    ["""un""", """##want""", """##ed""", """runn""", """##ing"""]     )

            self.assertListEqual(tokenizer.tokenize("""unwantedX running"""     )      ,    ["""[UNK]""", """runn""", """##ing"""]     )



       def        __lowercase    (				self     )			->   Any:






            '''simple docstring'''





            self.assertTrue(_is_whitespace(""" """     )     )
            self.assertTrue(_is_whitespace("""\t"""     )     )
            self.assertTrue(_is_whitespace("""\r"""     )     )
            self.assertTrue(_is_whitespace("""\n"""     )     )
            self.assertTrue(_is_whitespace("""\u00A0"""     )     )

            self.assertFalse(_is_whitespace("""A"""     )     )
            self.assertFalse(_is_whitespace("""-"""     )     )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            self.assertTrue(_is_control("""\u0005"""     )     )

            self.assertFalse(_is_control("""A"""     )     )
            self.assertFalse(_is_control(""" """     )     )
            self.assertFalse(_is_control("""\t"""     )     )
            self.assertFalse(_is_control("""\r"""     )     )



       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            self.assertTrue(_is_punctuation("""-"""     )     )
            self.assertTrue(_is_punctuation("""$"""     )     )
            self.assertTrue(_is_punctuation("""`"""     )     )
            self.assertTrue(_is_punctuation("""."""     )     )

            self.assertFalse(_is_punctuation("""A"""     )     )
            self.assertFalse(_is_punctuation(""" """     )     )



       def        __lowercase    (				self     )			->   Union[str, Any]:






            '''simple docstring'''





            __snake_case      :List[str] 							=    self.get_tokenizer()

            # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
            self.assertListEqual([tokenizer.tokenize(a__     ) for t in ["""Test""", """\xad""", """test"""]]      ,    [["""[UNK]"""], [], ["""[UNK]"""]]     )

            if self.test_rust_tokenizer:
                 __snake_case      :Any 							=    self.get_rust_tokenizer()
                 self.assertListEqual(
                     [rust_tokenizer.tokenize(a__     ) for t in ["""Test""", """\xad""", """test"""]]      ,    [["""[UNK]"""], [], ["""[UNK]"""]]     )



       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
                      __snake_case      :Any 							=    self.rust_tokenizer_class.from_pretrained(a__      ,    **a__     )

                      __snake_case      :Union[str, Any] 							=    F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                      __snake_case      :Optional[int] 							=    tokenizer_r.encode_plus(
                          a__      ,    return_attention_mask=a__      ,    return_token_type_ids=a__      ,    return_offsets_mapping=a__      ,    add_special_tokens=a__      ,    )

                      __snake_case      :List[Any] 							=    tokenizer_r.do_lower_case if hasattr(a__      ,    """do_lower_case"""     ) else False
                      __snake_case      :Dict 							=    (
                          [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), """A"""),
                              ((1, 2), ""","""),
                              ((3, 5), """na"""),
                              ((5, 6), """##ï"""),
                              ((6, 8), """##ve"""),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), """Allen"""),
                              ((21, 23), """##NL"""),
                              ((23, 24), """##P"""),
                              ((25, 33), """sentence"""),
                              ((33, 34), """."""),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                          if not do_lower_case
                          else [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), """a"""),
                              ((1, 2), ""","""),
                              ((3, 8), """naive"""),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), """allen"""),
                              ((21, 23), """##nl"""),
                              ((23, 24), """##p"""),
                              ((25, 33), """sentence"""),
                              ((33, 34), """."""),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                      )

                      self.assertEqual(
                          [e[1] for e in expected_results]      ,    tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]     )     )
                      self.assertEqual([e[0] for e in expected_results]      ,    tokens["""offset_mapping"""]     )



       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    ["""的""", """人""", """有"""]
            __snake_case      :str 							=    """""".join(a__     )
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
                      __snake_case      :int 							=    True
                      __snake_case      :List[Any] 							=    self.tokenizer_class.from_pretrained(a__      ,    **a__     )
                      __snake_case      :List[str] 							=    self.rust_tokenizer_class.from_pretrained(a__      ,    **a__     )

                      __snake_case      :List[Any] 							=    tokenizer_p.encode(a__      ,    add_special_tokens=a__     )
                      __snake_case      :Optional[int] 							=    tokenizer_r.encode(a__      ,    add_special_tokens=a__     )

                      __snake_case      :Optional[int] 							=    tokenizer_r.convert_ids_to_tokens(a__     )
                      __snake_case      :int 							=    tokenizer_p.convert_ids_to_tokens(a__     )

                      # it is expected that each Chinese character is not preceded by "##"
                      self.assertListEqual(a__      ,    a__     )
                      self.assertListEqual(a__      ,    a__     )

                      __snake_case      :List[Any] 							=    False
                      __snake_case      :List[str] 							=    self.rust_tokenizer_class.from_pretrained(a__      ,    **a__     )
                      __snake_case      :str 							=    self.tokenizer_class.from_pretrained(a__      ,    **a__     )

                      __snake_case      :List[str] 							=    tokenizer_r.encode(a__      ,    add_special_tokens=a__     )
                      __snake_case      :str 							=    tokenizer_p.encode(a__      ,    add_special_tokens=a__     )

                      __snake_case      :Tuple 							=    tokenizer_r.convert_ids_to_tokens(a__     )
                      __snake_case      :Dict 							=    tokenizer_p.convert_ids_to_tokens(a__     )

                      # it is expected that only the first Chinese character is not preceded by "##".
                      __snake_case      :Tuple 							=    [
                          F'''##{token}''' if idx != 0 else token for idx, token in enumerate(a__     )
                      ]
                      self.assertListEqual(a__      ,    a__     )
                      self.assertListEqual(a__      ,    a__     )



       @slow
       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :List[str] 							=    self.tokenizer_class(self.vocab_file      ,    self.word_shape_file      ,    self.word_pronunciation_file     )

            __snake_case      :Any 							=    tokenizer.encode("""你好"""      ,    add_special_tokens=a__     )
            __snake_case      :List[Any] 							=    tokenizer.encode("""你是谁"""      ,    add_special_tokens=a__     )

            __snake_case      :Optional[int] 							=    tokenizer.build_inputs_with_special_tokens(a__     )
            __snake_case      :str 							=    tokenizer.build_inputs_with_special_tokens(a__      ,    a__     )

            assert encoded_sentence == [1] + text + [2]
            assert encoded_pair == [1] + text + [2] + text_a + [2]



       def        __lowercase    (				self     )			->   Any:






            '''simple docstring'''





            __snake_case      :Dict 							=    self.get_tokenizers(do_lower_case=a__     )
            for tokenizer in tokenizers:
                 with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
                      __snake_case      :int 							=    """你好，你是谁"""
                      __snake_case      :Optional[Any] 							=    tokenizer.tokenize(a__     )
                      __snake_case      :Dict 							=    tokenizer.convert_tokens_to_ids(a__     )
                      __snake_case      :Tuple 							=    tokenizer.convert_tokens_to_shape_ids(a__     )
                      __snake_case      :int 							=    tokenizer.convert_tokens_to_pronunciation_ids(a__     )
                      __snake_case      :Union[str, Any] 							=    tokenizer.prepare_for_model(
                          a__      ,    a__      ,    a__      ,    add_special_tokens=a__     )

                      __snake_case      :int 							=    tokenizer.encode_plus(a__      ,    add_special_tokens=a__     )

                      self.assertEqual(a__      ,    a__     )










import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, ClassLabel, Features
from .base import TaskTemplate




@dataclass(frozen=lowercase_)
class       snake_case__ (				lowercase_):

       '''simple docstring'''
       lowerCamelCase				:					str								=			field(default="audio-classification"						,     metadata={"include_in_asdict_even_if_is_default": True})
       lowerCamelCase				:					ClassVar[Features]								=			Features({"audio": Audio()})
       lowerCamelCase				:					ClassVar[Features]								=			Features({"labels": ClassLabel})
       lowerCamelCase				:					str								=			"audio"
       lowerCamelCase				:					str								=			"labels"



       def        __lowercase    (				self      ,    a__     )			->   Dict:






            '''simple docstring'''





            if self.label_column not in features:
                 raise ValueError(F'''Column {self.label_column} is not present in features.'''     )
            if not isinstance(features[self.label_column]      ,    a__     ):
                 raise ValueError(F'''Column {self.label_column} is not a ClassLabel.'''     )
            __snake_case      :Any 							=    copy.deepcopy(self     )
            __snake_case      :List[Any] 							=    self.label_schema.copy()
            __snake_case      :List[Any] 							=    features[self.label_column]
            __snake_case      :Optional[Any] 							=    label_schema
            return task_template



       @property
       def        __lowercase    (				self     )			->   Dict[str, str]:






            '''simple docstring'''





            return {
                self.audio_column: "audio",
                self.label_column: "labels",
            }


