


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 						_snake_case    ( _snake_case		,       unittest.TestCase				):
 SCREAMING_SNAKE_CASE__ =			RoCBertTokenizer
 SCREAMING_SNAKE_CASE__ =			None
 SCREAMING_SNAKE_CASE__ =			False
 SCREAMING_SNAKE_CASE__ =			True
 SCREAMING_SNAKE_CASE__ =			filter_non_english




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       super().setUp()

       a		:Any									=			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
       a		:Optional[Any]									=			{}
       a		:Tuple									=			{}
       for i, value in enumerate(_lowerCamelCase       ):
             a		:str									=			i
             a		:List[Any]									=			i
       a		:Any									=			os.path.join(self.tmpdirname   ,    VOCAB_FILES_NAMES['''vocab_file''']       )
       a		:Union[str, Any]									=			os.path.join(self.tmpdirname   ,    VOCAB_FILES_NAMES['''word_shape_file''']       )
       a		:int									=			os.path.join(self.tmpdirname   ,    VOCAB_FILES_NAMES['''word_pronunciation_file''']       )
       with open(self.vocab_file   ,    '''w'''   ,    encoding='''utf-8'''       ) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )
       with open(self.word_shape_file   ,    '''w'''   ,    encoding='''utf-8'''       ) as word_shape_writer:
             json.dump(_lowerCamelCase   ,    _lowerCamelCase   ,    ensure_ascii=_lowerCamelCase       )
       with open(self.word_pronunciation_file   ,    '''w'''   ,    encoding='''utf-8'''       ) as word_pronunciation_writer:
             json.dump(_lowerCamelCase   ,    _lowerCamelCase   ,    ensure_ascii=_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:int									=			self.tokenizer_class(self.vocab_file   ,    self.word_shape_file   ,    self.word_pronunciation_file       )

       a		:Union[str, Any]									=			tokenizer.tokenize('''你好[SEP]你是谁'''       )
       self.assertListEqual(_lowerCamelCase   ,    ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']       )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase       )   ,    [5, 6, 2, 5, 7, 8]       )
       self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase       )   ,    [5, 6, 2, 5, 7, 8]       )
       self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase       )   ,    [5, 6, 2, 5, 7, 8]       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:int									=			RoCBertBasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''       )   ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:str									=			RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )   ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )   ,    ['''hello''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:List[str]									=			RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase   ,    strip_accents=_lowerCamelCase       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )   ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )   ,    ['''h\u00E9llo''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:List[Any]									=			RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase   ,    strip_accents=_lowerCamelCase       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )   ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )   ,    ['''hello''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Dict									=			RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )   ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )   ,    ['''hello''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:List[str]									=			RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )   ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:List[Any]									=			RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase   ,    strip_accents=_lowerCamelCase       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )   ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Dict									=			RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase   ,    strip_accents=_lowerCamelCase       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )   ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Any									=			RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase   ,    never_split=['''[UNK]''']       )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''       )   ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:str									=			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       a		:List[Any]									=			{}
       for i, token in enumerate(_lowerCamelCase       ):
             a		:int									=			i
       a		:Tuple									=			RoCBertWordpieceTokenizer(vocab=_lowerCamelCase   ,    unk_token='''[UNK]'''       )

       self.assertListEqual(tokenizer.tokenize(''''''       )   ,    []       )

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''       )   ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']       )

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''       )   ,    ['''[UNK]''', '''runn''', '''##ing''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       self.assertTrue(_is_whitespace(''' '''       )       )
       self.assertTrue(_is_whitespace('''\t'''       )       )
       self.assertTrue(_is_whitespace('''\r'''       )       )
       self.assertTrue(_is_whitespace('''\n'''       )       )
       self.assertTrue(_is_whitespace('''\u00A0'''       )       )

       self.assertFalse(_is_whitespace('''A'''       )       )
       self.assertFalse(_is_whitespace('''-'''       )       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       self.assertTrue(_is_control('''\u0005'''       )       )

       self.assertFalse(_is_control('''A'''       )       )
       self.assertFalse(_is_control(''' '''       )       )
       self.assertFalse(_is_control('''\t'''       )       )
       self.assertFalse(_is_control('''\r'''       )       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       self.assertTrue(_is_punctuation('''-'''       )       )
       self.assertTrue(_is_punctuation('''$'''       )       )
       self.assertTrue(_is_punctuation('''`'''       )       )
       self.assertTrue(_is_punctuation('''.'''       )       )

       self.assertFalse(_is_punctuation('''A'''       )       )
       self.assertFalse(_is_punctuation(''' '''       )       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Union[str, Any]									=			self.get_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(_lowerCamelCase       ) for t in ['''Test''', '''\xad''', '''test''']]   ,    [['''[UNK]'''], [], ['''[UNK]''']]       )

       if self.test_rust_tokenizer:
             a		:List[Any]									=			self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(_lowerCamelCase       ) for t in ['''Test''', '''\xad''', '''test''']]   ,    [['''[UNK]'''], [], ['''[UNK]''']]       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                   a		:str									=			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase   ,    **_lowerCamelCase       )

                   a		:Optional[Any]									=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                   a		:Any									=			tokenizer_r.encode_plus(
                       _lowerCamelCase   ,    return_attention_mask=_lowerCamelCase   ,    return_token_type_ids=_lowerCamelCase   ,    return_offsets_mapping=_lowerCamelCase   ,    add_special_tokens=_lowerCamelCase   ,    )

                   a		:str									=			tokenizer_r.do_lower_case if hasattr(_lowerCamelCase   ,    '''do_lower_case'''       ) else False
                   a		:Union[str, Any]									=			(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]   ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']       )       )
                   self.assertEqual([e[0] for e in expected_results]   ,    tokens['''offset_mapping''']       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Tuple									=			['''的''', '''人''', '''有''']
       a		:Optional[Any]									=			''''''.join(_lowerCamelCase       )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                   a		:List[Any]									=			True
                   a		:Optional[Any]									=			self.tokenizer_class.from_pretrained(_lowerCamelCase   ,    **_lowerCamelCase       )
                   a		:Union[str, Any]									=			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase   ,    **_lowerCamelCase       )

                   a		:List[Any]									=			tokenizer_p.encode(_lowerCamelCase   ,    add_special_tokens=_lowerCamelCase       )
                   a		:List[str]									=			tokenizer_r.encode(_lowerCamelCase   ,    add_special_tokens=_lowerCamelCase       )

                   a		:Any									=			tokenizer_r.convert_ids_to_tokens(_lowerCamelCase       )
                   a		:Dict									=			tokenizer_p.convert_ids_to_tokens(_lowerCamelCase       )

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(_lowerCamelCase   ,    _lowerCamelCase       )
                   self.assertListEqual(_lowerCamelCase   ,    _lowerCamelCase       )

                   a		:List[Any]									=			False
                   a		:List[str]									=			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase   ,    **_lowerCamelCase       )
                   a		:Dict									=			self.tokenizer_class.from_pretrained(_lowerCamelCase   ,    **_lowerCamelCase       )

                   a		:List[str]									=			tokenizer_r.encode(_lowerCamelCase   ,    add_special_tokens=_lowerCamelCase       )
                   a		:str									=			tokenizer_p.encode(_lowerCamelCase   ,    add_special_tokens=_lowerCamelCase       )

                   a		:Dict									=			tokenizer_r.convert_ids_to_tokens(_lowerCamelCase       )
                   a		:Optional[int]									=			tokenizer_p.convert_ids_to_tokens(_lowerCamelCase       )

                   # it is expected that only the first Chinese character is not preceded by "##".
                   a		:Dict									=			[
                       F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase       )
                   ]
                   self.assertListEqual(_lowerCamelCase   ,    _lowerCamelCase       )
                   self.assertListEqual(_lowerCamelCase   ,    _lowerCamelCase       )




 @slow
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:List[Any]									=			self.tokenizer_class(self.vocab_file   ,    self.word_shape_file   ,    self.word_pronunciation_file       )

       a		:Dict									=			tokenizer.encode('''你好'''   ,    add_special_tokens=_lowerCamelCase       )
       a		:Any									=			tokenizer.encode('''你是谁'''   ,    add_special_tokens=_lowerCamelCase       )

       a		:str									=			tokenizer.build_inputs_with_special_tokens(_lowerCamelCase       )
       a		:Dict									=			tokenizer.build_inputs_with_special_tokens(_lowerCamelCase   ,    _lowerCamelCase       )

       assert encoded_sentence == [1] + text + [2]
       assert encoded_pair == [1] + text + [2] + text_a + [2]




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:List[Any]									=			self.get_tokenizers(do_lower_case=_lowerCamelCase       )
       for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
                   a		:Any									=			'''你好，你是谁'''
                   a		:str									=			tokenizer.tokenize(_lowerCamelCase       )
                   a		:List[Any]									=			tokenizer.convert_tokens_to_ids(_lowerCamelCase       )
                   a		:Optional[Any]									=			tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase       )
                   a		:Any									=			tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase       )
                   a		:Optional[Any]									=			tokenizer.prepare_for_model(
                       _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    add_special_tokens=_lowerCamelCase       )

                   a		:str									=			tokenizer.encode_plus(_lowerCamelCase   ,    add_special_tokens=_lowerCamelCase       )

                   self.assertEqual(_lowerCamelCase   ,    _lowerCamelCase       )









snake_case							:       Any										=	{0: [2, 3], 1: [0], 2: [1], 3: [4], 4: []}

snake_case							:       Any										=	{0: [1, 2, 3], 1: [2], 2: [0], 3: [4], 4: [5], 5: [3]}
def 							__lowerCamelCase		(					UpperCAmelCase_		:						dict[int, list[int]]					,    UpperCAmelCase_		:						int					,    UpperCAmelCase_		:						list[bool]		):




      """simple docstring"""





      a		:Any									=			True
      a		:Any									=			[]

      for neighbour in graph[vert]:
            if not visited[neighbour]:
                  order += topology_sort(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_		)

      order.append(UpperCAmelCase_		)

      return order
def 							__lowerCamelCase		(					UpperCAmelCase_		:						dict[int, list[int]]					,    UpperCAmelCase_		:						int					,    UpperCAmelCase_		:						list[bool]		):




      """simple docstring"""





      a		:Optional[Any]									=			True
      a		:int									=			[vert]

      for neighbour in reversed_graph[vert]:
            if not visited[neighbour]:
                  component += find_components(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_		)

      return component






def 							__lowerCamelCase		(					UpperCAmelCase_		:						dict[int, list[int]]		):




      """simple docstring"""





      a		:int									=			len(UpperCAmelCase_		) * [False]
      a		:dict[int, list[int]]									=			{vert: [] for vert in range(len(UpperCAmelCase_		)		)}

      for vert, neighbours in graph.items():
            for neighbour in neighbours:
                  reversed_graph[neighbour].append(UpperCAmelCase_		)

      a		:Optional[Any]									=			[]
      for i, was_visited in enumerate(UpperCAmelCase_		):
            if not was_visited:
                  order += topology_sort(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_		)

      a		:Tuple									=			[]
      a		:int									=			len(UpperCAmelCase_		) * [False]

      for i in range(len(UpperCAmelCase_		)		):
            a		:Union[str, Any]									=			order[len(UpperCAmelCase_		) - i - 1]
            if not visited[vert]:
                  a		:Union[str, Any]									=			find_components(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_		)
                  components_list.append(UpperCAmelCase_		)

      return components_list





