
import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class    _snake_case (						UpperCAmelCase_	,    unittest.TestCase				):
 __lowerCAmelCase	:  Union[str, Any]	   =      RoCBertTokenizer
 __lowerCAmelCase	:  Union[str, Any]	   =      None
 __lowerCAmelCase	:  str	   =      False
 __lowerCAmelCase	:  List[Any]	   =      True
 __lowerCAmelCase	:  Optional[int]	   =      filter_non_english


 def   lowercase__  (       self):


   '''simple docstring'''




   super().setUp()

   lowercase__	: Optional[int]     		=   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
   lowercase__	: Dict     		=   {}
   lowercase__	: Tuple     		=   {}
   for i, value in enumerate(SCREAMING_SNAKE_CASE_):
     lowercase__	: Tuple     		=   i
     lowercase__	: Any     		=   i
   lowercase__	: str     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""vocab_file"""])
   lowercase__	: Union[str, Any]     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""word_shape_file"""])
   lowercase__	: Tuple     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""word_pronunciation_file"""])
   with open(self.vocab_file       ,	"""w"""       ,	encoding="""utf-8""") as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))
   with open(self.word_shape_file       ,	"""w"""       ,	encoding="""utf-8""") as word_shape_writer:
     json.dump(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	ensure_ascii=SCREAMING_SNAKE_CASE_)
   with open(self.word_pronunciation_file       ,	"""w"""       ,	encoding="""utf-8""") as word_pronunciation_writer:
     json.dump(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	ensure_ascii=SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   self.tokenizer_class(self.vocab_file       ,	self.word_shape_file       ,	self.word_pronunciation_file)

   lowercase__	: Optional[int]     		=   tokenizer.tokenize("""你好[SEP]你是谁""")
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""])
   self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])
   self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])
   self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: int     		=   RoCBertBasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""")       ,	["""ah""", """\u535A""", """\u63A8""", """zz"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)       ,	["""hello""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Any     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hällo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""h\u00E9llo"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hallo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hallo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)       ,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: str     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Tuple     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	never_split=["""[UNK]"""])

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""")       ,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[int]     		=   ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

   lowercase__	: Optional[int]     		=   {}
   for i, token in enumerate(SCREAMING_SNAKE_CASE_):
     lowercase__	: Optional[Any]     		=   i
   lowercase__	: Union[str, Any]     		=   RoCBertWordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_       ,	unk_token="""[UNK]""")

   self.assertListEqual(tokenizer.tokenize("""""")       ,	[])

   self.assertListEqual(tokenizer.tokenize("""unwanted running""")       ,	["""un""", """##want""", """##ed""", """runn""", """##ing"""])

   self.assertListEqual(tokenizer.tokenize("""unwantedX running""")       ,	["""[UNK]""", """runn""", """##ing"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_whitespace(""" """))
   self.assertTrue(_is_whitespace("""\t"""))
   self.assertTrue(_is_whitespace("""\r"""))
   self.assertTrue(_is_whitespace("""\n"""))
   self.assertTrue(_is_whitespace("""\u00A0"""))

   self.assertFalse(_is_whitespace("""A"""))
   self.assertFalse(_is_whitespace("""-"""))


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_control("""\u0005"""))

   self.assertFalse(_is_control("""A"""))
   self.assertFalse(_is_control(""" """))
   self.assertFalse(_is_control("""\t"""))
   self.assertFalse(_is_control("""\r"""))


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_punctuation("""-"""))
   self.assertTrue(_is_punctuation("""$"""))
   self.assertTrue(_is_punctuation("""`"""))
   self.assertTrue(_is_punctuation("""."""))

   self.assertFalse(_is_punctuation("""A"""))
   self.assertFalse(_is_punctuation(""" """))


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE_) for t in ["""Test""", """\xad""", """test"""]]       ,	[["""[UNK]"""], [], ["""[UNK]"""]])

   if self.test_rust_tokenizer:
     lowercase__	: int     		=   self.get_rust_tokenizer()
     self.assertListEqual(
         [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_) for t in ["""Test""", """\xad""", """test"""]]       ,	[["""[UNK]"""], [], ["""[UNK]"""]])


 def   lowercase__  (       self):


   '''simple docstring'''




   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
       lowercase__	: str     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[int]     		=   f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
       lowercase__	: List[str]     		=   tokenizer_r.encode_plus(
           SCREAMING_SNAKE_CASE_       ,	return_attention_mask=SCREAMING_SNAKE_CASE_       ,	return_token_type_ids=SCREAMING_SNAKE_CASE_       ,	return_offsets_mapping=SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_       ,	)

       lowercase__	: str     		=   tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE_       ,	"""do_lower_case""") else False
       lowercase__	: Optional[Any]     		=   (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), """A"""),
               ((1, 2), ""","""),
               ((3, 5), """na"""),
               ((5, 6), """##ï"""),
               ((6, 8), """##ve"""),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), """Allen"""),
               ((21, 23), """##NL"""),
               ((23, 24), """##P"""),
               ((25, 33), """sentence"""),
               ((33, 34), """."""),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), """a"""),
               ((1, 2), ""","""),
               ((3, 8), """naive"""),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), """allen"""),
               ((21, 23), """##nl"""),
               ((23, 24), """##p"""),
               ((25, 33), """sentence"""),
               ((33, 34), """."""),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]       ,	tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]))
       self.assertEqual([e[0] for e in expected_results]       ,	tokens["""offset_mapping"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Any     		=   ["""的""", """人""", """有"""]
   lowercase__	: List[str]     		=   """""".join(SCREAMING_SNAKE_CASE_)
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
       lowercase__	: Union[str, Any]     		=   True
       lowercase__	: Tuple     		=   self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)
       lowercase__	: List[Any]     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[Any]     		=   tokenizer_p.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
       lowercase__	: str     		=   tokenizer_r.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: List[str]     		=   tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)
       lowercase__	: List[str]     		=   tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

       lowercase__	: Any     		=   False
       lowercase__	: Optional[Any]     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)
       lowercase__	: Optional[Any]     		=   self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[int]     		=   tokenizer_r.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
       lowercase__	: Tuple     		=   tokenizer_p.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: Tuple     		=   tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)
       lowercase__	: Optional[Any]     		=   tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)

       # it is expected that only the first Chinese character is not preceded by "##".
       lowercase__	: Any     		=   [
           f'##{token}' if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE_)
       ]
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)


 @slow
 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   self.tokenizer_class(self.vocab_file       ,	self.word_shape_file       ,	self.word_pronunciation_file)

   lowercase__	: Optional[Any]     		=   tokenizer.encode("""你好"""       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
   lowercase__	: Any     		=   tokenizer.encode("""你是谁"""       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

   lowercase__	: Optional[Any]     		=   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_)
   lowercase__	: Tuple     		=   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

   assert encoded_sentence == [1] + text + [2]
   assert encoded_pair == [1] + text + [2] + text_a + [2]



 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[int]     		=   self.get_tokenizers(do_lower_case=SCREAMING_SNAKE_CASE_)
   for tokenizer in tokenizers:
     with self.subTest(f'{tokenizer.__class__.__name__}'):
       lowercase__	: Optional[int]     		=   """你好，你是谁"""
       lowercase__	: List[Any]     		=   tokenizer.tokenize(SCREAMING_SNAKE_CASE_)
       lowercase__	: Union[str, Any]     		=   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: Tuple     		=   tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: List[str]     		=   tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: Any     		=   tokenizer.prepare_for_model(
           SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: Dict     		=   tokenizer.encode_plus(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)











import unittest

from transformers import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING, is_vision_available
from transformers.pipelines import pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
			from PIL import Image
else:


			class 						lowercase    :
									'''simple docstring'''


									@staticmethod
									def       lowerCamelCase_				(  *snake_case    :				Dict       ,	**snake_case    :				List[Any]	):



														'''simple docstring'''


														pass


@is_pipeline_test
@require_torch
@require_vision
class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		Any		=		MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING







						def       lowerCamelCase_				(  self    :				List[Any]       ,	snake_case    :				List[str]       ,	snake_case    :				int       ,	snake_case    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				pipeline('visual-question-answering'       ,	model='hf-internal-testing/tiny-vilt-random-vqa'	)
											SCREAMING_SNAKE_CASE							:     Any						=				[
											    {
											        'image': Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'	),
											        'question': 'How many cats are there?',
											    },
											    {
											        'image': './tests/fixtures/tests_samples/COCO/000000039769.png',
											        'question': 'How many cats are there?',
											    },
											]
											return vqa_pipeline, examples







						def       lowerCamelCase_				(  self    :				int       ,	snake_case    :				int       ,	snake_case    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				vqa_pipeline(snake_case       ,	top_k=1	)
											self.assertEqual(
											    snake_case       ,	[
											        [{'score': ANY(snake_case	), 'answer': ANY(snake_case	)}],
											        [{'score': ANY(snake_case	), 'answer': ANY(snake_case	)}],
											    ]       ,	)







						@require_torch
						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				pipeline('visual-question-answering'       ,	model='hf-internal-testing/tiny-vilt-random-vqa'	)
											SCREAMING_SNAKE_CASE							:     List[Any]						=				'./tests/fixtures/tests_samples/COCO/000000039769.png'
											SCREAMING_SNAKE_CASE							:     Any						=				'How many cats are there?'

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				vqa_pipeline(image=snake_case       ,	question='How many cats are there?'       ,	top_k=2	)
											self.assertEqual(
											    snake_case       ,	[{'score': ANY(snake_case	), 'answer': ANY(snake_case	)}, {'score': ANY(snake_case	), 'answer': ANY(snake_case	)}]	)

											SCREAMING_SNAKE_CASE							:     int						=				vqa_pipeline({'image': image, 'question': question}       ,	top_k=2	)
											self.assertEqual(
											    snake_case       ,	[{'score': ANY(snake_case	), 'answer': ANY(snake_case	)}, {'score': ANY(snake_case	), 'answer': ANY(snake_case	)}]	)







						@slow
						@require_torch
						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				pipeline('visual-question-answering'       ,	model='dandelin/vilt-b32-finetuned-vqa'	)
											SCREAMING_SNAKE_CASE							:     int						=				'./tests/fixtures/tests_samples/COCO/000000039769.png'
											SCREAMING_SNAKE_CASE							:     Dict						=				'How many cats are there?'

											SCREAMING_SNAKE_CASE							:     int						=				vqa_pipeline(image=snake_case       ,	question=snake_case       ,	top_k=2	)
											self.assertEqual(
											    nested_simplify(snake_case       ,	decimals=4	)       ,	[{'score': 0.8799, 'answer': '2'}, {'score': 0.296, 'answer': '1'}]	)

											SCREAMING_SNAKE_CASE							:     List[str]						=				vqa_pipeline({'image': image, 'question': question}       ,	top_k=2	)
											self.assertEqual(
											    nested_simplify(snake_case       ,	decimals=4	)       ,	[{'score': 0.8799, 'answer': '2'}, {'score': 0.296, 'answer': '1'}]	)

											SCREAMING_SNAKE_CASE							:     Any						=				vqa_pipeline(
											    [{'image': image, 'question': question}, {'image': image, 'question': question}]       ,	top_k=2	)
											self.assertEqual(
											    nested_simplify(snake_case       ,	decimals=4	)       ,	[[{'score': 0.8799, 'answer': '2'}, {'score': 0.296, 'answer': '1'}]] * 2       ,	)







						@require_tf
						@unittest.skip('Visual question answering not implemented in TF'	)
						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											pass