






import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__							=     get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class 			snake_case__   (							lowercase_	,    unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase							:				List[str] 				= XLNetTokenizer
       lowerCamelCase							:				Optional[int] 				= XLNetTokenizerFast
       lowerCamelCase							:				Any 				= True
       lowerCamelCase							:				Tuple 				= True



       def 					__lowercase						(							self    ) ->      Tuple:






            '''simple docstring'''





            super().setUp()

            # We have a SentencePiece fixture for testing
            __snake_case			:Optional[int]          =						XLNetTokenizer(a__			,						keep_accents=a__    )
            tokenizer.sanitize_special_tokens()
            tokenizer.save_pretrained(self.tmpdirname    )



       def 					__lowercase						(							self    ) ->      Tuple:






            '''simple docstring'''





            __snake_case			:Tuple          =						"""<s>"""
            __snake_case			:str          =						1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__    )			,						a__    )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__    )			,						a__    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case			:str          =						list(self.get_tokenizer().get_vocab().keys()    )

            self.assertEqual(vocab_keys[0]			,						"""<unk>"""    )
            self.assertEqual(vocab_keys[1]			,						"""<s>"""    )
            self.assertEqual(vocab_keys[-1]			,						"""<eod>"""    )
            self.assertEqual(len(a__    )			,						10_06    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            self.assertEqual(self.get_tokenizer().vocab_size			,						10_00    )



       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            __snake_case			:Any          =						XLNetTokenizer(a__			,						keep_accents=a__    )

            __snake_case			:Optional[int]          =						tokenizer.tokenize("""This is a test"""    )
            self.assertListEqual(a__			,						["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__    )			,						[2_85, 46, 10, 1_70, 3_82]    )

            __snake_case			:int          =						tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
            self.assertListEqual(
                a__			,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]			,						)
            __snake_case			:Any          =						tokenizer.convert_tokens_to_ids(a__    )
            self.assertListEqual(a__			,						[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]    )

            __snake_case			:Any          =						tokenizer.convert_ids_to_tokens(a__    )
            self.assertListEqual(
                a__			,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]			,						)



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:str          =						XLNetTokenizer(a__			,						do_lower_case=a__    )
            __snake_case			:Optional[int]          =						tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
            self.assertListEqual(
                a__			,						[
                    SPIECE_UNDERLINE + """""",
                    """i""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """se""",
                    """.""",
                ]			,						)
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )			,						["""▁he""", """ll""", """o"""]    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:List[Any]          =						XLNetTokenizer(a__			,						do_lower_case=a__    )
            __snake_case			:List[str]          =						tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
            self.assertListEqual(
                a__			,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """se""",
                    """.""",
                ]			,						)



       @slow
       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            __snake_case			:List[Any]          =						XLNetTokenizer.from_pretrained("""xlnet-base-cased"""    )

            __snake_case			:List[str]          =						tokenizer.encode("""sequence builders"""			,						add_special_tokens=a__    )
            __snake_case			:Optional[int]          =						tokenizer.encode("""multi-sequence build"""			,						add_special_tokens=a__    )

            __snake_case			:str          =						tokenizer.build_inputs_with_special_tokens(a__    )
            __snake_case			:Optional[Any]          =						tokenizer.build_inputs_with_special_tokens(a__			,						a__    )

            assert encoded_sentence == text + [4, 3]
            assert encoded_pair == text + [4] + text_a + [4, 3]



       @slow
       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            __snake_case			:Tuple          =						{"""input_ids""": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=a__			,						model_name="""xlnet-base-cased"""			,						revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""			,						)










# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

import torch

from ..models.auto import AutoModelForVisualQuestionAnswering, AutoProcessor
from ..utils import requires_backends
from .base import PipelineTool


if TYPE_CHECKING:
     from PIL import Image




class 			snake_case__   (							lowercase_):

       '''simple docstring'''
       lowerCamelCase							:				Tuple 				= "dandelin/vilt-b32-finetuned-vqa"
       lowerCamelCase							:				List[str] 				= (
           "This is a tool that answers a question about an image. It takes an input named `image` which should be the "
           "image containing the information, as well as a `question` which should be the question in English. It "
           "returns a text that is the answer to the question."
       )
       lowerCamelCase							:				Optional[Any] 				= "image_qa"
       lowerCamelCase							:				str 				= AutoProcessor
       lowerCamelCase							:				Union[str, Any] 				= AutoModelForVisualQuestionAnswering

       lowerCamelCase							:				Any 				= ["image", "text"]
       lowerCamelCase							:				Dict 				= ["text"]



       def __init__(							self			,						*a__			,						**a__    ) ->      Any:






            '''simple docstring'''





            requires_backends(self			,						["""vision"""]    )
            super().__init__(*a__			,						**a__    )



       def 					__lowercase						(							self			,						a__			,						a__    ) ->      int:






            '''simple docstring'''





            return self.pre_processor(a__			,						a__			,						return_tensors="""pt"""    )



       def 					__lowercase						(							self			,						a__    ) ->      Union[str, Any]:






            '''simple docstring'''





            with torch.no_grad():
                 return self.model(**a__    ).logits



       def 					__lowercase						(							self			,						a__    ) ->      Tuple:






            '''simple docstring'''





            __snake_case			:str          =						outputs.argmax(-1    ).item()
            return self.model.config.idalabel[idx]


