

"""simple docstring"""




import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class      __lowercase			(    __snake_case	,			unittest.TestCase					):
		snake_case_				       =       LongformerTokenizer
		snake_case_				       =       True
		snake_case_				       =       LongformerTokenizerFast
		snake_case_				       =       True

		def    __lowercase       (  self		:			Any    ):






			'''simple docstring'''
			super().setUp()

			# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
			UpperCAmelCase__					:  List[Any]					   =					[
			    "l",
			    "o",
			    "w",
			    "e",
			    "r",
			    "s",
			    "t",
			    "i",
			    "d",
			    "n",
			    "\u0120",
			    "\u0120l",
			    "\u0120n",
			    "\u0120lo",
			    "\u0120low",
			    "er",
			    "\u0120lowest",
			    "\u0120newer",
			    "\u0120wider",
			    "<unk>",
			]
			UpperCAmelCase__					:  List[Any]					   =					dict(zip(A_					,range(len(A_    )    )    )    )
			UpperCAmelCase__					:  Any					   =					["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
			UpperCAmelCase__					:  Optional[Any]					   =					{"unk_token": "<unk>"}

			UpperCAmelCase__					:  Dict					   =					os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["""vocab_file"""]    )
			UpperCAmelCase__					:  Any					   =					os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["""merges_file"""]    )
			with open(self.vocab_file					,"""w"""					,encoding="""utf-8"""    ) as fp:
				fp.write(json.dumps(A_    ) + """\n"""    )
			with open(self.merges_file					,"""w"""					,encoding="""utf-8"""    ) as fp:
				fp.write("""\n""".join(A_    )    )

		def    __lowercase       (  self		:			int					,**A		:			Dict    ):






			'''simple docstring'''
			kwargs.update(self.special_tokens_map    )
			return self.tokenizer_class.from_pretrained(self.tmpdirname					,**A_    )

		def    __lowercase       (  self		:			Optional[Any]					,**A		:			Union[str, Any]    ):






			'''simple docstring'''
			kwargs.update(self.special_tokens_map    )
			return self.rust_tokenizer_class.from_pretrained(self.tmpdirname					,**A_    )

		def    __lowercase       (  self		:			int					,A		:			Tuple    ):






			'''simple docstring'''
			UpperCAmelCase__					:  Any					   =					"lower newer"
			UpperCAmelCase__					:  str					   =					"lower newer"
			return input_text, output_text

		def    __lowercase       (  self		:			Optional[int]    ):






			'''simple docstring'''
			UpperCAmelCase__					:  Union[str, Any]					   =					self.tokenizer_class(self.vocab_file					,self.merges_file					,**self.special_tokens_map    )
			UpperCAmelCase__					:  str					   =					"lower newer"
			UpperCAmelCase__					:  Tuple					   =					["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
			UpperCAmelCase__					:  int					   =					tokenizer.tokenize(A_    )  # , add_prefix_space=True)
			self.assertListEqual(A_					,A_    )

			UpperCAmelCase__					:  Optional[int]					   =					tokens + [tokenizer.unk_token]
			UpperCAmelCase__					:  List[Any]					   =					[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
			self.assertListEqual(tokenizer.convert_tokens_to_ids(A_    )					,A_    )

		def    __lowercase       (  self		:			Tuple    ):






			'''simple docstring'''
			UpperCAmelCase__					:  Optional[Any]					   =					self.get_tokenizer()

			self.assertListEqual(tokenizer.encode("""Hello world!"""					,add_special_tokens=A_    )					,[0, 31_414, 232, 328, 2]    )
			self.assertListEqual(
			    tokenizer.encode("""Hello world! cécé herlolip 418"""					,add_special_tokens=A_    )					,[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]					,)

		@slow
		def    __lowercase       (  self		:			Any    ):






			'''simple docstring'''
			UpperCAmelCase__					:  Optional[int]					   =					self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""    )

			UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer.encode("""sequence builders"""					,add_special_tokens=A_    )
			UpperCAmelCase__					:  Dict					   =					tokenizer.encode("""multi-sequence build"""					,add_special_tokens=A_    )

			UpperCAmelCase__					:  Optional[Any]					   =					tokenizer.encode(
			    """sequence builders"""					,add_special_tokens=A_					,add_prefix_space=A_    )
			UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer.encode(
			    """sequence builders"""					,"""multi-sequence build"""					,add_special_tokens=A_					,add_prefix_space=A_    )

			UpperCAmelCase__					:  List[str]					   =					tokenizer.build_inputs_with_special_tokens(A_    )
			UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer.build_inputs_with_special_tokens(A_					,A_    )

			assert encoded_sentence == encoded_text_from_decode
			assert encoded_pair == encoded_pair_from_decode

		def    __lowercase       (  self		:			Optional[Any]    ):






			'''simple docstring'''
			UpperCAmelCase__					:  Optional[int]					   =					self.get_tokenizer()

			UpperCAmelCase__					:  Dict					   =					"Encode this sequence."
			UpperCAmelCase__					:  Optional[int]					   =					tokenizer.byte_encoder[" ".encode("""utf-8"""    )[0]]

			# Testing encoder arguments
			UpperCAmelCase__					:  Optional[int]					   =					tokenizer.encode(A_					,add_special_tokens=A_					,add_prefix_space=A_    )
			UpperCAmelCase__					:  str					   =					tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
			self.assertNotEqual(A_					,A_    )

			UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer.encode(A_					,add_special_tokens=A_					,add_prefix_space=A_    )
			UpperCAmelCase__					:  Tuple					   =					tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
			self.assertEqual(A_					,A_    )

			tokenizer.add_special_tokens({"""bos_token""": """<s>"""}    )
			UpperCAmelCase__					:  Dict					   =					tokenizer.encode(A_					,add_special_tokens=A_    )
			UpperCAmelCase__					:  Optional[Any]					   =					tokenizer.convert_ids_to_tokens(encoded[1]    )[0]
			self.assertNotEqual(A_					,A_    )

			# Testing spaces after special tokens
			UpperCAmelCase__					:  Union[str, Any]					   =					"<mask>"
			tokenizer.add_special_tokens(
			    {"""mask_token""": AddedToken(A_					,lstrip=A_					,rstrip=A_    )}    )  # mask token has a left space
			UpperCAmelCase__					:  List[Any]					   =					tokenizer.convert_tokens_to_ids(A_    )

			UpperCAmelCase__					:  Any					   =					"Encode <mask> sequence"
			UpperCAmelCase__					:  Optional[int]					   =					"Encode <mask>sequence"

			UpperCAmelCase__					:  List[Any]					   =					tokenizer.encode(A_    )
			UpperCAmelCase__					:  Optional[int]					   =					encoded.index(A_    )
			UpperCAmelCase__					:  Optional[Any]					   =					tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
			self.assertEqual(A_					,A_    )

			UpperCAmelCase__					:  Dict					   =					tokenizer.encode(A_    )
			UpperCAmelCase__					:  List[str]					   =					encoded.index(A_    )
			UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
			self.assertNotEqual(A_					,A_    )

		def    __lowercase       (  self		:			str    ):






			'''simple docstring'''
			pass

		def    __lowercase       (  self		:			Optional[Any]    ):






			'''simple docstring'''




			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
				with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
					UpperCAmelCase__					:  Tuple					   =					self.rust_tokenizer_class.from_pretrained(A_					,**A_    )
					UpperCAmelCase__					:  Union[str, Any]					   =					self.tokenizer_class.from_pretrained(A_					,**A_    )
					UpperCAmelCase__					:  Any					   =					"A, <mask> AllenNLP sentence."
					UpperCAmelCase__					:  int					   =					tokenizer_r.encode_plus(A_					,add_special_tokens=A_					,return_token_type_ids=A_    )
					UpperCAmelCase__					:  Any					   =					tokenizer_p.encode_plus(A_					,add_special_tokens=A_					,return_token_type_ids=A_    )

					# token_type_ids should put 0 everywhere
					self.assertEqual(sum(tokens_r["""token_type_ids"""]    )					,sum(tokens_p["""token_type_ids"""]    )    )

					# attention_mask should put 1 everywhere, so sum over length should be 1
					self.assertEqual(
					    sum(tokens_r["""attention_mask"""]    ) / len(tokens_r["""attention_mask"""]    )					,sum(tokens_p["""attention_mask"""]    ) / len(tokens_p["""attention_mask"""]    )					,)

					UpperCAmelCase__					:  Optional[Any]					   =					tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]    )
					UpperCAmelCase__					:  int					   =					tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]    )

					# Rust correctly handles the space before the mask while python doesnt
					self.assertSequenceEqual(tokens_p["""input_ids"""]					,[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]    )
					self.assertSequenceEqual(tokens_r["""input_ids"""]					,[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]    )

					self.assertSequenceEqual(
					    A_					,["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]    )
					self.assertSequenceEqual(
					    A_					,["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]    )

		def    __lowercase       (  self		:			Dict    ):






			'''simple docstring'''




			for trim_offsets, add_prefix_space in itertools.product([True, False]					,repeat=2    ):
				UpperCAmelCase__					:  str					   =					self.rust_tokenizer_class.from_pretrained(
				    self.tmpdirname					,use_fast=A_					,add_prefix_space=A_					,trim_offsets=A_    )

				UpperCAmelCase__					:  Optional[Any]					   =					json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()    )
				UpperCAmelCase__					:  Union[str, Any]					   =					json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()    )

				self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]					,A_    )

				self.assertEqual(post_processor_state["""add_prefix_space"""]					,A_    )
				self.assertEqual(post_processor_state["""trim_offsets"""]					,A_    )




		def    __lowercase       (  self		:			List[str]    ):






			'''simple docstring'''




			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
				with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
					UpperCAmelCase__					:  Union[str, Any]					   =					"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
					UpperCAmelCase__					:  Tuple					   =					f"{text_of_1_token} {text_of_1_token}"

					UpperCAmelCase__					:  List[Any]					   =					self.rust_tokenizer_class.from_pretrained(
					    A_					,use_fast=A_					,add_prefix_space=A_					,trim_offsets=A_    )
					UpperCAmelCase__					:  Any					   =					tokenizer_r(A_					,return_offsets_mapping=A_					,add_special_tokens=A_    )
					self.assertEqual(encoding.offset_mapping[0]					,(0, len(A_    ))    )
					self.assertEqual(
					    encoding.offset_mapping[1]					,(len(A_    ) + 1, len(A_    ) + 1 + len(A_    ))					,)

					UpperCAmelCase__					:  Any					   =					self.rust_tokenizer_class.from_pretrained(
					    A_					,use_fast=A_					,add_prefix_space=A_					,trim_offsets=A_    )
					UpperCAmelCase__					:  Optional[int]					   =					tokenizer_r(A_					,return_offsets_mapping=A_					,add_special_tokens=A_    )
					self.assertEqual(encoding.offset_mapping[0]					,(0, len(A_    ))    )
					self.assertEqual(
					    encoding.offset_mapping[1]					,(len(A_    ) + 1, len(A_    ) + 1 + len(A_    ))					,)

					UpperCAmelCase__					:  int					   =					self.rust_tokenizer_class.from_pretrained(
					    A_					,use_fast=A_					,add_prefix_space=A_					,trim_offsets=A_    )
					UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer_r(A_					,return_offsets_mapping=A_					,add_special_tokens=A_    )
					self.assertEqual(encoding.offset_mapping[0]					,(0, len(A_    ))    )
					self.assertEqual(
					    encoding.offset_mapping[1]					,(len(A_    ), len(A_    ) + 1 + len(A_    ))					,)

					UpperCAmelCase__					:  Any					   =					self.rust_tokenizer_class.from_pretrained(
					    A_					,use_fast=A_					,add_prefix_space=A_					,trim_offsets=A_    )
					UpperCAmelCase__					:  Optional[int]					   =					tokenizer_r(A_					,return_offsets_mapping=A_					,add_special_tokens=A_    )
					self.assertEqual(encoding.offset_mapping[0]					,(0, len(A_    ))    )
					self.assertEqual(
					    encoding.offset_mapping[1]					,(len(A_    ), len(A_    ) + 1 + len(A_    ))					,)

					UpperCAmelCase__					:  List[Any]					   =					f" {text}"

					# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
					#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
					# )
					# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
					# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
					# self.assertEqual(
					#     encoding.offset_mapping[1],
					#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
					# )

					UpperCAmelCase__					:  str					   =					self.rust_tokenizer_class.from_pretrained(
					    A_					,use_fast=A_					,add_prefix_space=A_					,trim_offsets=A_    )
					UpperCAmelCase__					:  Dict					   =					tokenizer_r(A_					,return_offsets_mapping=A_					,add_special_tokens=A_    )
					self.assertEqual(encoding.offset_mapping[0]					,(1, 1 + len(A_    ))    )
					self.assertEqual(
					    encoding.offset_mapping[1]					,(1 + len(A_    ) + 1, 1 + len(A_    ) + 1 + len(A_    ))					,)

					UpperCAmelCase__					:  Dict					   =					self.rust_tokenizer_class.from_pretrained(
					    A_					,use_fast=A_					,add_prefix_space=A_					,trim_offsets=A_    )
					UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer_r(A_					,return_offsets_mapping=A_					,add_special_tokens=A_    )
					self.assertEqual(encoding.offset_mapping[0]					,(0, 1 + len(A_    ))    )
					self.assertEqual(
					    encoding.offset_mapping[1]					,(1 + len(A_    ), 1 + len(A_    ) + 1 + len(A_    ))					,)

					UpperCAmelCase__					:  Dict					   =					self.rust_tokenizer_class.from_pretrained(
					    A_					,use_fast=A_					,add_prefix_space=A_					,trim_offsets=A_    )
					UpperCAmelCase__					:  Optional[Any]					   =					tokenizer_r(A_					,return_offsets_mapping=A_					,add_special_tokens=A_    )
					self.assertEqual(encoding.offset_mapping[0]					,(0, 1 + len(A_    ))    )
					self.assertEqual(
					    encoding.offset_mapping[1]					,(1 + len(A_    ), 1 + len(A_    ) + 1 + len(A_    ))					,)








"""simple docstring"""




from typing import Union

from ..utils import add_end_docstrings, is_torch_available, is_vision_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
      from PIL import Image

      from ..image_utils import load_image

if is_torch_available():
      from ..models.auto.modeling_auto import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING

__UpperCAmelCase =				logging.get_logger(__name__)







@add_end_docstrings(__lowerCamelCase					)
class      __lowercase			(    __lowerCamelCase					):

  def __init__(  self		:			List[str]					,*A		:			Optional[int]					,**A		:			List[str]    ):






   '''simple docstring'''
   super().__init__(*A					,**A    )
   self.check_model_type(A    )

  def    __lowercase       (  self		:			Optional[int]					,A		:			Dict=None					,A		:			Optional[int]=None					,A		:			Tuple=None					,**A		:			str    ):






   '''simple docstring'''
   UpperCAmelCase__   ,      UpperCAmelCase__					:  Any					   =					{}, {}
   if padding is not None:
    UpperCAmelCase__					:  Union[str, Any]					   =					padding
   if truncation is not None:
    UpperCAmelCase__					:  List[str]					   =					truncation
   if top_k is not None:
    UpperCAmelCase__					:  List[Any]					   =					top_k
   return preprocess_params, {}, postprocess_params

  def __call__(  self		:			List[Any]					,A		:			Union["Image.Image", str]					,A		:			str = None					,**A		:			Union[str, Any]    ):






   '''simple docstring'''
   if isinstance(A					,(Image.Image, str)    ) and isinstance(A					,A    ):
    UpperCAmelCase__					:  Any					   =					{"""image""": image, """question""": question}
   else:
    UpperCAmelCase__					:  Dict					   =					image
   UpperCAmelCase__					:  Dict					   =					super().__call__(A					,**A    )
   return results

  def    __lowercase       (  self		:			Optional[int]					,A		:			Optional[Any]					,A		:			Optional[Any]=False					,A		:			List[Any]=False    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					load_image(inputs["""image"""]    )
   UpperCAmelCase__					:  Optional[Any]					   =					self.tokenizer(
       inputs["""question"""]					,return_tensors=self.framework					,padding=A					,truncation=A    )
   UpperCAmelCase__					:  int					   =					self.image_processor(images=A					,return_tensors=self.framework    )
   model_inputs.update(A    )
   return model_inputs

  def    __lowercase       (  self		:			Dict					,A		:			Tuple    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					self.model(**A    )
   return model_outputs




  def    __lowercase       (  self		:			int					,A		:			List[Any]					,A		:			List[str]=5    ):






   '''simple docstring'''
   if top_k > self.model.config.num_labels:
    UpperCAmelCase__					:  Any					   =					self.model.config.num_labels

   if self.framework == "pt":
    UpperCAmelCase__					:  Optional[int]					   =					model_outputs.logits.sigmoid()[0]
    UpperCAmelCase__   ,      UpperCAmelCase__					:  Optional[int]					   =					probs.topk(A    )
   else:
    raise ValueError(f"Unsupported framework: {self.framework}"    )

   UpperCAmelCase__					:  List[str]					   =					scores.tolist()
   UpperCAmelCase__					:  Optional[int]					   =					ids.tolist()
   return [{"score": score, "answer": self.model.config.idalabel[_id]} for score, _id in zip(A					,A    )]





