



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


UpperCAmelCase							 =					{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase							 =					{
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase							 =					{
    """unc-nlp/lxmert-base-uncased""": 512,
}

UpperCAmelCase							 =					{
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}

class  __snake_case      (   SCREAMING_SNAKE_CASE):







							'''simple docstring'''






							UpperCamelCase__				:      int         =					VOCAB_FILES_NAMES
							UpperCamelCase__				:      str         =					PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__				:      Optional[Any]         =					PRETRAINED_INIT_CONFIGURATION
							UpperCamelCase__				:      Tuple         =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__				:      Dict         =					LxmertTokenizer


							def __init__(							self  ,  a_=None  ,  a_=None  ,  a_=True  ,  a_="[UNK]"  ,  a_="[SEP]"  ,  a_="[PAD]"  ,  a_="[CLS]"  ,  a_="[MASK]"  ,  a_=True  ,  a_=None  ,  **a_  ,  ):
											super().__init__(
											    a_  ,  tokenizer_file=a_  ,  do_lower_case=a_  ,  unk_token=a_  ,  sep_token=a_  ,  pad_token=a_  ,  cls_token=a_  ,  mask_token=a_  ,  tokenize_chinese_chars=a_  ,  strip_accents=a_  ,  **a_  ,  )

											a__				      =		json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
											if (
											    normalizer_state.get("""lowercase"""  ,  a_		) != do_lower_case
											    or normalizer_state.get("""strip_accents"""  ,  a_		) != strip_accents
											    or normalizer_state.get("""handle_chinese_chars"""  ,  a_		) != tokenize_chinese_chars
											):
															a__				      =		getattr(a_  ,  normalizer_state.pop("""type"""		)		)
															a__				      =		do_lower_case
															a__				      =		strip_accents
															a__				      =		tokenize_chinese_chars
															a__				      =		normalizer_class(**a_		)

											a__				      =		do_lower_case


							def _a      (							self  ,  a_  ,  a_=None		):
											a__				      =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

											return output


							def _a      (							self  ,  a_  ,  a_ = None		):
											a__				      =		[self.sep_token_id]
											a__				      =		[self.cls_token_id]
											if token_ids_a is None:
															return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







							def _a      (							self  ,  a_  ,  a_ = None		):
											a__				      =		self._tokenizer.model.save(a_  ,  name=a_		)
											return tuple(a_		)






import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
	from PIL import Image

	from transformers import OwlViTImageProcessor, OwlViTProcessor

@require_vision
class  __snake_case      (   unittest.TestCase):







							'''simple docstring'''






							def _a      (							self		):
											a__				      =		tempfile.mkdtemp()

											# fmt: off
											a__				      =		["""""", """l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
											# fmt: on
											a__				      =		dict(zip(a_  ,  range(len(a_		)		)		)		)
											a__				      =		["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
											a__				      =		{"""unk_token""": """<unk>"""}

											a__				      =		os.path.join(self.tmpdirname  ,  VOCAB_FILES_NAMES["""vocab_file"""]		)
											a__				      =		os.path.join(self.tmpdirname  ,  VOCAB_FILES_NAMES["""merges_file"""]		)
											with open(self.vocab_file  ,  """w"""  ,  encoding="""utf-8"""		) as fp:
															fp.write(json.dumps(a_		) + """\n"""		)
											with open(self.merges_file  ,  """w"""  ,  encoding="""utf-8"""		) as fp:
															fp.write("""\n""".join(a_		)		)

											a__				      =		{
											    """do_resize""": True,
											    """size""": 20,
											    """do_center_crop""": True,
											    """crop_size""": 18,
											    """do_normalize""": True,
											    """image_mean""": [0.48_145_466, 0.4_578_275, 0.40_821_073],
											    """image_std""": [0.26_862_954, 0.26_130_258, 0.27_577_711],
											}
											a__				      =		os.path.join(self.tmpdirname  ,  a_		)
											with open(self.image_processor_file  ,  """w"""  ,  encoding="""utf-8"""		) as fp:
															json.dump(a_  ,  a_		)


							def _a      (							self  ,  **a_		):
											return CLIPTokenizer.from_pretrained(self.tmpdirname  ,  pad_token="""!"""  ,  **a_		)


							def _a      (							self  ,  **a_		):
											return CLIPTokenizerFast.from_pretrained(self.tmpdirname  ,  pad_token="""!"""  ,  **a_		)


							def _a      (							self  ,  **a_		):
											return OwlViTImageProcessor.from_pretrained(self.tmpdirname  ,  **a_		)


							def _a      (							self		):
											shutil.rmtree(self.tmpdirname		)


							def _a      (							self		):
											a__				      =		[np.random.randint(255  ,  size=(3, 30, 400)  ,  dtype=np.uinta		)]

											a__				      =		[Image.fromarray(np.moveaxis(a_  ,  0  ,  -1		)		) for x in image_inputs]

											return image_inputs


							def _a      (							self		):
											a__				      =		self.get_tokenizer()
											a__				      =		self.get_rust_tokenizer()
											a__				      =		self.get_image_processor()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)
											processor_slow.save_pretrained(self.tmpdirname		)
											a__				      =		OwlViTProcessor.from_pretrained(self.tmpdirname  ,  use_fast=a_		)

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)
											processor_fast.save_pretrained(self.tmpdirname		)
											a__				      =		OwlViTProcessor.from_pretrained(self.tmpdirname		)

											self.assertEqual(processor_slow.tokenizer.get_vocab()  ,  tokenizer_slow.get_vocab()		)
											self.assertEqual(processor_fast.tokenizer.get_vocab()  ,  tokenizer_fast.get_vocab()		)
											self.assertEqual(tokenizer_slow.get_vocab()  ,  tokenizer_fast.get_vocab()		)
											self.assertIsInstance(processor_slow.tokenizer  ,  a_		)
											self.assertIsInstance(processor_fast.tokenizer  ,  a_		)

											self.assertEqual(processor_slow.image_processor.to_json_string()  ,  image_processor.to_json_string()		)
											self.assertEqual(processor_fast.image_processor.to_json_string()  ,  image_processor.to_json_string()		)
											self.assertIsInstance(processor_slow.image_processor  ,  a_		)
											self.assertIsInstance(processor_fast.image_processor  ,  a_		)


							def _a      (							self		):
											a__				      =		OwlViTProcessor(tokenizer=self.get_tokenizer()  ,  image_processor=self.get_image_processor()		)
											processor.save_pretrained(self.tmpdirname		)

											a__				      =		self.get_tokenizer(bos_token="""(BOS)"""  ,  eos_token="""(EOS)"""		)
											a__				      =		self.get_image_processor(do_normalize=a_		)

											a__				      =		OwlViTProcessor.from_pretrained(
											    self.tmpdirname  ,  bos_token="""(BOS)"""  ,  eos_token="""(EOS)"""  ,  do_normalize=a_		)

											self.assertEqual(processor.tokenizer.get_vocab()  ,  tokenizer_add_kwargs.get_vocab()		)
											self.assertIsInstance(processor.tokenizer  ,  a_		)

											self.assertEqual(processor.image_processor.to_json_string()  ,  image_processor_add_kwargs.to_json_string()		)
											self.assertIsInstance(processor.image_processor  ,  a_		)


							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		self.prepare_image_inputs()

											a__				      =		image_processor(a_  ,  return_tensors="""np"""		)
											a__				      =		processor(images=a_  ,  return_tensors="""np"""		)

											for key in input_image_proc.keys():
															self.assertAlmostEqual(input_image_proc[key].sum()  ,  input_processor[key].sum()  ,  delta=1E-2		)


							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		"""lower newer"""

											a__				      =		processor(text=a_  ,  return_tensors="""np"""		)

											a__				      =		tokenizer(a_  ,  return_tensors="""np"""		)

											for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key][0].tolist()  ,  encoded_processor[key][0].tolist()		)


							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		"""lower newer"""
											a__				      =		self.prepare_image_inputs()

											a__				      =		processor(text=a_  ,  images=a_		)

											self.assertListEqual(list(inputs.keys()		)  ,  ["""input_ids""", """attention_mask""", """pixel_values"""]		)

											# test if it raises when no input is passed
											with pytest.raises(a_		):
															processor()


							def _a      (							self		):
											a__				      =		"""google/owlvit-base-patch32"""
											a__				      =		OwlViTProcessor.from_pretrained(a_		)

											a__				      =		["""cat""", """nasa badge"""]
											a__				      =		processor(text=a_		)

											a__				      =		16
											self.assertListEqual(list(inputs.keys()		)  ,  ["""input_ids""", """attention_mask"""]		)
											self.assertEqual(inputs["""input_ids"""].shape  ,  (2, seq_length)		)

											# test if it raises when no input is passed
											with pytest.raises(a_		):
															processor()


							def _a      (							self		):
											a__				      =		"""google/owlvit-base-patch32"""
											a__				      =		OwlViTProcessor.from_pretrained(a_		)

											a__				      =		[["""cat""", """nasa badge"""], ["""person"""]]
											a__				      =		processor(text=a_		)

											a__				      =		16
											a__				      =		len(a_		)
											a__				      =		max([len(a_		) for texts in input_texts]		)

											self.assertListEqual(list(inputs.keys()		)  ,  ["""input_ids""", """attention_mask"""]		)
											self.assertEqual(inputs["""input_ids"""].shape  ,  (batch_size * num_max_text_queries, seq_length)		)

											# test if it raises when no input is passed
											with pytest.raises(a_		):
															processor()


							def _a      (							self		):
											a__				      =		"""google/owlvit-base-patch32"""
											a__				      =		OwlViTProcessor.from_pretrained(a_		)

											a__				      =		["""cat""", """nasa badge"""]
											a__				      =		processor(text=a_		)

											a__				      =		16
											a__				      =		inputs["""input_ids"""]
											a__				      =		[
											    [49_406, 2_368, 49_407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											    [49_406, 6_841, 11_301, 49_407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											]

											self.assertListEqual(list(inputs.keys()		)  ,  ["""input_ids""", """attention_mask"""]		)
											self.assertEqual(inputs["""input_ids"""].shape  ,  (2, seq_length)		)
											self.assertListEqual(list(input_ids[0]		)  ,  predicted_ids[0]		)
											self.assertListEqual(list(input_ids[1]		)  ,  predicted_ids[1]		)


							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		self.prepare_image_inputs()
											a__				      =		self.prepare_image_inputs()

											a__				      =		processor(images=a_  ,  query_images=a_		)

											self.assertListEqual(list(inputs.keys()		)  ,  ["""query_pixel_values""", """pixel_values"""]		)

											# test if it raises when no input is passed
											with pytest.raises(a_		):
															processor()







							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											a__				      =		processor.batch_decode(a_		)
											a__				      =		tokenizer.batch_decode(a_		)

											self.assertListEqual(a_  ,  a_		)

