




import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
		from PIL import Image

		from transformers import OwlViTImageProcessor, OwlViTProcessor

@require_vision
class 						__lowerCAmelCase      (      unittest.TestCase					):
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				tempfile.mkdtemp()

								# fmt: off
								a									=				["""""", """l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
								# fmt: on
								a									=				dict(zip(__magic_name__						, range(len(__magic_name__							)							)							)							)
								a									=				["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
								a									=				{"""unk_token""": """<unk>"""}

								a									=				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["""vocab_file"""]							)
								a									=				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["""merges_file"""]							)
								with open(self.vocab_file						, """w"""						, encoding="""utf-8"""							) as fp:
									fp.write(json.dumps(__magic_name__							) + """\n"""							)
								with open(self.merges_file						, """w"""						, encoding="""utf-8"""							) as fp:
									fp.write("""\n""".join(__magic_name__							)							)

								a									=				{
								    """do_resize""": True,
								    """size""": 20,
								    """do_center_crop""": True,
								    """crop_size""": 18,
								    """do_normalize""": True,
								    """image_mean""": [0.48145466, 0.4578275, 0.40821073],
								    """image_std""": [0.26862954, 0.26130258, 0.27577711],
								}
								a									=				os.path.join(self.tmpdirname						, __magic_name__							)
								with open(self.image_processor_file						, """w"""						, encoding="""utf-8"""							) as fp:
									json.dump(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Optional[Any]						, **__magic_name__		:int							):







								'''simple docstring'''







								return CLIPTokenizer.from_pretrained(self.tmpdirname						, pad_token="""!"""						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:List[Any]						, **__magic_name__		:str							):







								'''simple docstring'''







								return CLIPTokenizerFast.from_pretrained(self.tmpdirname						, pad_token="""!"""						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:Union[str, Any]						, **__magic_name__		:int							):







								'''simple docstring'''







								return OwlViTImageProcessor.from_pretrained(self.tmpdirname						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								shutil.rmtree(self.tmpdirname							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								a									=				[np.random.randint(255						, size=(3, 30, 400)						, dtype=np.uinta							)]

								a									=				[Image.fromarray(np.moveaxis(__magic_name__						, 0						, -1							)							) for x in image_inputs]

								return image_inputs
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				self.get_tokenizer()
								a									=				self.get_rust_tokenizer()
								a									=				self.get_image_processor()

								a									=				OwlViTProcessor(tokenizer=__magic_name__						, image_processor=__magic_name__							)
								processor_slow.save_pretrained(self.tmpdirname							)
								a									=				OwlViTProcessor.from_pretrained(self.tmpdirname						, use_fast=__magic_name__							)

								a									=				OwlViTProcessor(tokenizer=__magic_name__						, image_processor=__magic_name__							)
								processor_fast.save_pretrained(self.tmpdirname							)
								a									=				OwlViTProcessor.from_pretrained(self.tmpdirname							)

								self.assertEqual(processor_slow.tokenizer.get_vocab()						, tokenizer_slow.get_vocab()							)
								self.assertEqual(processor_fast.tokenizer.get_vocab()						, tokenizer_fast.get_vocab()							)
								self.assertEqual(tokenizer_slow.get_vocab()						, tokenizer_fast.get_vocab()							)
								self.assertIsInstance(processor_slow.tokenizer						, __magic_name__							)
								self.assertIsInstance(processor_fast.tokenizer						, __magic_name__							)

								self.assertEqual(processor_slow.image_processor.to_json_string()						, image_processor.to_json_string()							)
								self.assertEqual(processor_fast.image_processor.to_json_string()						, image_processor.to_json_string()							)
								self.assertIsInstance(processor_slow.image_processor						, __magic_name__							)
								self.assertIsInstance(processor_fast.image_processor						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				OwlViTProcessor(tokenizer=self.get_tokenizer()						, image_processor=self.get_image_processor()							)
								processor.save_pretrained(self.tmpdirname							)

								a									=				self.get_tokenizer(bos_token="""(BOS)"""						, eos_token="""(EOS)"""							)
								a									=				self.get_image_processor(do_normalize=__magic_name__							)

								a									=				OwlViTProcessor.from_pretrained(
								    self.tmpdirname						, bos_token="""(BOS)"""						, eos_token="""(EOS)"""						, do_normalize=__magic_name__							)

								self.assertEqual(processor.tokenizer.get_vocab()						, tokenizer_add_kwargs.get_vocab()							)
								self.assertIsInstance(processor.tokenizer						, __magic_name__							)

								self.assertEqual(processor.image_processor.to_json_string()						, image_processor_add_kwargs.to_json_string()							)
								self.assertIsInstance(processor.image_processor						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				self.get_image_processor()
								a									=				self.get_tokenizer()

								a									=				OwlViTProcessor(tokenizer=__magic_name__						, image_processor=__magic_name__							)

								a									=				self.prepare_image_inputs()

								a									=				image_processor(__magic_name__						, return_tensors="""np"""							)
								a									=				processor(images=__magic_name__						, return_tensors="""np"""							)

								for key in input_image_proc.keys():
									self.assertAlmostEqual(input_image_proc[key].sum()						, input_processor[key].sum()						, delta=1E-2							)
							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.get_image_processor()
								a									=				self.get_tokenizer()

								a									=				OwlViTProcessor(tokenizer=__magic_name__						, image_processor=__magic_name__							)

								a									=				"""lower newer"""

								a									=				processor(text=__magic_name__						, return_tensors="""np"""							)

								a									=				tokenizer(__magic_name__						, return_tensors="""np"""							)

								for key in encoded_tok.keys():
									self.assertListEqual(encoded_tok[key][0].tolist()						, encoded_processor[key][0].tolist()							)
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				self.get_image_processor()
								a									=				self.get_tokenizer()

								a									=				OwlViTProcessor(tokenizer=__magic_name__						, image_processor=__magic_name__							)

								a									=				"""lower newer"""
								a									=				self.prepare_image_inputs()

								a									=				processor(text=__magic_name__						, images=__magic_name__							)

								self.assertListEqual(list(inputs.keys()							)						, ["""input_ids""", """attention_mask""", """pixel_values"""]							)

								# test if it raises when no input is passed
								with pytest.raises(__magic_name__							):
									processor()
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				"""google/owlvit-base-patch32"""
								a									=				OwlViTProcessor.from_pretrained(__magic_name__							)

								a									=				["""cat""", """nasa badge"""]
								a									=				processor(text=__magic_name__							)

								a									=				16
								self.assertListEqual(list(inputs.keys()							)						, ["""input_ids""", """attention_mask"""]							)
								self.assertEqual(inputs["""input_ids"""].shape						, (2, seq_length)							)

								# test if it raises when no input is passed
								with pytest.raises(__magic_name__							):
									processor()
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				"""google/owlvit-base-patch32"""
								a									=				OwlViTProcessor.from_pretrained(__magic_name__							)

								a									=				[["""cat""", """nasa badge"""], ["""person"""]]
								a									=				processor(text=__magic_name__							)

								a									=				16
								a									=				len(__magic_name__							)
								a									=				max([len(__magic_name__							) for texts in input_texts]							)

								self.assertListEqual(list(inputs.keys()							)						, ["""input_ids""", """attention_mask"""]							)
								self.assertEqual(inputs["""input_ids"""].shape						, (batch_size * num_max_text_queries, seq_length)							)

								# test if it raises when no input is passed
								with pytest.raises(__magic_name__							):
									processor()
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				"""google/owlvit-base-patch32"""
								a									=				OwlViTProcessor.from_pretrained(__magic_name__							)

								a									=				["""cat""", """nasa badge"""]
								a									=				processor(text=__magic_name__							)

								a									=				16
								a									=				inputs["""input_ids"""]
								a									=				[
								    [4_9406, 2368, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
								    [4_9406, 6841, 1_1301, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
								]

								self.assertListEqual(list(inputs.keys()							)						, ["""input_ids""", """attention_mask"""]							)
								self.assertEqual(inputs["""input_ids"""].shape						, (2, seq_length)							)
								self.assertListEqual(list(input_ids[0]							)						, predicted_ids[0]							)
								self.assertListEqual(list(input_ids[1]							)						, predicted_ids[1]							)
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				self.get_image_processor()
								a									=				self.get_tokenizer()

								a									=				OwlViTProcessor(tokenizer=__magic_name__						, image_processor=__magic_name__							)

								a									=				self.prepare_image_inputs()
								a									=				self.prepare_image_inputs()

								a									=				processor(images=__magic_name__						, query_images=__magic_name__							)

								self.assertListEqual(list(inputs.keys()							)						, ["""query_pixel_values""", """pixel_values"""]							)

								# test if it raises when no input is passed
								with pytest.raises(__magic_name__							):
									processor()





							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				self.get_image_processor()
								a									=				self.get_tokenizer()

								a									=				OwlViTProcessor(tokenizer=__magic_name__						, image_processor=__magic_name__							)

								a									=				[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

								a									=				processor.batch_decode(__magic_name__							)
								a									=				tokenizer.batch_decode(__magic_name__							)

								self.assertListEqual(__magic_name__						, __magic_name__							)







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase   :			Union[str, Any]          = logging.get_logger(__name__)

__UpperCamelCase   :			Optional[Any]          = "▁"

__UpperCamelCase   :			Any          = {"vocab_file": "sentencepiece.bpe.model"}

__UpperCamelCase   :			int          = {
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    }
}

__UpperCamelCase   :			Dict          = {
    "facebook/xglm-564M": 2_048,
}

class 						__lowerCAmelCase      (      __magic_name__					):
							UpperCamelCase__   	=    VOCAB_FILES_NAMES
							UpperCamelCase__   	=    PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__   	=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__   	=    ['''input_ids''', '''attention_mask''']
							def __init__(		self		:Optional[Any]						, __magic_name__		:List[str]						, __magic_name__		:Tuple="<s>"						, __magic_name__		:List[Any]="</s>"						, __magic_name__		:List[str]="</s>"						, __magic_name__		:Any="<s>"						, __magic_name__		:Tuple="<unk>"						, __magic_name__		:int="<pad>"						, __magic_name__		:Optional[Dict[str, Any]] = None						, **__magic_name__		:Optional[Any]						, ):







								'''simple docstring'''







								a									=				{} if sp_model_kwargs is None else sp_model_kwargs

								# Compatibility with the original tokenizer
								a									=				7
								a									=				[F'<madeupword{i}>' for i in range(self.num_madeup_words							)]

								a									=				kwargs.get("""additional_special_tokens"""						, []							)
								kwargs["additional_special_tokens"] += [
								    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    bos_token=__magic_name__						, eos_token=__magic_name__						, unk_token=__magic_name__						, sep_token=__magic_name__						, cls_token=__magic_name__						, pad_token=__magic_name__						, sp_model_kwargs=self.sp_model_kwargs						, **__magic_name__						, )

								a									=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.Load(str(__magic_name__							)							)
								a									=				vocab_file

								# Original fairseq vocab and spm vocab must be "aligned":
								# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
								# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
								# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
								# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

								# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
								a									=				1

								# Mimic fairseq token-to-id alignment for the first 4 token
								a									=				{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

								a									=				len(self.sp_model							)
								a									=				{F'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words							)}
								self.fairseq_tokens_to_ids.update(__magic_name__							)

								a									=				{v: k for k, v in self.fairseq_tokens_to_ids.items()}
							def __getstate__(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.__dict__.copy()
								a									=				None
								a									=				self.sp_model.serialized_model_proto()
								return state
							def __setstate__(		self		:Any						, __magic_name__		:Any							):







								'''simple docstring'''







								a									=				d

								# for backward compatibility
								if not hasattr(self						, """sp_model_kwargs"""							):
									a									=				{}

								a									=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.LoadFromSerializedProto(self.sp_model_proto							)
							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:List[int]						, __magic_name__		:Optional[List[int]] = None							):







								'''simple docstring'''







								if token_ids_a is None:
									return [self.sep_token_id] + token_ids_a
								a									=				[self.sep_token_id]
								return sep + token_ids_a + sep + sep + token_ids_a
							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:List[int]						, __magic_name__		:Optional[List[int]] = None						, __magic_name__		:bool = False							):







								'''simple docstring'''







								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__magic_name__						, token_ids_a=__magic_name__						, already_has_special_tokens=__magic_name__							)

								if token_ids_a is None:
									return [1] + ([0] * len(__magic_name__							))
								return [1] + ([0] * len(__magic_name__							)) + [1, 1] + ([0] * len(__magic_name__							))
							def  lowerCamelCase__				(		self		:Optional[Any]						, __magic_name__		:List[int]						, __magic_name__		:Optional[List[int]] = None							):







								'''simple docstring'''







								a									=				[self.sep_token_id]

								if token_ids_a is None:
									return len(sep + token_ids_a							) * [0]
								return len(sep + token_ids_a + sep + sep + token_ids_a							) * [0]
							@property
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								return len(self.sp_model							) + self.fairseq_offset + self.num_madeup_words
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				{self.convert_ids_to_tokens(__magic_name__							): i for i in range(self.vocab_size							)}
								vocab.update(self.added_tokens_encoder							)
								return vocab
							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:str							):







								'''simple docstring'''







								return self.sp_model.encode(__magic_name__						, out_type=__magic_name__							)
							def  lowerCamelCase__				(		self		:List[str]						, __magic_name__		:Any							):







								'''simple docstring'''







								if token in self.fairseq_tokens_to_ids:
									return self.fairseq_tokens_to_ids[token]
								a									=				self.sp_model.PieceToId(__magic_name__							)

								# Need to return unknown token if the SP model returned 0
								return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
							def  lowerCamelCase__				(		self		:List[str]						, __magic_name__		:Dict							):







								'''simple docstring'''







								if index in self.fairseq_ids_to_tokens:
									return self.fairseq_ids_to_tokens[index]
								return self.sp_model.IdToPiece(index - self.fairseq_offset							)
							def  lowerCamelCase__				(		self		:List[str]						, __magic_name__		:List[str]							):







								'''simple docstring'''







								a									=				"""""".join(__magic_name__							).replace(__magic_name__						, """ """							).strip()
								return out_string





							def  lowerCamelCase__				(		self		:List[Any]						, __magic_name__		:str						, __magic_name__		:Optional[str] = None							):







								'''simple docstring'''







								if not os.path.isdir(__magic_name__							):
									logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
									return
								a									=				os.path.join(
								    __magic_name__						, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

								if os.path.abspath(self.vocab_file							) != os.path.abspath(__magic_name__							) and os.path.isfile(self.vocab_file							):
									copyfile(self.vocab_file						, __magic_name__							)
								elif not os.path.isfile(self.vocab_file							):
									with open(__magic_name__						, """wb"""							) as fi:
										a									=				self.sp_model.serialized_model_proto()
										fi.write(__magic_name__							)

								return (out_vocab_file,)

