
"""simple docstring"""




import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
  from PIL import Image

  from transformers import OwlViTImageProcessor, OwlViTProcessor


@require_vision
class       __magic_name__							(							unittest.TestCase			):
       def 				lowercase_				(  self							)  ->						Any:





         """simple docstring"""




         _lowercase:      str   			=  tempfile.mkdtemp()

         # fmt: off
         _lowercase:      Tuple   			=  ['''''', '''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
         # fmt: on
         _lowercase:      Tuple   			=  dict(zip(lowerCAmelCase__			,					range(len(lowerCAmelCase__							)							)							)							)
         _lowercase:      int   			=  ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''', '''''']
         _lowercase:      List[Any]   			=  {'''unk_token''': '''<unk>'''}

         _lowercase:      Union[str, Any]   			=  os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['''vocab_file''']							)
         _lowercase:      str   			=  os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['''merges_file''']							)
         with open(self.vocab_file			,					'''w'''			,					encoding='''utf-8'''							) as fp:
           fp.write(json.dumps(lowerCAmelCase__							) + '''\n'''							)
         with open(self.merges_file			,					'''w'''			,					encoding='''utf-8'''							) as fp:
           fp.write('''\n'''.join(lowerCAmelCase__							)							)

         _lowercase:      Tuple   			=  {
             '''do_resize''': True,
             '''size''': 20,
             '''do_center_crop''': True,
             '''crop_size''': 18,
             '''do_normalize''': True,
             '''image_mean''': [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
             '''image_std''': [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
         }
         _lowercase:      Tuple   			=  os.path.join(self.tmpdirname			,					lowerCAmelCase__							)
         with open(self.image_processor_file			,					'''w'''			,					encoding='''utf-8'''							) as fp:
           json.dump(lowerCAmelCase__			,					lowerCAmelCase__							)
       def 				lowercase_				(  self			,					**A_							)  ->						List[str]:





         """simple docstring"""




         return CLIPTokenizer.from_pretrained(self.tmpdirname			,					pad_token='''!'''			,					**lowerCAmelCase__							)
       def 				lowercase_				(  self			,					**A_							)  ->						str:





         """simple docstring"""




         return CLIPTokenizerFast.from_pretrained(self.tmpdirname			,					pad_token='''!'''			,					**lowerCAmelCase__							)
       def 				lowercase_				(  self			,					**A_							)  ->						Dict:





         """simple docstring"""




         return OwlViTImageProcessor.from_pretrained(self.tmpdirname			,					**lowerCAmelCase__							)
       def 				lowercase_				(  self							)  ->						Any:





         """simple docstring"""




         shutil.rmtree(self.tmpdirname							)
       def 				lowercase_				(  self							)  ->						int:





         """simple docstring"""




         _lowercase:      Union[str, Any]   			=  [np.random.randint(255			,					size=(3, 30, 400)			,					dtype=np.uinta							)]

         _lowercase:      int   			=  [Image.fromarray(np.moveaxis(lowerCAmelCase__			,					0			,					-1							)							) for x in image_inputs]

         return image_inputs
       def 				lowercase_				(  self							)  ->						List[str]:





         """simple docstring"""




         _lowercase:      List[Any]   			=  self.get_tokenizer()
         _lowercase:      Union[str, Any]   			=  self.get_rust_tokenizer()
         _lowercase:      Optional[int]   			=  self.get_image_processor()

         _lowercase:      Union[str, Any]   			=  OwlViTProcessor(tokenizer=lowerCAmelCase__			,					image_processor=lowerCAmelCase__							)
         processor_slow.save_pretrained(self.tmpdirname							)
         _lowercase:      List[str]   			=  OwlViTProcessor.from_pretrained(self.tmpdirname			,					use_fast=lowerCAmelCase__							)

         _lowercase:      Optional[int]   			=  OwlViTProcessor(tokenizer=lowerCAmelCase__			,					image_processor=lowerCAmelCase__							)
         processor_fast.save_pretrained(self.tmpdirname							)
         _lowercase:      Union[str, Any]   			=  OwlViTProcessor.from_pretrained(self.tmpdirname							)

         self.assertEqual(processor_slow.tokenizer.get_vocab()			,					tokenizer_slow.get_vocab()							)
         self.assertEqual(processor_fast.tokenizer.get_vocab()			,					tokenizer_fast.get_vocab()							)
         self.assertEqual(tokenizer_slow.get_vocab()			,					tokenizer_fast.get_vocab()							)
         self.assertIsInstance(processor_slow.tokenizer			,					lowerCAmelCase__							)
         self.assertIsInstance(processor_fast.tokenizer			,					lowerCAmelCase__							)

         self.assertEqual(processor_slow.image_processor.to_json_string()			,					image_processor.to_json_string()							)
         self.assertEqual(processor_fast.image_processor.to_json_string()			,					image_processor.to_json_string()							)
         self.assertIsInstance(processor_slow.image_processor			,					lowerCAmelCase__							)
         self.assertIsInstance(processor_fast.image_processor			,					lowerCAmelCase__							)
       def 				lowercase_				(  self							)  ->						str:





         """simple docstring"""




         _lowercase:      Any   			=  OwlViTProcessor(tokenizer=self.get_tokenizer()			,					image_processor=self.get_image_processor()							)
         processor.save_pretrained(self.tmpdirname							)

         _lowercase:      str   			=  self.get_tokenizer(bos_token='''(BOS)'''			,					eos_token='''(EOS)'''							)
         _lowercase:      str   			=  self.get_image_processor(do_normalize=lowerCAmelCase__							)

         _lowercase:      List[str]   			=  OwlViTProcessor.from_pretrained(
             self.tmpdirname			,					bos_token='''(BOS)'''			,					eos_token='''(EOS)'''			,					do_normalize=lowerCAmelCase__							)

         self.assertEqual(processor.tokenizer.get_vocab()			,					tokenizer_add_kwargs.get_vocab()							)
         self.assertIsInstance(processor.tokenizer			,					lowerCAmelCase__							)

         self.assertEqual(processor.image_processor.to_json_string()			,					image_processor_add_kwargs.to_json_string()							)
         self.assertIsInstance(processor.image_processor			,					lowerCAmelCase__							)
       def 				lowercase_				(  self							)  ->						Union[str, Any]:





         """simple docstring"""




         _lowercase:      str   			=  self.get_image_processor()
         _lowercase:      Optional[int]   			=  self.get_tokenizer()

         _lowercase:      List[Any]   			=  OwlViTProcessor(tokenizer=lowerCAmelCase__			,					image_processor=lowerCAmelCase__							)

         _lowercase:      List[str]   			=  self.prepare_image_inputs()

         _lowercase:      Tuple   			=  image_processor(lowerCAmelCase__			,					return_tensors='''np'''							)
         _lowercase:      Dict   			=  processor(images=lowerCAmelCase__			,					return_tensors='''np'''							)

         for key in input_image_proc.keys():
           self.assertAlmostEqual(input_image_proc[key].sum()			,					input_processor[key].sum()			,					delta=1E-2							)
       def 				lowercase_				(  self							)  ->						Dict:





         """simple docstring"""




         _lowercase:      Optional[int]   			=  self.get_image_processor()
         _lowercase:      Any   			=  self.get_tokenizer()

         _lowercase:      int   			=  OwlViTProcessor(tokenizer=lowerCAmelCase__			,					image_processor=lowerCAmelCase__							)

         _lowercase:      str   			=  '''lower newer'''

         _lowercase:      List[Any]   			=  processor(text=lowerCAmelCase__			,					return_tensors='''np'''							)

         _lowercase:      str   			=  tokenizer(lowerCAmelCase__			,					return_tensors='''np'''							)

         for key in encoded_tok.keys():
           self.assertListEqual(encoded_tok[key][0].tolist()			,					encoded_processor[key][0].tolist()							)
       def 				lowercase_				(  self							)  ->						List[str]:





         """simple docstring"""




         _lowercase:      int   			=  self.get_image_processor()
         _lowercase:      Tuple   			=  self.get_tokenizer()

         _lowercase:      Union[str, Any]   			=  OwlViTProcessor(tokenizer=lowerCAmelCase__			,					image_processor=lowerCAmelCase__							)

         _lowercase:      Optional[int]   			=  '''lower newer'''
         _lowercase:      List[str]   			=  self.prepare_image_inputs()

         _lowercase:      Tuple   			=  processor(text=lowerCAmelCase__			,					images=lowerCAmelCase__							)

         self.assertListEqual(list(inputs.keys()							)			,					['''input_ids''', '''attention_mask''', '''pixel_values''']							)

         # test if it raises when no input is passed
         with pytest.raises(lowerCAmelCase__							):
           processor()
       def 				lowercase_				(  self							)  ->						Optional[int]:





         """simple docstring"""




         _lowercase:      str   			=  '''google/owlvit-base-patch32'''
         _lowercase:      Union[str, Any]   			=  OwlViTProcessor.from_pretrained(lowerCAmelCase__							)

         _lowercase:      Dict   			=  ['''cat''', '''nasa badge''']
         _lowercase:      List[str]   			=  processor(text=lowerCAmelCase__							)

         _lowercase:      List[Any]   			=  16
         self.assertListEqual(list(inputs.keys()							)			,					['''input_ids''', '''attention_mask''']							)
         self.assertEqual(inputs['''input_ids'''].shape			,					(2, seq_length)							)

         # test if it raises when no input is passed
         with pytest.raises(lowerCAmelCase__							):
           processor()
       def 				lowercase_				(  self							)  ->						List[Any]:





         """simple docstring"""




         _lowercase:      Union[str, Any]   			=  '''google/owlvit-base-patch32'''
         _lowercase:      str   			=  OwlViTProcessor.from_pretrained(lowerCAmelCase__							)

         _lowercase:      List[str]   			=  [['''cat''', '''nasa badge'''], ['''person''']]
         _lowercase:      Tuple   			=  processor(text=lowerCAmelCase__							)

         _lowercase:      Any   			=  16
         _lowercase:      Dict   			=  len(lowerCAmelCase__							)
         _lowercase:      List[str]   			=  max([len(lowerCAmelCase__							) for texts in input_texts]							)

         self.assertListEqual(list(inputs.keys()							)			,					['''input_ids''', '''attention_mask''']							)
         self.assertEqual(inputs['''input_ids'''].shape			,					(batch_size * num_max_text_queries, seq_length)							)

         # test if it raises when no input is passed
         with pytest.raises(lowerCAmelCase__							):
           processor()
       def 				lowercase_				(  self							)  ->						Union[str, Any]:





         """simple docstring"""




         _lowercase:      Dict   			=  '''google/owlvit-base-patch32'''
         _lowercase:      Optional[int]   			=  OwlViTProcessor.from_pretrained(lowerCAmelCase__							)

         _lowercase:      str   			=  ['''cat''', '''nasa badge''']
         _lowercase:      List[Any]   			=  processor(text=lowerCAmelCase__							)

         _lowercase:      Optional[int]   			=  16
         _lowercase:      str   			=  inputs['''input_ids''']
         _lowercase:      Dict   			=  [
             [4_9406, 2368, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [4_9406, 6841, 1_1301, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         ]

         self.assertListEqual(list(inputs.keys()							)			,					['''input_ids''', '''attention_mask''']							)
         self.assertEqual(inputs['''input_ids'''].shape			,					(2, seq_length)							)
         self.assertListEqual(list(input_ids[0]							)			,					predicted_ids[0]							)
         self.assertListEqual(list(input_ids[1]							)			,					predicted_ids[1]							)
       def 				lowercase_				(  self							)  ->						Optional[int]:





         """simple docstring"""




         _lowercase:      List[Any]   			=  self.get_image_processor()
         _lowercase:      Optional[Any]   			=  self.get_tokenizer()

         _lowercase:      List[str]   			=  OwlViTProcessor(tokenizer=lowerCAmelCase__			,					image_processor=lowerCAmelCase__							)

         _lowercase:      Union[str, Any]   			=  self.prepare_image_inputs()
         _lowercase:      Dict   			=  self.prepare_image_inputs()

         _lowercase:      str   			=  processor(images=lowerCAmelCase__			,					query_images=lowerCAmelCase__							)

         self.assertListEqual(list(inputs.keys()							)			,					['''query_pixel_values''', '''pixel_values''']							)

         # test if it raises when no input is passed
         with pytest.raises(lowerCAmelCase__							):
           processor()



       def 				lowercase_				(  self							)  ->						Union[str, Any]:





         """simple docstring"""




         _lowercase:      str   			=  self.get_image_processor()
         _lowercase:      str   			=  self.get_tokenizer()

         _lowercase:      str   			=  OwlViTProcessor(tokenizer=lowerCAmelCase__			,					image_processor=lowerCAmelCase__							)

         _lowercase:      Optional[Any]   			=  [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

         _lowercase:      List[Any]   			=  processor.batch_decode(lowerCAmelCase__							)
         _lowercase:      Optional[int]   			=  tokenizer.batch_decode(lowerCAmelCase__							)

         self.assertListEqual(lowerCAmelCase__			,					lowerCAmelCase__							)





'''simple docstring'''



import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


lowerCAmelCase_  :							Any             =    'http://www.mocksite.com/file1.txt'
lowerCAmelCase_  :							List[str]             =    '"text": ["foo", "foo"]'
lowerCAmelCase_  :							Any             =    '6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'
class 						lowerCamelCase_				:
						_lowerCAmelCase  : Dict						  =					2_0_0
						_lowerCAmelCase  : int						  =					{'Content-Length': '100'}
						_lowerCAmelCase  : Optional[Any]						  =					{}







						def        __lowercase     (	self :   str					,     **lowerCAmelCase__ :   Any					):

													"""simple docstring"""
													return [bytes(lowerCAmelCase__					,     '''utf-8'''					)]


def        UpperCAmelCase (    *A	:					int       ,		**A	:					Optional[Any]					):
							return MockResponse()
@pytest.mark.parametrize('''urls_type'''       ,		[str, list, dict]					)
def        UpperCAmelCase (    A	:					int       ,		A	:					Union[str, Any]       ,		A	:					Union[str, Any]					):
							import requests

							monkeypatch.setattr(A       ,		'''request'''       ,		A					)

							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      URL
							if issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   List[Any]         =      url
							elif issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   List[Any]         =      [url]
							elif issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   str         =      {'''train''': url}
							SCREAMING_SNAKE_CASE       :   Any         =      '''dummy'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      '''downloads'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      tmp_path
							SCREAMING_SNAKE_CASE       :   List[str]         =      DownloadConfig(
							    cache_dir=os.path.join(A       ,		A					)       ,		use_etag=A       ,		)
							SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      DownloadManager(dataset_name=A       ,		download_config=A					)
							SCREAMING_SNAKE_CASE       :   List[str]         =      dl_manager.download(A					)
							SCREAMING_SNAKE_CASE       :   Dict         =      urls
							for downloaded_paths in [downloaded_paths]:
														if isinstance(A       ,		A					):
																					SCREAMING_SNAKE_CASE       :   List[str]         =      [downloaded_paths]
																					SCREAMING_SNAKE_CASE       :   int         =      [urls]
														elif isinstance(A       ,		A					):
																					assert "train" in downloaded_paths.keys()
																					SCREAMING_SNAKE_CASE       :   Optional[int]         =      downloaded_paths.values()
																					SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      urls.values()
														assert downloaded_paths
														for downloaded_path, input_url in zip(A       ,		A					):
																					assert downloaded_path == dl_manager.downloaded_paths[input_url]
																					SCREAMING_SNAKE_CASE       :   Dict         =      Path(A					)
																					SCREAMING_SNAKE_CASE       :   Optional[Any]         =      downloaded_path.parts
																					assert parts[-1] == HASH
																					assert parts[-2] == cache_subdir
																					assert downloaded_path.exists()
																					SCREAMING_SNAKE_CASE       :   str         =      downloaded_path.read_text()
																					assert content == CONTENT
																					SCREAMING_SNAKE_CASE       :   str         =      downloaded_path.with_suffix('''.json'''					)
																					assert metadata_downloaded_path.exists()
																					SCREAMING_SNAKE_CASE       :   Optional[int]         =      json.loads(metadata_downloaded_path.read_text()					)
																					assert metadata_content == {"url": URL, "etag": None}
@pytest.mark.parametrize('''paths_type'''       ,		[str, list, dict]					)
def        UpperCAmelCase (    A	:					Tuple       ,		A	:					Union[str, Any]       ,		A	:					Dict					):
							SCREAMING_SNAKE_CASE       :   List[str]         =      str(A					)
							if issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   Optional[int]         =      filename
							elif issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   Optional[int]         =      [filename]
							elif issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   List[Any]         =      {'''train''': filename}
							SCREAMING_SNAKE_CASE       :   Dict         =      '''dummy'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      xz_file.parent
							SCREAMING_SNAKE_CASE       :   List[Any]         =      '''extracted'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      DownloadConfig(
							    cache_dir=A       ,		use_etag=A       ,		)
							SCREAMING_SNAKE_CASE       :   Optional[int]         =      DownloadManager(dataset_name=A       ,		download_config=A					)
							SCREAMING_SNAKE_CASE       :   Optional[int]         =      dl_manager.extract(A					)
							SCREAMING_SNAKE_CASE       :   Tuple         =      paths
							for extracted_paths in [extracted_paths]:
														if isinstance(A       ,		A					):
																					SCREAMING_SNAKE_CASE       :   List[Any]         =      [extracted_paths]
																					SCREAMING_SNAKE_CASE       :   str         =      [paths]
														elif isinstance(A       ,		A					):
																					assert "train" in extracted_paths.keys()
																					SCREAMING_SNAKE_CASE       :   Optional[int]         =      extracted_paths.values()
																					SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      paths.values()
														assert extracted_paths
														for extracted_path, input_path in zip(A       ,		A					):
																					assert extracted_path == dl_manager.extracted_paths[input_path]
																					SCREAMING_SNAKE_CASE       :   Any         =      Path(A					)
																					SCREAMING_SNAKE_CASE       :   Optional[Any]         =      extracted_path.parts
																					assert parts[-1] == hash_url_to_filename(A       ,		etag=A					)
																					assert parts[-2] == extracted_subdir
																					assert extracted_path.exists()
																					SCREAMING_SNAKE_CASE       :   int         =      extracted_path.read_text()
																					SCREAMING_SNAKE_CASE       :   Optional[int]         =      text_file.read_text()
																					assert extracted_file_content == expected_file_content
def        UpperCAmelCase (    A	:					Optional[int]       ,		A	:					int					):
							assert path.endswith('''.jsonl'''					)
							for num_items, line in enumerate(A       ,		start=1					):
														SCREAMING_SNAKE_CASE       :   List[Any]         =      json.loads(line.decode('''utf-8'''					)					)
														assert item.keys() == {"col_1", "col_2", "col_3"}
							assert num_items == 4
@pytest.mark.parametrize('''archive_jsonl'''       ,		['''tar_jsonl_path''', '''zip_jsonl_path''']					)
def        UpperCAmelCase (    A	:					Any       ,		A	:					str					):
							SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      request.getfixturevalue(A					)
							SCREAMING_SNAKE_CASE       :   List[Any]         =      DownloadManager()
							for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(A					)       ,		start=1					):
														_test_jsonl(A       ,		A					)
							assert num_jsonl == 2
@pytest.mark.parametrize('''archive_nested_jsonl'''       ,		['''tar_nested_jsonl_path''', '''zip_nested_jsonl_path''']					)
def        UpperCAmelCase (    A	:					Dict       ,		A	:					Optional[Any]					):
							SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      request.getfixturevalue(A					)
							SCREAMING_SNAKE_CASE       :   Optional[int]         =      DownloadManager()
							for num_tar, (path, file) in enumerate(dl_manager.iter_archive(A					)       ,		start=1					):
														for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(A					)       ,		start=1					):
																					_test_jsonl(A       ,		A					)
							assert num_tar == 1
							assert num_jsonl == 2






def        UpperCAmelCase (    A	:					List[Any]					):
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      DownloadManager()
							for num_file, file in enumerate(dl_manager.iter_files(A					)       ,		start=1					):
														assert os.path.basename(A					) == ("test.txt" if num_file == 1 else "train.txt")
							assert num_file == 2




