





import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil


class       __lowerCAmelCase					(       snake_case__      ):


	'''simple docstring'''





	def    snake_case__(						self:						Optional[Any]			):
			lowercase__					:      Optional[int]      =		pa.array(TypedSequence([1, 2, 3]			)			)
			self.assertEqual(arr.type,					pa.intaa()			)

	def    snake_case__(						self:						str			):


			with self.assertRaises(_UpperCamelCase			):
					lowercase__					:      str      =		pa.array(TypedSequence([1, 2, 3]			),					type=pa.intaa()			)

	def    snake_case__(						self:						Optional[Any]			):


			with self.assertRaises(_UpperCamelCase			):
					lowercase__					:      int      =		pa.array(TypedSequence([1, 2, 3],					try_type=Value('bool'			),					type=Value('int64'			)			)			)

	def    snake_case__(						self:						Tuple			):
			lowercase__					:      Dict      =		pa.array(TypedSequence([1, 2, 3],					type=Value('int32'			)			)			)
			self.assertEqual(arr.type,					pa.intaa()			)

	def    snake_case__(						self:						List[Any]			):


			with self.assertRaises((TypeError, pa.lib.ArrowInvalid)			):
					lowercase__					:      Optional[Any]      =		pa.array(TypedSequence(['foo', 'bar'],					type=Value('int64'			)			)			)

	def    snake_case__(						self:						Union[str, Any]			):
			lowercase__					:      Any      =		pa.array(TypedSequence([1, 2, 3],					try_type=Value('int32'			)			)			)
			self.assertEqual(arr.type,					pa.intaa()			)

	def    snake_case__(						self:						Optional[int]			):
			lowercase__					:      str      =		pa.array(TypedSequence(['foo', 'bar'],					try_type=Value('int64'			)			)			)
			self.assertEqual(arr.type,					pa.string()			)

	def    snake_case__(						self:						Tuple			):
			lowercase__					:      Optional[Any]      =		pa.array(TypedSequence([[[1, 2, 3]]],					type=ArrayaD((1, 3),					'int64'			)			)			)
			self.assertEqual(arr.type,					ArrayaDExtensionType((1, 3),					'int64'			)			)

	def    snake_case__(						self:						int			):


			with self.assertRaises((TypeError, pa.lib.ArrowInvalid)			):
					lowercase__					:      int      =		pa.array(TypedSequence(['foo', 'bar'],					type=ArrayaD((1, 3),					'int64'			)			)			)

	def    snake_case__(						self:						Optional[int]			):
			lowercase__					:      Union[str, Any]      =		pa.array(TypedSequence([[[1, 2, 3]]],					try_type=ArrayaD((1, 3),					'int64'			)			)			)
			self.assertEqual(arr.type,					ArrayaDExtensionType((1, 3),					'int64'			)			)

	def    snake_case__(						self:						Any			):
			lowercase__					:      Optional[int]      =		pa.array(TypedSequence(['foo', 'bar'],					try_type=ArrayaD((1, 3),					'int64'			)			)			)
			self.assertEqual(arr.type,					pa.string()			)





	@require_pil
	def    snake_case__(						self:						str			):
			import PIL.Image

			lowercase__					:      Optional[Any]      =		PIL.Image.fromarray(np.arange(10,					dtype=np.uinta			).reshape(2,					5			)			)
			with patch(
			    'datasets.arrow_writer.cast_to_python_objects',					side_effect=_UpperCamelCase			) as mock_cast_to_python_objects:
					lowercase__					:      Any      =		pa.array(TypedSequence([{'path': None, 'bytes': B'image_bytes'}, pil_image],					type=Image()			)			)
					lowercase__   ,				lowercase__					:      Tuple      =		mock_cast_to_python_objects.call_args_list[-1]
					self.assertIn('optimize_list_casting',					_UpperCamelCase			)
					self.assertFalse(kwargs['optimize_list_casting']			)


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : str   ,    _lowercase     : int			)   ->							Any:




		'''simple docstring'''
		lowercase__					:      Optional[int]      =		pa.BufferReader(__A			) if isinstance(__A   ,    pa.Buffer			) else pa.memory_map(__A			)
		lowercase__					:      List[Any]      =		pa.ipc.open_stream(__A			)
		lowercase__					:      str      =		f.read_all()
		assert len(pa_table.to_batches()			) == expected_num_chunks
		assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
		del pa_table


@pytest.mark.parametrize('writer_batch_size'   ,    [None, 1, 10]			)
@pytest.mark.parametrize(
    'fields'   ,    [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : List[Any]   ,    _lowercase     : Optional[Any]			)   ->							Dict:




		'''simple docstring'''
		lowercase__					:      List[Any]      =		pa.BufferOutputStream()
		lowercase__					:      Tuple      =		pa.schema(__A			) if fields else None
		with ArrowWriter(stream=__A   ,    schema=__A   ,    writer_batch_size=__A			) as writer:
				writer.write({'col_1': 'foo', 'col_2': 1}			)
				writer.write({'col_1': 'bar', 'col_2': 2}			)
				lowercase__   ,				lowercase__					:      Any      =		writer.finalize()
		assert num_examples == 2
		assert num_bytes > 0
		if not fields:
				lowercase__					:      Tuple      =		{'col_1': pa.string(), 'col_2': pa.intaa()}
		assert writer._schema == pa.schema(__A   ,    metadata=writer._schema.metadata			)
		_check_output(output.getvalue()   ,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)


def       SCREAMING_SNAKE_CASE__  (       )   ->							List[Any]:




		'''simple docstring'''
		lowercase__					:      str      =		pa.BufferOutputStream()
		lowercase__					:      Optional[Any]      =		Features({'labels': ClassLabel(names=['neg', 'pos']			)}			)
		with ArrowWriter(stream=__A   ,    features=__A			) as writer:
				writer.write({'labels': 0}			)
				writer.write({'labels': 1}			)
				lowercase__   ,				lowercase__					:      Tuple      =		writer.finalize()
		assert num_examples == 2
		assert num_bytes > 0
		assert writer._schema == features.arrow_schema
		assert writer._schema.metadata == features.arrow_schema.metadata
		lowercase__					:      Tuple      =		pa.BufferReader(output.getvalue()			)
		lowercase__					:      List[str]      =		pa.ipc.open_stream(__A			)
		lowercase__					:      List[str]      =		f.read_all()
		lowercase__					:      Optional[int]      =		pa_table.schema
		assert pa_table.num_rows == 2
		assert schema == features.arrow_schema
		assert schema.metadata == features.arrow_schema.metadata
		assert features == Features.from_arrow_schema(__A			)


@pytest.mark.parametrize('writer_batch_size'   ,    [None, 1, 10]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : List[Any]			)   ->							Optional[int]:




		'''simple docstring'''
		lowercase__					:      Union[str, Any]      =		pa.BufferOutputStream()
		with ArrowWriter(
		    stream=__A   ,    writer_batch_size=__A   ,    hash_salt='split_name'   ,    check_duplicates=__A   ,    ) as writer:
				with pytest.raises(__A			):
						writer.write({'col_1': 'foo', 'col_2': 1}   ,    key=[1, 2]			)
						lowercase__   ,				lowercase__					:      str      =		writer.finalize()


@pytest.mark.parametrize('writer_batch_size'   ,    [None, 2, 10]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : List[Any]			)   ->							Tuple:




		'''simple docstring'''
		lowercase__					:      str      =		pa.BufferOutputStream()
		with ArrowWriter(
		    stream=__A   ,    writer_batch_size=__A   ,    hash_salt='split_name'   ,    check_duplicates=__A   ,    ) as writer:
				with pytest.raises(__A			):
						writer.write({'col_1': 'foo', 'col_2': 1}   ,    key=10			)
						writer.write({'col_1': 'bar', 'col_2': 2}   ,    key=10			)
						lowercase__   ,				lowercase__					:      Union[str, Any]      =		writer.finalize()


@pytest.mark.parametrize('writer_batch_size'   ,    [None, 2, 10]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Optional[Any]			)   ->							Union[str, Any]:




		'''simple docstring'''
		lowercase__					:      Dict      =		pa.BufferOutputStream()
		with ArrowWriter(
		    stream=__A   ,    writer_batch_size=__A   ,    hash_salt='split_name'   ,    check_duplicates=__A   ,    ) as writer:
				writer.write({'col_1': 'foo', 'col_2': 1}   ,    key=1			)
				writer.write({'col_1': 'bar', 'col_2': 2}   ,    key=2			)
				lowercase__   ,				lowercase__					:      Any      =		writer.finalize()
		assert num_examples == 2
		assert num_bytes > 0
		_check_output(output.getvalue()   ,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)


@pytest.mark.parametrize('writer_batch_size'   ,    [None, 1, 10]			)
@pytest.mark.parametrize(
    'fields'   ,    [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Optional[Any]   ,    _lowercase     : Dict			)   ->							Union[str, Any]:




		'''simple docstring'''
		lowercase__					:      Any      =		pa.BufferOutputStream()
		lowercase__					:      Union[str, Any]      =		pa.schema(__A			) if fields else None
		with ArrowWriter(stream=__A   ,    schema=__A   ,    writer_batch_size=__A			) as writer:
				writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}			)
				writer.write_batch({'col_1': [], 'col_2': []}			)
				lowercase__   ,				lowercase__					:      Union[str, Any]      =		writer.finalize()
		assert num_examples == 2
		assert num_bytes > 0
		if not fields:
				lowercase__					:      Union[str, Any]      =		{'col_1': pa.string(), 'col_2': pa.intaa()}
		assert writer._schema == pa.schema(__A   ,    metadata=writer._schema.metadata			)
		_check_output(output.getvalue()   ,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)


@pytest.mark.parametrize('writer_batch_size'   ,    [None, 1, 10]			)
@pytest.mark.parametrize(
    'fields'   ,    [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : str   ,    _lowercase     : Dict			)   ->							Dict:




		'''simple docstring'''
		lowercase__					:      Dict      =		pa.BufferOutputStream()
		lowercase__					:      Dict      =		pa.schema(__A			) if fields else None
		with ArrowWriter(stream=__A   ,    schema=__A   ,    writer_batch_size=__A			) as writer:
				writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}			)			)
				lowercase__   ,				lowercase__					:      Optional[int]      =		writer.finalize()
		assert num_examples == 2
		assert num_bytes > 0
		if not fields:
				lowercase__					:      str      =		{'col_1': pa.string(), 'col_2': pa.intaa()}
		assert writer._schema == pa.schema(__A   ,    metadata=writer._schema.metadata			)
		_check_output(output.getvalue()   ,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)


@pytest.mark.parametrize('writer_batch_size'   ,    [None, 1, 10]			)
@pytest.mark.parametrize(
    'fields'   ,    [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Dict   ,    _lowercase     : Optional[Any]			)   ->							Any:




		'''simple docstring'''
		lowercase__					:      List[str]      =		pa.BufferOutputStream()
		lowercase__					:      Any      =		pa.schema(__A			) if fields else None
		with ArrowWriter(stream=__A   ,    schema=__A   ,    writer_batch_size=__A			) as writer:
				writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}			)			)
				writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}			)			)
				lowercase__   ,				lowercase__					:      str      =		writer.finalize()
		assert num_examples == 2
		assert num_bytes > 0
		if not fields:
				lowercase__					:      List[Any]      =		{'col_1': pa.string(), 'col_2': pa.intaa()}
		assert writer._schema == pa.schema(__A   ,    metadata=writer._schema.metadata			)
		_check_output(output.getvalue()   ,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)


def       SCREAMING_SNAKE_CASE__  (       )   ->							Dict:




		'''simple docstring'''




		with tempfile.TemporaryDirectory() as tmp_dir:
				lowercase__					:      List[Any]      =		{'col_1': pa.string(), 'col_2': pa.intaa()}
				lowercase__					:      Dict      =		os.path.join(__A   ,    'test.arrow'			)
				with ArrowWriter(path=__A   ,    schema=pa.schema(__A			)			) as writer:
						writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}			)
						lowercase__   ,				lowercase__					:      str      =		writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				assert writer._schema == pa.schema(__A   ,    metadata=writer._schema.metadata			)
				_check_output(__A   ,    1			)


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Tuple			)   ->							Union[str, Any]:




		'''simple docstring'''




		if pa.types.is_list(__A			):
				return get_base_dtype(arr_type.value_type			)
		else:
				return arr_type


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : str   ,    _lowercase     : Optional[int]			)   ->							Dict:




		'''simple docstring'''




		if isinstance(lst[0]   ,    __A			):
				change_first_primitive_element_in_list(lst[0]   ,    __A			)
		else:
				lowercase__					:      Optional[int]      =		value


@pytest.mark.parametrize('optimized_int_type, expected_dtype'   ,    [(None, pa.intaa()), (Value('int32'			), pa.intaa())]			)
@pytest.mark.parametrize('sequence'   ,    [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Optional[Any]   ,    _lowercase     : str   ,    _lowercase     : List[str]			)   ->							Optional[int]:




		'''simple docstring'''
		lowercase__					:      Optional[Any]      =		pa.array(TypedSequence(__A   ,    optimized_int_type=__A			)			)
		assert get_base_dtype(arr.type			) == expected_dtype


@pytest.mark.parametrize(
    'col, expected_dtype'   ,    [
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]   ,    )
@pytest.mark.parametrize('sequence'   ,    [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : List[str]   ,    _lowercase     : Dict   ,    _lowercase     : Dict			)   ->							Tuple:




		'''simple docstring'''
		lowercase__					:      List[Any]      =		pa.array(OptimizedTypedSequence(__A   ,    col=__A			)			)
		assert get_base_dtype(arr.type			) == expected_dtype

		# not in range
		if col != "other":
				# avoids errors due to in-place modifications
				lowercase__					:      Optional[Any]      =		copy.deepcopy(__A			)
				lowercase__					:      Optional[int]      =		np.iinfo(expected_dtype.to_pandas_dtype()			).max + 1
				change_first_primitive_element_in_list(__A   ,    __A			)
				lowercase__					:      List[Any]      =		pa.array(OptimizedTypedSequence(__A   ,    col=__A			)			)
				assert get_base_dtype(arr.type			) == pa.intaa()


@pytest.mark.parametrize('raise_exception'   ,    [False, True]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : List[Any]   ,    _lowercase     : Union[str, Any]			)   ->							int:




		'''simple docstring'''
		lowercase__					:      str      =		str(tmp_path / 'dataset-train.arrow'			)
		try:
				with ArrowWriter(path=__A			) as writer:
						if raise_exception:
								raise pa.lib.ArrowInvalid()
						else:
								writer.stream.close()
		except pa.lib.ArrowInvalid:
				pass
		finally:
				assert writer.stream.closed


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Tuple			)   ->							List[Any]:




		'''simple docstring'''
		lowercase__					:      int      =		'mock://dataset-train.arrow'
		with ArrowWriter(path=__A   ,    storage_options=mockfs.storage_options			) as writer:
				assert isinstance(writer._fs   ,    type(__A			)			)
				assert writer._fs.storage_options == mockfs.storage_options
				writer.write({'col_1': 'foo', 'col_2': 1}			)
				writer.write({'col_1': 'bar', 'col_2': 2}			)
				lowercase__   ,				lowercase__					:      Dict      =		writer.finalize()
		assert num_examples == 2
		assert num_bytes > 0
		assert mockfs.exists(__A			)


def       SCREAMING_SNAKE_CASE__  (       )   ->							str:




		'''simple docstring'''
		lowercase__					:      Any      =		pa.BufferOutputStream()
		with ParquetWriter(stream=__A			) as writer:
				writer.write({'col_1': 'foo', 'col_2': 1}			)
				writer.write({'col_1': 'bar', 'col_2': 2}			)
				lowercase__   ,				lowercase__					:      str      =		writer.finalize()
		assert num_examples == 2
		assert num_bytes > 0
		lowercase__					:      List[str]      =		pa.BufferReader(output.getvalue()			)
		lowercase__					:      Union[str, Any]      =		pq.read_table(__A			)
		assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}


@require_pil
@pytest.mark.parametrize('embed_local_files'   ,    [False, True]			)
def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Dict   ,    _lowercase     : Optional[int]			)   ->							Any:




		'''simple docstring'''
		import PIL.Image

		lowercase__					:      str      =		str(tmp_path / 'test_image_rgb.jpg'			)
		PIL.Image.fromarray(np.zeros((5, 5)   ,    dtype=np.uinta			)			).save(__A   ,    format='png'			)
		lowercase__					:      str      =		pa.BufferOutputStream()
		with ParquetWriter(
		    stream=__A   ,    features=Features({'image': Image()}			)   ,    embed_local_files=__A			) as writer:
				writer.write({'image': image_path}			)
				writer.finalize()
		lowercase__					:      Dict      =		pa.BufferReader(output.getvalue()			)
		lowercase__					:      List[str]      =		pq.read_table(__A			)
		lowercase__					:      Dict      =		pa_table.to_pydict()
		if embed_local_files:
				assert isinstance(out['image'][0]['path']   ,    __A			)
				with open(__A   ,    'rb'			) as f:
						assert out["image"][0]["bytes"] == f.read()
		else:
				assert out["image"][0]["path"] == image_path
				assert out["image"][0]["bytes"] is None





def       SCREAMING_SNAKE_CASE__  (       )   ->							Union[str, Any]:




		'''simple docstring'''
		lowercase__					:      str      =		pa.schema([pa.field('col_1'   ,    pa.string()   ,    nullable=__A			)]			)
		lowercase__					:      Optional[int]      =		pa.BufferOutputStream()
		with ArrowWriter(stream=__A			) as writer:
				writer._build_writer(inferred_schema=__A			)
				assert writer._schema == pa.schema([pa.field('col_1'   ,    pa.string()			)]			)





import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
    from PIL import Image

    from transformers import CLIPImageProcessor, CLIPProcessor

@require_vision
class     __snake_case    (				unittest.TestCase  ):
       """simple docstring"""







       def 			a		(		self   )      ->							Optional[int]:






              """simple docstring"""


              __snake_case       	=							tempfile.mkdtemp()

              # fmt: off
              __snake_case       	=							["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
              # fmt: on
              __snake_case       	=							dict(zip(_UpperCamelCase      , range(len(_UpperCamelCase   )   )   )   )
              __snake_case       	=							["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
              __snake_case       	=							{"""unk_token""": """<unk>"""}

              __snake_case       	=							os.path.join(self.tmpdirname      , VOCAB_FILES_NAMES["""vocab_file"""]   )
              __snake_case       	=							os.path.join(self.tmpdirname      , VOCAB_FILES_NAMES["""merges_file"""]   )
              with open(self.vocab_file      , """w"""      , encoding="""utf-8"""   ) as fp:
                     fp.write(json.dumps(_UpperCamelCase   ) + """\n"""   )
              with open(self.merges_file      , """w"""      , encoding="""utf-8"""   ) as fp:
                     fp.write("""\n""".join(_UpperCamelCase   )   )

              __snake_case       	=							{
                  """do_resize""": True,
                  """size""": 20,
                  """do_center_crop""": True,
                  """crop_size""": 18,
                  """do_normalize""": True,
                  """image_mean""": [0.4814_5466, 0.457_8275, 0.4082_1073],
                  """image_std""": [0.2686_2954, 0.2613_0258, 0.2757_7711],
              }
              __snake_case       	=							os.path.join(self.tmpdirname      , _UpperCamelCase   )
              with open(self.image_processor_file      , """w"""      , encoding="""utf-8"""   ) as fp:
                     json.dump(_UpperCamelCase      , _UpperCamelCase   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							Optional[int]:






              """simple docstring"""


              return CLIPTokenizer.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							Union[str, Any]:






              """simple docstring"""


              return CLIPTokenizerFast.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							List[str]:






              """simple docstring"""


              return CLIPImageProcessor.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )


       def 			a		(		self   )      ->							int:






              """simple docstring"""


              shutil.rmtree(self.tmpdirname   )


       def 			a		(		self   )      ->							Optional[Any]:






              """simple docstring"""


              __snake_case       	=							[np.random.randint(2_55      , size=(3, 30, 4_00)      , dtype=np.uinta   )]

              __snake_case       	=							[Image.fromarray(np.moveaxis(_UpperCamelCase      , 0      , -1   )   ) for x in image_inputs]

              return image_inputs


       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              __snake_case       	=							self.get_tokenizer()
              __snake_case       	=							self.get_rust_tokenizer()
              __snake_case       	=							self.get_image_processor()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )
              processor_slow.save_pretrained(self.tmpdirname   )
              __snake_case       	=							CLIPProcessor.from_pretrained(self.tmpdirname      , use_fast=_UpperCamelCase   )

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )
              processor_fast.save_pretrained(self.tmpdirname   )
              __snake_case       	=							CLIPProcessor.from_pretrained(self.tmpdirname   )

              self.assertEqual(processor_slow.tokenizer.get_vocab()      , tokenizer_slow.get_vocab()   )
              self.assertEqual(processor_fast.tokenizer.get_vocab()      , tokenizer_fast.get_vocab()   )
              self.assertEqual(tokenizer_slow.get_vocab()      , tokenizer_fast.get_vocab()   )
              self.assertIsInstance(processor_slow.tokenizer      , _UpperCamelCase   )
              self.assertIsInstance(processor_fast.tokenizer      , _UpperCamelCase   )

              self.assertEqual(processor_slow.image_processor.to_json_string()      , image_processor.to_json_string()   )
              self.assertEqual(processor_fast.image_processor.to_json_string()      , image_processor.to_json_string()   )
              self.assertIsInstance(processor_slow.image_processor      , _UpperCamelCase   )
              self.assertIsInstance(processor_fast.image_processor      , _UpperCamelCase   )


       def 			a		(		self   )      ->							List[str]:






              """simple docstring"""


              __snake_case       	=							CLIPProcessor(tokenizer=self.get_tokenizer()      , image_processor=self.get_image_processor()   )
              processor.save_pretrained(self.tmpdirname   )

              __snake_case       	=							self.get_tokenizer(bos_token="""(BOS)"""      , eos_token="""(EOS)"""   )
              __snake_case       	=							self.get_image_processor(do_normalize=_UpperCamelCase      , padding_value=1.0   )

              __snake_case       	=							CLIPProcessor.from_pretrained(
                  self.tmpdirname      , bos_token="""(BOS)"""      , eos_token="""(EOS)"""      , do_normalize=_UpperCamelCase      , padding_value=1.0   )

              self.assertEqual(processor.tokenizer.get_vocab()      , tokenizer_add_kwargs.get_vocab()   )
              self.assertIsInstance(processor.tokenizer      , _UpperCamelCase   )

              self.assertEqual(processor.image_processor.to_json_string()      , image_processor_add_kwargs.to_json_string()   )
              self.assertIsInstance(processor.image_processor      , _UpperCamelCase   )


       def 			a		(		self   )      ->							Any:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							self.prepare_image_inputs()

              __snake_case       	=							image_processor(_UpperCamelCase      , return_tensors="""np"""   )
              __snake_case       	=							processor(images=_UpperCamelCase      , return_tensors="""np"""   )

              for key in input_image_proc.keys():
                     self.assertAlmostEqual(input_image_proc[key].sum()      , input_processor[key].sum()      , delta=1E-2   )


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							"""lower newer"""

              __snake_case       	=							processor(text=_UpperCamelCase   )

              __snake_case       	=							tokenizer(_UpperCamelCase   )

              for key in encoded_tok.keys():
                     self.assertListEqual(encoded_tok[key]      , encoded_processor[key]   )


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							"""lower newer"""
              __snake_case       	=							self.prepare_image_inputs()

              __snake_case       	=							processor(text=_UpperCamelCase      , images=_UpperCamelCase   )

              self.assertListEqual(list(inputs.keys()   )      , ["""input_ids""", """attention_mask""", """pixel_values"""]   )

              # test if it raises when no input is passed
              with pytest.raises(_UpperCamelCase   ):
                     processor()


       def 			a		(		self   )      ->							Any:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

              __snake_case       	=							processor.batch_decode(_UpperCamelCase   )
              __snake_case       	=							tokenizer.batch_decode(_UpperCamelCase   )

              self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )






       def 			a		(		self   )      ->							int:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							"""lower newer"""
              __snake_case       	=							self.prepare_image_inputs()

              __snake_case       	=							processor(text=_UpperCamelCase      , images=_UpperCamelCase   )

              self.assertListEqual(list(inputs.keys()   )      , processor.model_input_names   )

