






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class       __lowercase      (	lowercase_  ):







     '''simple docstring'''




     def __init__(				self       :			str ,	UpperCamelCase_       :			NestedDataStructureLike[PathLike] ,	UpperCamelCase_       :			Optional[NamedSplit] = None ,	UpperCamelCase_       :			Optional[Features] = None ,	UpperCamelCase_       :			str = None ,	UpperCamelCase_       :			bool = False ,	UpperCamelCase_       :			bool = False ,	UpperCamelCase_       :			Optional[str] = None ,	UpperCamelCase_       :			Optional[int] = None ,	**UpperCamelCase_       :			int ,	):


         """simple docstring"""



         super().__init__(
             UpperCamelCase_ ,	split=UpperCamelCase_ ,	features=UpperCamelCase_ ,	cache_dir=UpperCamelCase_ ,	keep_in_memory=UpperCamelCase_ ,	streaming=UpperCamelCase_ ,	num_proc=UpperCamelCase_ ,	**UpperCamelCase_ ,	)
         __A         =	field
         __A         =	path_or_paths if isinstance(UpperCamelCase_ ,	UpperCamelCase_				) else {self.split: path_or_paths}
         __A         =	Json(
             cache_dir=UpperCamelCase_ ,	data_files=UpperCamelCase_ ,	features=UpperCamelCase_ ,	field=UpperCamelCase_ ,	**UpperCamelCase_ ,	)





     def 		lowerCAmelCase_     (				self       :			Optional[Any]				):


         """simple docstring"""



         if self.streaming:
             __A         =	self.builder.as_streaming_dataset(split=self.split				)
         # Build regular (map-style) dataset
         else:
             __A         =	None
             __A         =	None
             __A         =	None
             __A         =	None

             self.builder.download_and_prepare(
                 download_config=UpperCamelCase_ ,	download_mode=UpperCamelCase_ ,	verification_mode=UpperCamelCase_ ,	base_path=UpperCamelCase_ ,	num_proc=self.num_proc ,	)
             __A         =	self.builder.as_dataset(
                 split=self.split ,	verification_mode=UpperCamelCase_ ,	in_memory=self.keep_in_memory				)
         return dataset


class       __lowercase      :







     '''simple docstring'''




     def __init__(				self       :			int ,	UpperCamelCase_       :			Dataset ,	UpperCamelCase_       :			Union[PathLike, BinaryIO] ,	UpperCamelCase_       :			Optional[int] = None ,	UpperCamelCase_       :			Optional[int] = None ,	**UpperCamelCase_       :			Any ,	):


         """simple docstring"""



         if num_proc is not None and num_proc <= 0:
             raise ValueError(F"num_proc {num_proc} must be an integer > 0."				)

         __A         =	dataset
         __A         =	path_or_buf
         __A         =	batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         __A         =	num_proc
         __A         =	"""utf-8"""
         __A         =	to_json_kwargs




     def 		lowerCAmelCase_     (				self       :			Optional[Any]				):


         """simple docstring"""



         __A         =	self.to_json_kwargs.pop("""path_or_buf""" ,	UpperCamelCase_				)
         __A         =	self.to_json_kwargs.pop("""orient""" ,	"""records"""				)
         __A         =	self.to_json_kwargs.pop("""lines""" ,	True if orient == """records""" else False				)
         __A         =	self.to_json_kwargs.pop("""index""" ,	False if orient in ["""split""", """table"""] else True				)
         __A         =	self.to_json_kwargs.pop("""compression""" ,	UpperCamelCase_				)

         if compression not in [None, "infer", "gzip", "bz2", "xz"]:
             raise NotImplementedError(F"`datasets` currently does not support {compression} compression"				)

         if isinstance(self.path_or_buf ,	(str, bytes, os.PathLike)				):
             with fsspec.open(self.path_or_buf ,	"""wb""" ,	compression=UpperCamelCase_				) as buffer:
                 __A         =	self._write(file_obj=UpperCamelCase_ ,	orient=UpperCamelCase_ ,	lines=UpperCamelCase_ ,	index=UpperCamelCase_ ,	**self.to_json_kwargs				)
         else:
             if compression:
                 raise NotImplementedError(
                     F"The compression parameter is not supported when writing to a buffer, but compression={compression}"
                     """ was passed. Please provide a local path instead."""				)
             __A         =	self._write(
                 file_obj=self.path_or_buf ,	orient=UpperCamelCase_ ,	lines=UpperCamelCase_ ,	index=UpperCamelCase_ ,	**self.to_json_kwargs				)
         return written




     def 		lowerCAmelCase_     (				self       :			Tuple ,	UpperCamelCase_       :			Union[str, Any]				):


         """simple docstring"""



         __A    ,	__A    ,	__A    ,	__A    ,	__A         =	args

         __A         =	query_table(
             table=self.dataset.data ,	key=slice(UpperCamelCase_ ,	offset + self.batch_size				) ,	indices=self.dataset._indices ,	)
         __A         =	batch.to_pandas().to_json(
             path_or_buf=UpperCamelCase_ ,	orient=UpperCamelCase_ ,	lines=UpperCamelCase_ ,	index=UpperCamelCase_ ,	**UpperCamelCase_				)
         if not json_str.endswith("""\n"""				):
             json_str += "\n"
         return json_str.encode(self.encoding				)





     def 		lowerCAmelCase_     (				self       :			Optional[int] ,	UpperCamelCase_       :			BinaryIO ,	UpperCamelCase_       :			Any ,	UpperCamelCase_       :			int ,	UpperCamelCase_       :			Tuple ,	**UpperCamelCase_       :			List[str] ,	):


         """simple docstring"""



         __A         =	0

         if self.num_proc is None or self.num_proc == 1:
             for offset in logging.tqdm(
                 range(0 ,	len(self.dataset				) ,	self.batch_size				) ,	unit="""ba""" ,	disable=not logging.is_progress_bar_enabled() ,	desc="""Creating json from Arrow format""" ,	):
                 __A         =	self._batch_json((offset, orient, lines, index, to_json_kwargs)				)
                 written += file_obj.write(UpperCamelCase_				)
         else:
             __A    ,	__A         =	len(self.dataset				), self.batch_size
             with multiprocessing.Pool(self.num_proc				) as pool:
                 for json_str in logging.tqdm(
                     pool.imap(
                         self._batch_json ,	[(offset, orient, lines, index, to_json_kwargs) for offset in range(0 ,	UpperCamelCase_ ,	UpperCamelCase_				)] ,	) ,	total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size ,	unit="""ba""" ,	disable=not logging.is_progress_bar_enabled() ,	desc="""Creating json from Arrow format""" ,	):
                     written += file_obj.write(UpperCamelCase_				)

         return written












from __future__ import annotations

import inspect
import unittest

from transformers import ViTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import TFViTForImageClassification, TFViTModel


if is_vision_available():
  from PIL import Image

  from transformers import ViTImageProcessor


class       __lowercase      :







     '''simple docstring'''




     def __init__(				self       :			Tuple ,	UpperCamelCase_       :			Tuple ,	UpperCamelCase_       :			Union[str, Any]=13 ,	UpperCamelCase_       :			Optional[Any]=30 ,	UpperCamelCase_       :			Optional[int]=2 ,	UpperCamelCase_       :			Optional[Any]=3 ,	UpperCamelCase_       :			int=True ,	UpperCamelCase_       :			str=True ,	UpperCamelCase_       :			str=32 ,	UpperCamelCase_       :			Tuple=2 ,	UpperCamelCase_       :			List[Any]=4 ,	UpperCamelCase_       :			List[str]=37 ,	UpperCamelCase_       :			Any="gelu" ,	UpperCamelCase_       :			Any=0.1 ,	UpperCamelCase_       :			Optional[Any]=0.1 ,	UpperCamelCase_       :			Any=10 ,	UpperCamelCase_       :			str=0.02 ,	UpperCamelCase_       :			Dict=3 ,	UpperCamelCase_       :			int=None ,	):


         """simple docstring"""



         __A         =	parent
         __A         =	batch_size
         __A         =	image_size
         __A         =	patch_size
         __A         =	num_channels
         __A         =	is_training
         __A         =	use_labels
         __A         =	hidden_size
         __A         =	num_hidden_layers
         __A         =	num_attention_heads
         __A         =	intermediate_size
         __A         =	hidden_act
         __A         =	hidden_dropout_prob
         __A         =	attention_probs_dropout_prob
         __A         =	type_sequence_label_size
         __A         =	initializer_range
         __A         =	scope

         # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
         __A         =	(image_size // patch_size) ** 2
         __A         =	num_patches + 1




     def 		lowerCAmelCase_     (				self       :			Optional[Any]				):


         """simple docstring"""



         __A         =	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

         __A         =	None
         if self.use_labels:
             __A         =	ids_tensor([self.batch_size] ,	self.type_sequence_label_size				)

         __A         =	self.get_config()

         return config, pixel_values, labels




     def 		lowerCAmelCase_     (				self       :			int				):


         """simple docstring"""



         return ViTConfig(
             image_size=self.image_size ,	patch_size=self.patch_size ,	num_channels=self.num_channels ,	hidden_size=self.hidden_size ,	num_hidden_layers=self.num_hidden_layers ,	num_attention_heads=self.num_attention_heads ,	intermediate_size=self.intermediate_size ,	hidden_act=self.hidden_act ,	hidden_dropout_prob=self.hidden_dropout_prob ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob ,	is_decoder=UpperCamelCase_ ,	initializer_range=self.initializer_range ,	)




     def 		lowerCAmelCase_     (				self       :			Tuple ,	UpperCamelCase_       :			int ,	UpperCamelCase_       :			List[str] ,	UpperCamelCase_       :			Optional[int]				):


         """simple docstring"""



         __A         =	TFViTModel(config=UpperCamelCase_				)
         __A         =	model(UpperCamelCase_ ,	training=UpperCamelCase_				)
         self.parent.assertEqual(result.last_hidden_state.shape ,	(self.batch_size, self.seq_length, self.hidden_size)				)

         # Test with an image with different size than the one specified in config.
         __A         =	self.image_size // 2
         __A         =	pixel_values[:, :, :image_size, :image_size]
         __A         =	model(UpperCamelCase_ ,	interpolate_pos_encoding=UpperCamelCase_ ,	training=UpperCamelCase_				)
         __A         =	(image_size // self.patch_size) ** 2 + 1
         self.parent.assertEqual(result.last_hidden_state.shape ,	(self.batch_size, seq_length, self.hidden_size)				)




     def 		lowerCAmelCase_     (				self       :			Tuple ,	UpperCamelCase_       :			Union[str, Any] ,	UpperCamelCase_       :			Optional[Any] ,	UpperCamelCase_       :			List[Any]				):


         """simple docstring"""



         __A         =	self.type_sequence_label_size
         __A         =	TFViTForImageClassification(UpperCamelCase_				)
         __A         =	model(UpperCamelCase_ ,	labels=UpperCamelCase_ ,	training=UpperCamelCase_				)
         self.parent.assertEqual(result.logits.shape ,	(self.batch_size, self.type_sequence_label_size)				)

         # Test with an image with different size than the one specified in config.
         __A         =	self.image_size // 2
         __A         =	pixel_values[:, :, :image_size, :image_size]
         __A         =	model(UpperCamelCase_ ,	interpolate_pos_encoding=UpperCamelCase_ ,	training=UpperCamelCase_				)
         self.parent.assertEqual(result.logits.shape ,	(self.batch_size, self.type_sequence_label_size)				)

         # test greyscale images
         __A         =	1
         __A         =	TFViTForImageClassification(UpperCamelCase_				)
         __A         =	floats_tensor([self.batch_size, 1, self.image_size, self.image_size]				)
         __A         =	model(UpperCamelCase_				)
         self.parent.assertEqual(result.logits.shape ,	(self.batch_size, self.type_sequence_label_size)				)





     def 		lowerCAmelCase_     (				self       :			Optional[int]				):


         """simple docstring"""



         __A         =	self.prepare_config_and_inputs()
         __A    ,	__A    ,	__A         =	config_and_inputs
         __A         =	{"""pixel_values""": pixel_values}
         return config, inputs_dict


@require_tf
class       __lowercase      (	lowercase_			,      lowercase_			,      unittest.TestCase  ):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE   						=				(TFViTModel, TFViTForImageClassification) if is_tf_available() else ()
     SCREAMING_SNAKE_CASE   						=				(
         {"feature-extraction": TFViTModel, "image-classification": TFViTForImageClassification}
         if is_tf_available()
         else {}
     )

     SCREAMING_SNAKE_CASE   						=				False
     SCREAMING_SNAKE_CASE   						=				False
     SCREAMING_SNAKE_CASE   						=				False




     def 		lowerCAmelCase_     (				self       :			List[Any]				):


         """simple docstring"""



         __A         =	TFViTModelTester(self				)
         __A         =	ConfigTester(self ,	config_class=UpperCamelCase_ ,	has_text_modality=UpperCamelCase_ ,	hidden_size=37				)




     def 		lowerCAmelCase_     (				self       :			Tuple				):


         """simple docstring"""



         self.config_tester.run_common_tests()




     @unittest.skip(reason="""ViT does not use inputs_embeds"""				)
     def 		lowerCAmelCase_     (				self       :			Dict				):


         """simple docstring"""



         pass




     @unittest.skip(reason="""ViT does not use inputs_embeds"""				)
     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         pass




     def 		lowerCAmelCase_     (				self       :			Tuple				):


         """simple docstring"""



         __A    ,	__A         =	self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
             __A         =	model_class(UpperCamelCase_				)
             self.assertIsInstance(model.get_input_embeddings() ,	(tf.keras.layers.Layer)				)
             __A         =	model.get_output_embeddings()
             self.assertTrue(x is None or isinstance(UpperCamelCase_ ,	tf.keras.layers.Layer				)				)




     def 		lowerCAmelCase_     (				self       :			List[str]				):


         """simple docstring"""



         __A    ,	__A         =	self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
             __A         =	model_class(UpperCamelCase_				)
             __A         =	inspect.signature(model.call				)
             # signature.parameters is an OrderedDict => so arg_names order is deterministic
             __A         =	[*signature.parameters.keys()]

             __A         =	["""pixel_values"""]
             self.assertListEqual(arg_names[:1] ,	UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Optional[Any]				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			str				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*UpperCamelCase_				)





     @slow
     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         __A         =	TFViTModel.from_pretrained("""google/vit-base-patch16-224"""				)
         self.assertIsNotNone(UpperCamelCase_				)



def 			_SCREAMING_SNAKE_CASE						( )						->      Union[str, Any]:





    """simple docstring"""

    __A         =	Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""    )
    return image






@require_tf
@require_vision
class       __lowercase      (	unittest.TestCase  ):







     '''simple docstring'''




     @cached_property
     def 		lowerCAmelCase_     (				self       :			List[Any]				):


         """simple docstring"""



         return ViTImageProcessor.from_pretrained("""google/vit-base-patch16-224"""				) if is_vision_available() else None





     @slow
     def 		lowerCAmelCase_     (				self       :			Optional[Any]				):


         """simple docstring"""



         __A         =	TFViTForImageClassification.from_pretrained("""google/vit-base-patch16-224"""				)

         __A         =	self.default_image_processor
         __A         =	prepare_img()
         __A         =	image_processor(images=UpperCamelCase_ ,	return_tensors="""tf"""				)

         # forward pass
         __A         =	model(**UpperCamelCase_				)

         # verify the logits
         __A         =	tf.TensorShape((1, 1_000)				)
         self.assertEqual(outputs.logits.shape ,	UpperCamelCase_				)

         __A         =	tf.constant([-0.2744, 0.8215, -0.0836]				)

         tf.debugging.assert_near(outputs.logits[0, :3] ,	UpperCamelCase_ ,	atol=1e-4				)




