



"""simple docstring"""






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 			__snake_case			(      __a):




  def __init__(					self	:		Tuple		,   __lowerCAmelCase	:		Optional[Any]		,   __lowerCAmelCase	:		Optional[int] = None		,   __lowerCAmelCase	:		Optional[int] = None		,   __lowerCAmelCase	:		List[str] = None		,   __lowerCAmelCase	:		Any = False		,   __lowerCAmelCase	:		int = False		,   __lowerCAmelCase	:		Dict = None		,   __lowerCAmelCase	:		str = None		,   **__lowerCAmelCase	:		Dict		,   ):






         """simple docstring"""






         super().__init__(
             _UpperCamelCase		,   split=_UpperCamelCase		,   features=_UpperCamelCase		,   cache_dir=_UpperCamelCase		,   keep_in_memory=_UpperCamelCase		,   streaming=_UpperCamelCase		,   num_proc=_UpperCamelCase		,   **_UpperCamelCase		,   )
         _lowerCamelCase		:							Optional[Any]            =					field
         _lowerCamelCase		:							List[str]            =					path_or_paths if isinstance(_UpperCamelCase		,   _UpperCamelCase						) else {self.split: path_or_paths}
         _lowerCamelCase		:							Union[str, Any]            =					Json(
             cache_dir=_UpperCamelCase		,   data_files=_UpperCamelCase		,   features=_UpperCamelCase		,   field=_UpperCamelCase		,   **_UpperCamelCase		,   )




  def 		SCREAMING_SNAKE_CASE							(					self	:		Dict						):






         """simple docstring"""






         if self.streaming:
                _lowerCamelCase		:							Any            =					self.builder.as_streaming_dataset(split=self.split						)
         # Build regular (map-style) dataset
         else:
                _lowerCamelCase		:							Optional[Any]            =					None
                _lowerCamelCase		:							List[str]            =					None
                _lowerCamelCase		:							int            =					None
                _lowerCamelCase		:							Optional[int]            =					None

                self.builder.download_and_prepare(
                    download_config=_UpperCamelCase		,   download_mode=_UpperCamelCase		,   verification_mode=_UpperCamelCase		,   base_path=_UpperCamelCase		,   num_proc=self.num_proc		,   )
                _lowerCamelCase		:							Union[str, Any]            =					self.builder.as_dataset(
                    split=self.split		,   verification_mode=_UpperCamelCase		,   in_memory=self.keep_in_memory						)
         return dataset






class 			__snake_case			:




  def __init__(					self	:		Optional[int]		,   __lowerCAmelCase	:		Optional[Any]		,   __lowerCAmelCase	:		Optional[int]		,   __lowerCAmelCase	:		Any = None		,   __lowerCAmelCase	:		Dict = None		,   **__lowerCAmelCase	:		Union[str, Any]		,   ):






         """simple docstring"""






         if num_proc is not None and num_proc <= 0:
                raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''						)

         _lowerCamelCase		:							Optional[int]            =					dataset
         _lowerCamelCase		:							List[Any]            =					path_or_buf
         _lowerCamelCase		:							Any            =					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         _lowerCamelCase		:							Union[str, Any]            =					num_proc
         _lowerCamelCase		:							Optional[Any]            =					'''utf-8'''
         _lowerCamelCase		:							Dict            =					to_json_kwargs




  def 		SCREAMING_SNAKE_CASE							(					self	:		Any						):






         """simple docstring"""






         _lowerCamelCase		:							List[str]            =					self.to_json_kwargs.pop('''path_or_buf'''		,   _UpperCamelCase						)
         _lowerCamelCase		:							Any            =					self.to_json_kwargs.pop('''orient'''		,   '''records'''						)
         _lowerCamelCase		:							str            =					self.to_json_kwargs.pop('''lines'''		,   True if orient == '''records''' else False						)
         _lowerCamelCase		:							Optional[int]            =					self.to_json_kwargs.pop('''index'''		,   False if orient in ['''split''', '''table'''] else True						)
         _lowerCamelCase		:							Optional[int]            =					self.to_json_kwargs.pop('''compression'''		,   _UpperCamelCase						)

         if compression not in [None, "infer", "gzip", "bz2", "xz"]:
                raise NotImplementedError(f'''`datasets` currently does not support {compression} compression'''						)

         if isinstance(self.path_or_buf		,   (str, bytes, os.PathLike)						):
                with fsspec.open(self.path_or_buf		,   '''wb'''		,   compression=_UpperCamelCase						) as buffer:
                       _lowerCamelCase		:							List[str]            =					self._write(file_obj=_UpperCamelCase		,   orient=_UpperCamelCase		,   lines=_UpperCamelCase		,   index=_UpperCamelCase		,   **self.to_json_kwargs						)
         else:
                if compression:
                       raise NotImplementedError(
                           f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
                           ''' was passed. Please provide a local path instead.'''						)
                _lowerCamelCase		:							List[str]            =					self._write(
                    file_obj=self.path_or_buf		,   orient=_UpperCamelCase		,   lines=_UpperCamelCase		,   index=_UpperCamelCase		,   **self.to_json_kwargs						)
         return written




  def 		SCREAMING_SNAKE_CASE							(					self	:		str		,   __lowerCAmelCase	:		Any						):






         """simple docstring"""






         _lowerCamelCase     ,       _lowerCamelCase     ,       _lowerCamelCase     ,       _lowerCamelCase     ,       _lowerCamelCase		:							int            =					args

         _lowerCamelCase		:							Tuple            =					query_table(
             table=self.dataset.data		,   key=slice(_UpperCamelCase		,   offset + self.batch_size						)		,   indices=self.dataset._indices		,   )
         _lowerCamelCase		:							Tuple            =					batch.to_pandas().to_json(
             path_or_buf=_UpperCamelCase		,   orient=_UpperCamelCase		,   lines=_UpperCamelCase		,   index=_UpperCamelCase		,   **_UpperCamelCase						)
         if not json_str.endswith('''\n'''						):
                json_str += "\n"
         return json_str.encode(self.encoding						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		str		,   __lowerCAmelCase	:		Tuple		,   __lowerCAmelCase	:		Union[str, Any]		,   __lowerCAmelCase	:		Optional[int]		,   __lowerCAmelCase	:		List[Any]		,   **__lowerCAmelCase	:		List[str]		,   ):






         """simple docstring"""






         _lowerCamelCase		:							Union[str, Any]            =					0

         if self.num_proc is None or self.num_proc == 1:
                for offset in logging.tqdm(
                    range(0		,   len(self.dataset						)		,   self.batch_size						)		,   unit='''ba'''		,   disable=not logging.is_progress_bar_enabled()		,   desc='''Creating json from Arrow format'''		,   ):
                       _lowerCamelCase		:							Union[str, Any]            =					self._batch_json((offset, orient, lines, index, to_json_kwargs)						)
                       written += file_obj.write(_UpperCamelCase						)
         else:
                _lowerCamelCase     ,       _lowerCamelCase		:							Optional[int]            =					len(self.dataset						), self.batch_size
                with multiprocessing.Pool(self.num_proc						) as pool:
                       for json_str in logging.tqdm(
                           pool.imap(
                               self._batch_json		,   [(offset, orient, lines, index, to_json_kwargs) for offset in range(0		,   _UpperCamelCase		,   _UpperCamelCase						)]		,   )		,   total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,   unit='''ba'''		,   disable=not logging.is_progress_bar_enabled()		,   desc='''Creating json from Arrow format'''		,   ):
                              written += file_obj.write(_UpperCamelCase						)

         return written




import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
			import torch

if is_vision_available():
			from PIL import Image

			from transformers import ChineseCLIPImageProcessor





class       _lowerCAmelCase   (      unittest.TestCase			):




				def __init__(   self							,							_UpperCamelCase							,							_UpperCamelCase=7							,							_UpperCamelCase=3							,							_UpperCamelCase=18							,							_UpperCamelCase=30							,							_UpperCamelCase=400							,							_UpperCamelCase=True							,							_UpperCamelCase=None							,							_UpperCamelCase=True							,							_UpperCamelCase=None							,							_UpperCamelCase=True							,							_UpperCamelCase=[0.48145466, 0.4578275, 0.40821073]							,							_UpperCamelCase=[0.26862954, 0.26130258, 0.27577711]							,							_UpperCamelCase=True							,							)							-> Dict:
							lowerCAmelCase_         =      size if size is not None else {"height": 224, "width": 224}
							lowerCAmelCase_         =      crop_size if crop_size is not None else {"height": 18, "width": 18}
							lowerCAmelCase_         =      parent
							lowerCAmelCase_         =      batch_size
							lowerCAmelCase_         =      num_channels
							lowerCAmelCase_         =      image_size
							lowerCAmelCase_         =      min_resolution
							lowerCAmelCase_         =      max_resolution
							lowerCAmelCase_         =      do_resize
							lowerCAmelCase_         =      size
							lowerCAmelCase_         =      do_center_crop
							lowerCAmelCase_         =      crop_size
							lowerCAmelCase_         =      do_normalize
							lowerCAmelCase_         =      image_mean
							lowerCAmelCase_         =      image_std
							lowerCAmelCase_         =      do_convert_rgb




				def 	__a		(   self	)							-> List[Any]:
							return {
							    "do_resize": self.do_resize,
							    "size": self.size,
							    "do_center_crop": self.do_center_crop,
							    "crop_size": self.crop_size,
							    "do_normalize": self.do_normalize,
							    "image_mean": self.image_mean,
							    "image_std": self.image_std,
							    "do_convert_rgb": self.do_convert_rgb,
							}




				def 	__a		(   self							,							_UpperCamelCase=False							,							_UpperCamelCase=False							,							_UpperCamelCase=False	)							-> Dict:
							assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

							if equal_resolution:
										lowerCAmelCase_         =      []
										for i in range(self.batch_size	):
													image_inputs.append(
													    np.random.randint(
													        255							,							size=(self.num_channels, self.max_resolution, self.max_resolution)							,							dtype=np.uinta	)	)
							else:
										lowerCAmelCase_         =      []
										for i in range(self.batch_size	):
													lowerCAmelCase_      ,    lowerCAmelCase_         =      np.random.choice(np.arange(self.min_resolution							,							self.max_resolution	)							,							2	)
													image_inputs.append(np.random.randint(255							,							size=(self.num_channels, width, height)							,							dtype=np.uinta	)	)

							if not numpify and not torchify:
										# PIL expects the channel dimension as last dimension
										lowerCAmelCase_         =      [Image.fromarray(np.moveaxis(_UpperCamelCase							,							0							,							-1	)	) for x in image_inputs]

							if torchify:
										lowerCAmelCase_         =      [torch.from_numpy(_UpperCamelCase	) for x in image_inputs]

							return image_inputs





@require_torch
@require_vision
class       _lowerCAmelCase   (      __a   , unittest.TestCase			):
				_lowercase     						=ChineseCLIPImageProcessor if is_vision_available() else None




				def 	__a		(   self	)							-> Union[str, Any]:
							lowerCAmelCase_         =      ChineseCLIPImageProcessingTester(self							,							do_center_crop=_UpperCamelCase	)




				@property
				def 	__a		(   self	)							-> Tuple:
							return self.image_processor_tester.prepare_image_processor_dict()




				def 	__a		(   self	)							-> Any:
							lowerCAmelCase_         =      self.image_processing_class(**self.image_processor_dict	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"do_resize"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"size"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"do_center_crop"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"center_crop"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"do_normalize"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"image_mean"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"image_std"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"do_convert_rgb"	)	)




				def 	__a		(   self	)							-> List[str]:
							lowerCAmelCase_         =      self.image_processing_class.from_dict(self.image_processor_dict	)
							self.assertEqual(image_processor.size							,							{"height": 224, "width": 224}	)
							self.assertEqual(image_processor.crop_size							,							{"height": 18, "width": 18}	)

							lowerCAmelCase_         =      self.image_processing_class.from_dict(self.image_processor_dict							,							size=42							,							crop_size=84	)
							self.assertEqual(image_processor.size							,							{"shortest_edge": 42}	)
							self.assertEqual(image_processor.crop_size							,							{"height": 84, "width": 84}	)




				def 	__a		(   self	)							-> str:
							pass




				def 	__a		(   self	)							-> Any:
							# Initialize image_processing
							lowerCAmelCase_         =      self.image_processing_class(**self.image_processor_dict	)
							# create random PIL images
							lowerCAmelCase_         =      self.image_processor_tester.prepare_inputs(equal_resolution=_UpperCamelCase	)
							for image in image_inputs:
										self.assertIsInstance(_UpperCamelCase							,							Image.Image	)

							# Test not batched input
							lowerCAmelCase_         =      image_processing(image_inputs[0]							,							return_tensors="pt"	).pixel_values
							self.assertEqual(
							    encoded_images.shape							,							(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )							,							)

							# Test batched
							lowerCAmelCase_         =      image_processing(_UpperCamelCase							,							return_tensors="pt"	).pixel_values
							self.assertEqual(
							    encoded_images.shape							,							(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )							,							)




				def 	__a		(   self	)							-> str:
							# Initialize image_processing
							lowerCAmelCase_         =      self.image_processing_class(**self.image_processor_dict	)
							# create random numpy tensors
							lowerCAmelCase_         =      self.image_processor_tester.prepare_inputs(equal_resolution=_UpperCamelCase							,							numpify=_UpperCamelCase	)
							for image in image_inputs:
										self.assertIsInstance(_UpperCamelCase							,							np.ndarray	)

							# Test not batched input
							lowerCAmelCase_         =      image_processing(image_inputs[0]							,							return_tensors="pt"	).pixel_values
							self.assertEqual(
							    encoded_images.shape							,							(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )							,							)

							# Test batched
							lowerCAmelCase_         =      image_processing(_UpperCamelCase							,							return_tensors="pt"	).pixel_values
							self.assertEqual(
							    encoded_images.shape							,							(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )							,							)




				def 	__a		(   self	)							-> Union[str, Any]:
							# Initialize image_processing
							lowerCAmelCase_         =      self.image_processing_class(**self.image_processor_dict	)
							# create random PyTorch tensors
							lowerCAmelCase_         =      self.image_processor_tester.prepare_inputs(equal_resolution=_UpperCamelCase							,							torchify=_UpperCamelCase	)
							for image in image_inputs:
										self.assertIsInstance(_UpperCamelCase							,							torch.Tensor	)

							# Test not batched input
							lowerCAmelCase_         =      image_processing(image_inputs[0]							,							return_tensors="pt"	).pixel_values
							self.assertEqual(
							    encoded_images.shape							,							(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )							,							)

							# Test batched
							lowerCAmelCase_         =      image_processing(_UpperCamelCase							,							return_tensors="pt"	).pixel_values
							self.assertEqual(
							    encoded_images.shape							,							(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )							,							)





@require_torch
@require_vision
class       _lowerCAmelCase   (      __a   , unittest.TestCase			):
				_lowercase     						=ChineseCLIPImageProcessor if is_vision_available() else None




				def 	__a		(   self	)							-> List[Any]:
							lowerCAmelCase_         =      ChineseCLIPImageProcessingTester(self							,							num_channels=4							,							do_center_crop=_UpperCamelCase	)
							lowerCAmelCase_         =      3




				@property
				def 	__a		(   self	)							-> List[str]:
							return self.image_processor_tester.prepare_image_processor_dict()




				def 	__a		(   self	)							-> Union[str, Any]:
							lowerCAmelCase_         =      self.image_processing_class(**self.image_processor_dict	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"do_resize"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"size"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"do_center_crop"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"center_crop"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"do_normalize"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"image_mean"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"image_std"	)	)
							self.assertTrue(hasattr(_UpperCamelCase							,							"do_convert_rgb"	)	)




				def 	__a		(   self	)							-> int:
							pass




				def 	__a		(   self	)							-> str:
							# Initialize image_processing
							lowerCAmelCase_         =      self.image_processing_class(**self.image_processor_dict	)
							# create random PIL images
							lowerCAmelCase_         =      self.image_processor_tester.prepare_inputs(equal_resolution=_UpperCamelCase	)
							for image in image_inputs:
										self.assertIsInstance(_UpperCamelCase							,							Image.Image	)

							# Test not batched input
							lowerCAmelCase_         =      image_processing(image_inputs[0]							,							return_tensors="pt"	).pixel_values
							self.assertEqual(
							    encoded_images.shape							,							(
							        1,
							        self.expected_encoded_image_num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )							,							)

							# Test batched
							lowerCAmelCase_         =      image_processing(_UpperCamelCase							,							return_tensors="pt"	).pixel_values
							self.assertEqual(
							    encoded_images.shape							,							(
							        self.image_processor_tester.batch_size,
							        self.expected_encoded_image_num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )							,							)



