






'''simple docstring'''



import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Any,       SCREAMING_SNAKE_CASE__:						List[str]     )	->      Any:

   """simple docstring"""
   assert isinstance(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )
   assert dataset.num_rows == 4
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_1", "col_2", "col_3"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize('keep_in_memory',       [False, True]     )
def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Optional[Any],       SCREAMING_SNAKE_CASE__:						Union[str, Any],       SCREAMING_SNAKE_CASE__:						Dict     )	->      List[str]:

   """simple docstring"""
   __a					  =     tmp_path / 'cache'
   __a					  =     {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      __a					  =     JsonDatasetReader(SCREAMING_SNAKE_CASE__,       cache_dir=SCREAMING_SNAKE_CASE__,       keep_in_memory=SCREAMING_SNAKE_CASE__     ).read()
   _check_json_dataset(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )



@pytest.mark.parametrize(
    'features',       [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ],       )
def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						str,       SCREAMING_SNAKE_CASE__:						List[Any],       SCREAMING_SNAKE_CASE__:						List[str]     )	->      Dict:

   """simple docstring"""
   __a					  =     tmp_path / 'cache'
   __a					  =     {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
   __a					  =     features.copy() if features else default_expected_features
   __a					  =     (
       Features({feature: Value(SCREAMING_SNAKE_CASE__     ) for feature, dtype in features.items()}     ) if features is not None else None
   )
   __a					  =     JsonDatasetReader(SCREAMING_SNAKE_CASE__,       features=SCREAMING_SNAKE_CASE__,       cache_dir=SCREAMING_SNAKE_CASE__     ).read()
   _check_json_dataset(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )



@pytest.mark.parametrize(
    'features',       [
        None,
        {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'},
    ],       )
def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Tuple,       SCREAMING_SNAKE_CASE__:						Union[str, Any],       SCREAMING_SNAKE_CASE__:						int     )	->      Optional[Any]:

   """simple docstring"""
   __a					  =     tmp_path / 'cache'
   __a					  =     {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'}
   __a					  =     features.copy() if features else default_expected_features
   __a					  =     (
       Features({feature: Value(SCREAMING_SNAKE_CASE__     ) for feature, dtype in features.items()}     ) if features is not None else None
   )
   __a					  =     JsonDatasetReader(SCREAMING_SNAKE_CASE__,       features=SCREAMING_SNAKE_CASE__,       cache_dir=SCREAMING_SNAKE_CASE__     ).read()
   assert isinstance(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )
   assert dataset.num_rows == 2
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_3", "col_1", "col_2"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Union[str, Any],       SCREAMING_SNAKE_CASE__:						Dict     )	->      int:

   """simple docstring"""
   # jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
   __a					  =     {'col_2': 'int64', 'col_3': 'float64', 'col_1': 'string'}
   __a					  =     features.copy()
   __a					  =     (
       Features({feature: Value(SCREAMING_SNAKE_CASE__     ) for feature, dtype in features.items()}     ) if features is not None else None
   )
   __a					  =     tmp_path / 'cache'
   __a					  =     JsonDatasetReader(SCREAMING_SNAKE_CASE__,       features=SCREAMING_SNAKE_CASE__,       cache_dir=SCREAMING_SNAKE_CASE__     ).read()
   assert isinstance(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )
   assert dataset.num_rows == 2
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_2", "col_3", "col_1"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize('split',       [None, NamedSplit('train'     ), 'train', 'test']     )
def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Optional[int],       SCREAMING_SNAKE_CASE__:						List[Any],       SCREAMING_SNAKE_CASE__:						Optional[int]     )	->      Union[str, Any]:

   """simple docstring"""
   __a					  =     tmp_path / 'cache'
   __a					  =     {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
   __a					  =     JsonDatasetReader(SCREAMING_SNAKE_CASE__,       cache_dir=SCREAMING_SNAKE_CASE__,       split=SCREAMING_SNAKE_CASE__     ).read()
   _check_json_dataset(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )
   assert dataset.split == split if split else "train"



@pytest.mark.parametrize('path_type',       [str, list]     )
def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Optional[int],       SCREAMING_SNAKE_CASE__:						Any,       SCREAMING_SNAKE_CASE__:						Tuple     )	->      List[str]:

   """simple docstring"""
   if issubclass(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     ):
      __a					  =     jsonl_path
   elif issubclass(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     ):
      __a					  =     [jsonl_path]
   __a					  =     tmp_path / 'cache'
   __a					  =     {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
   __a					  =     JsonDatasetReader(SCREAMING_SNAKE_CASE__,       cache_dir=SCREAMING_SNAKE_CASE__     ).read()
   _check_json_dataset(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Any,       SCREAMING_SNAKE_CASE__:						Any,       SCREAMING_SNAKE_CASE__:						Optional[int]=("train",)     )	->      List[str]:

   """simple docstring"""
   assert isinstance(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )
   for split in splits:
      __a					  =     dataset_dict[split]
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
         assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize('keep_in_memory',       [False, True]     )
def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Union[str, Any],       SCREAMING_SNAKE_CASE__:						Optional[Any],       SCREAMING_SNAKE_CASE__:						str     )	->      Any:

   """simple docstring"""
   __a					  =     tmp_path / 'cache'
   __a					  =     {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      __a					  =     JsonDatasetReader({'train': jsonl_path},       cache_dir=SCREAMING_SNAKE_CASE__,       keep_in_memory=SCREAMING_SNAKE_CASE__     ).read()
   _check_json_datasetdict(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )



@pytest.mark.parametrize(
    'features',       [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ],       )
def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Dict,       SCREAMING_SNAKE_CASE__:						Optional[Any],       SCREAMING_SNAKE_CASE__:						Union[str, Any]     )	->      Any:

   """simple docstring"""
   __a					  =     tmp_path / 'cache'
   __a					  =     {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
   __a					  =     features.copy() if features else default_expected_features
   __a					  =     (
       Features({feature: Value(SCREAMING_SNAKE_CASE__     ) for feature, dtype in features.items()}     ) if features is not None else None
   )
   __a					  =     JsonDatasetReader({'train': jsonl_path},       features=SCREAMING_SNAKE_CASE__,       cache_dir=SCREAMING_SNAKE_CASE__     ).read()
   _check_json_datasetdict(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )



@pytest.mark.parametrize('split',       [None, NamedSplit('train'     ), 'train', 'test']     )
def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Any,       SCREAMING_SNAKE_CASE__:						Union[str, Any],       SCREAMING_SNAKE_CASE__:						str     )	->      Union[str, Any]:

   """simple docstring"""
   if split:
      __a					  =     {split: jsonl_path}
   else:
      __a					  =     'train'
      __a					  =     {'train': jsonl_path, 'test': jsonl_path}
   __a					  =     tmp_path / 'cache'
   __a					  =     {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
   __a					  =     JsonDatasetReader(SCREAMING_SNAKE_CASE__,       cache_dir=SCREAMING_SNAKE_CASE__     ).read()
   _check_json_datasetdict(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__,       splits=list(path.keys()     )     )
   assert all(dataset[split].split == split for split in path.keys()     )



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						List[str]     )	->      Tuple:

   """simple docstring"""
   return json.load(SCREAMING_SNAKE_CASE__     )



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						int     )	->      Any:

   """simple docstring"""
   return [json.loads(SCREAMING_SNAKE_CASE__     ) for line in buffer]



class 				__SCREAMING_SNAKE_CASE							:




      @pytest.mark.parametrize('lines, load_json_function'			, [(True, load_json_lines), (False, load_json)]   )
      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )				->Tuple:



         '''simple docstring'''
         with io.BytesIO() as buffer:
            JsonDatasetWriter(lowerCamelCase			, lowerCamelCase			, lines=lowerCamelCase   ).write()
            buffer.seek(0   )
            __a					  =     load_json_function(lowerCamelCase   )
         assert isinstance(lowerCamelCase			, lowerCamelCase   )
         assert isinstance(exported_content[0]			, lowerCamelCase   )
         assert len(lowerCamelCase   ) == 10




      @pytest.mark.parametrize(
          'orient, container, keys, len_at'			, [
              ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
              ('split', dict, {'columns', 'data'}, 'data'),
              ('index', dict, set('0123456789'   ), None),
              ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
              ('values', list, None, None),
              ('table', dict, {'schema', 'data'}, 'data'),
          ]			, )
      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )				->Tuple:



         '''simple docstring'''
         with io.BytesIO() as buffer:
            JsonDatasetWriter(lowerCamelCase			, lowerCamelCase			, lines=lowerCamelCase			, orient=lowerCamelCase   ).write()
            buffer.seek(0   )
            __a					  =     load_json(lowerCamelCase   )
         assert isinstance(lowerCamelCase			, lowerCamelCase   )
         if keys:
            if container is dict:
               assert exported_content.keys() == keys
            else:
               assert exported_content[0].keys() == keys
         else:
            assert not hasattr(lowerCamelCase			, 'keys'   ) and not hasattr(exported_content[0]			, 'keys'   )
         if len_at:
            assert len(exported_content[len_at]   ) == 10
         else:
            assert len(lowerCamelCase   ) == 10




      @pytest.mark.parametrize('lines, load_json_function'			, [(True, load_json_lines), (False, load_json)]   )
      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )				->Optional[int]:



         '''simple docstring'''
         with io.BytesIO() as buffer:
            JsonDatasetWriter(lowerCamelCase			, lowerCamelCase			, lines=lowerCamelCase			, num_proc=2   ).write()
            buffer.seek(0   )
            __a					  =     load_json_function(lowerCamelCase   )
         assert isinstance(lowerCamelCase			, lowerCamelCase   )
         assert isinstance(exported_content[0]			, lowerCamelCase   )
         assert len(lowerCamelCase   ) == 10




      @pytest.mark.parametrize(
          'orient, container, keys, len_at'			, [
              ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
              ('split', dict, {'columns', 'data'}, 'data'),
              ('index', dict, set('0123456789'   ), None),
              ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
              ('values', list, None, None),
              ('table', dict, {'schema', 'data'}, 'data'),
          ]			, )
      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )				->List[Any]:



         '''simple docstring'''
         with io.BytesIO() as buffer:
            JsonDatasetWriter(lowerCamelCase			, lowerCamelCase			, lines=lowerCamelCase			, orient=lowerCamelCase			, num_proc=2   ).write()
            buffer.seek(0   )
            __a					  =     load_json(lowerCamelCase   )
         assert isinstance(lowerCamelCase			, lowerCamelCase   )
         if keys:
            if container is dict:
               assert exported_content.keys() == keys
            else:
               assert exported_content[0].keys() == keys
         else:
            assert not hasattr(lowerCamelCase			, 'keys'   ) and not hasattr(exported_content[0]			, 'keys'   )
         if len_at:
            assert len(exported_content[len_at]   ) == 10
         else:
            assert len(lowerCamelCase   ) == 10




      def 			__UpperCamelCase				(	self			, lowerCamelCase   )				->str:



         '''simple docstring'''
         with pytest.raises(lowerCamelCase   ):
            with io.BytesIO() as buffer:
               JsonDatasetWriter(lowerCamelCase			, lowerCamelCase			, num_proc=0   )






      @pytest.mark.parametrize('compression, extension'			, [('gzip', 'gz'), ('bz2', 'bz2'), ('xz', 'xz')]   )
      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )				->List[str]:



         '''simple docstring'''
         __a					  =     tmp_path_factory.mktemp('data'   ) / F"""test.json.{extension}"""
         __a					  =     str(shared_datadir / F"""test_file.json.{extension}"""   )
         JsonDatasetWriter(lowerCamelCase			, lowerCamelCase			, compression=lowerCamelCase   ).write()

         with fsspec.open(lowerCamelCase			, 'rb'			, compression='infer'   ) as f:
            __a					  =     f.read()
         with fsspec.open(lowerCamelCase			, 'rb'			, compression='infer'   ) as f:
            __a					  =     f.read()
         assert exported_content == original_content







'''simple docstring'''



from typing import List, Optional, Union

import numpy as np
import PIL.Image

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


__UpperCamelCase   :							Dict         =   logging.get_logger(__name__)
class 				__SCREAMING_SNAKE_CASE							(				_lowerCAmelCase  ):
      __a	       =["pixel_values"]




      def __init__(	self			, lowerCamelCase = True			, lowerCamelCase = 32			, lowerCamelCase=PILImageResampling.BILINEAR			, lowerCamelCase = True			, **lowerCamelCase			, )				->None:



         '''simple docstring'''
         __a					  =     do_resize
         __a					  =     do_rescale
         __a					  =     size_divisor
         __a					  =     resample
         super().__init__(**lowerCamelCase   )




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase = None			, **lowerCamelCase   )				->np.ndarray:



         '''simple docstring'''
         __a  ,      __a					  =     get_image_size(lowerCamelCase   )
         # Rounds the height and width down to the closest multiple of size_divisor
         __a					  =     height // size_divisor * size_divisor
         __a					  =     width // size_divisor * size_divisor
         __a					  =     resize(lowerCamelCase			, (new_h, new_w)			, resample=lowerCamelCase			, data_format=lowerCamelCase			, **lowerCamelCase   )
         return image




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase = None			, **lowerCamelCase   )				->np.ndarray:



         '''simple docstring'''
         return rescale(image=lowerCamelCase			, scale=lowerCamelCase			, data_format=lowerCamelCase			, **lowerCamelCase   )






      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase = None			, lowerCamelCase = None			, lowerCamelCase=None			, lowerCamelCase = None			, lowerCamelCase = None			, lowerCamelCase = ChannelDimension.FIRST			, **lowerCamelCase			, )				->BatchFeature:



         '''simple docstring'''
         __a					  =     do_resize if do_resize is not None else self.do_resize
         __a					  =     do_rescale if do_rescale is not None else self.do_rescale
         __a					  =     size_divisor if size_divisor is not None else self.size_divisor
         __a					  =     resample if resample is not None else self.resample

         if do_resize and size_divisor is None:
            raise ValueError('size_divisor is required for resizing'   )

         __a					  =     make_list_of_images(lowerCamelCase   )

         if not valid_images(lowerCamelCase   ):
            raise ValueError('Invalid image(s)'   )

         # All transformations expect numpy arrays.
         __a					  =     [to_numpy_array(lowerCamelCase   ) for img in images]

         if do_resize:
            __a					  =     [self.resize(lowerCamelCase			, size_divisor=lowerCamelCase			, resample=lowerCamelCase   ) for image in images]

         if do_rescale:
            __a					  =     [self.rescale(lowerCamelCase			, scale=1 / 255   ) for image in images]

         __a					  =     [to_channel_dimension_format(lowerCamelCase			, lowerCamelCase   ) for image in images]

         __a					  =     {'pixel_values': images}
         return BatchFeature(data=lowerCamelCase			, tensor_type=lowerCamelCase   )