




'''simple docstring'''







import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def 		snake_case_							(_a			:							str					,     _a			:							int  ):
    assert isinstance(_a					,     _a  )
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory'''					,     [False, True]  )
def 		snake_case_							(_a			:							List[str]					,     _a			:							Tuple					,     _a			:							Optional[Any]  ):
    UpperCAmelCase     =	tmp_path / '''cache'''
    UpperCAmelCase     =	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        UpperCAmelCase     =	ParquetDatasetReader(_a					,     cache_dir=_a					,     keep_in_memory=_a  ).read()
    _check_parquet_dataset(_a					,     _a  )





@pytest.mark.parametrize(
    '''features'''					,     [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]					,     )
def 		snake_case_							(_a			:							Tuple					,     _a			:							int					,     _a			:							Dict  ):
    UpperCAmelCase     =	tmp_path / '''cache'''
    UpperCAmelCase     =	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
    UpperCAmelCase     =	features.copy() if features else default_expected_features
    UpperCAmelCase     =	(
        Features({feature: Value(_a  ) for feature, dtype in features.items()}  ) if features is not None else None
    )
    UpperCAmelCase     =	ParquetDatasetReader(_a					,     features=_a					,     cache_dir=_a  ).read()
    _check_parquet_dataset(_a					,     _a  )





@pytest.mark.parametrize('''split'''					,     [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def 		snake_case_							(_a			:							Any					,     _a			:							int					,     _a			:							Dict  ):
    UpperCAmelCase     =	tmp_path / '''cache'''
    UpperCAmelCase     =	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
    UpperCAmelCase     =	ParquetDatasetReader(_a					,     cache_dir=_a					,     split=_a  ).read()
    _check_parquet_dataset(_a					,     _a  )
    assert dataset.split == split if split else "train"





@pytest.mark.parametrize('''path_type'''					,     [str, list]  )
def 		snake_case_							(_a			:							Dict					,     _a			:							Optional[Any]					,     _a			:							str  ):
    if issubclass(_a					,     _a  ):
        UpperCAmelCase     =	parquet_path
    elif issubclass(_a					,     _a  ):
        UpperCAmelCase     =	[parquet_path]
    UpperCAmelCase     =	tmp_path / '''cache'''
    UpperCAmelCase     =	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
    UpperCAmelCase     =	ParquetDatasetReader(_a					,     cache_dir=_a  ).read()
    _check_parquet_dataset(_a					,     _a  )





def 		snake_case_							(_a			:							Any					,     _a			:							Union[str, Any]					,     _a			:							Tuple=("train",)  ):
    assert isinstance(_a					,     _a  )
    for split in splits:
        UpperCAmelCase     =	dataset_dict[split]
        assert dataset.num_rows == 4
        assert dataset.num_columns == 3
        assert dataset.column_names == ["col_1", "col_2", "col_3"]
        for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory'''					,     [False, True]  )
def 		snake_case_							(_a			:							Optional[Any]					,     _a			:							Optional[int]					,     _a			:							Dict  ):
    UpperCAmelCase     =	tmp_path / '''cache'''
    UpperCAmelCase     =	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        UpperCAmelCase     =	ParquetDatasetReader(
            {'''train''': parquet_path}					,     cache_dir=_a					,     keep_in_memory=_a  ).read()
    _check_parquet_datasetdict(_a					,     _a  )





@pytest.mark.parametrize(
    '''features'''					,     [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]					,     )
def 		snake_case_							(_a			:							Tuple					,     _a			:							List[str]					,     _a			:							Dict  ):
    UpperCAmelCase     =	tmp_path / '''cache'''
    UpperCAmelCase     =	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
    UpperCAmelCase     =	features.copy() if features else default_expected_features
    UpperCAmelCase     =	(
        Features({feature: Value(_a  ) for feature, dtype in features.items()}  ) if features is not None else None
    )
    UpperCAmelCase     =	ParquetDatasetReader({'''train''': parquet_path}					,     features=_a					,     cache_dir=_a  ).read()
    _check_parquet_datasetdict(_a					,     _a  )





@pytest.mark.parametrize('''split'''					,     [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def 		snake_case_							(_a			:							List[Any]					,     _a			:							Dict					,     _a			:							List[str]  ):
    if split:
        UpperCAmelCase     =	{split: parquet_path}
    else:
        UpperCAmelCase     =	'''train'''
        UpperCAmelCase     =	{'''train''': parquet_path, '''test''': parquet_path}
    UpperCAmelCase     =	tmp_path / '''cache'''
    UpperCAmelCase     =	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
    UpperCAmelCase     =	ParquetDatasetReader(_a					,     cache_dir=_a  ).read()
    _check_parquet_datasetdict(_a					,     _a					,     splits=list(path.keys()  )  )
    assert all(dataset[split].split == split for split in path.keys()  )





def 		snake_case_							(_a			:							Union[str, Any]					,     _a			:							int  ):
    UpperCAmelCase     =	ParquetDatasetWriter(_a					,     tmp_path / '''foo.parquet'''  )
    assert writer.write() > 0
    UpperCAmelCase     =	pq.ParquetFile(tmp_path / '''foo.parquet'''  )
    UpperCAmelCase     =	pf.read()
    assert dataset.data.table == output_table





def 		snake_case_							(_a			:							Any					,     _a			:							Optional[Any]  ):
    UpperCAmelCase     =	str(shared_datadir / '''test_image_rgb.jpg'''  )
    UpperCAmelCase     =	{'''image''': [image_path]}
    UpperCAmelCase     =	Features({'''image''': Image()}  )
    UpperCAmelCase     =	Dataset.from_dict(_a					,     features=_a  )
    UpperCAmelCase     =	ParquetDatasetWriter(_a					,     tmp_path / '''foo.parquet'''  )
    assert writer.write() > 0

    UpperCAmelCase     =	Dataset.from_parquet(str(tmp_path / '''foo.parquet'''  )  )
    assert dataset.features == reloaded_dataset.features

    UpperCAmelCase     =	ParquetDatasetReader(str(tmp_path / '''foo.parquet'''  )					,     streaming=_a  ).read()
    assert dataset.features == reloaded_iterable_dataset.features







@pytest.mark.parametrize(
    '''feature, expected'''					,     [
        (Features({'''foo''': Value('''int32'''  )}  ), None),
        (Features({'''image''': Image(), '''foo''': Value('''int32'''  )}  ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'''nested''': Sequence(Audio()  )}  ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]					,     )
def 		snake_case_							(_a			:							str					,     _a			:							Optional[int]  ):
    assert get_writer_batch_size(_a  ) == expected










'''simple docstring'''







import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEPipeline
from diffusers.pipelines.shap_e import ShapERenderer
from diffusers.utils import load_numpy, slow
from diffusers.utils.testing_utils import require_torch_gpu, torch_device

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


class     _a						(			__a							,							unittest.TestCase				):
       __a       :  Tuple            =      ShapEPipeline
       __a       :  Tuple            =      ["""prompt"""]
       __a       :  Tuple            =      ["""prompt"""]
       __a       :  str            =      [
           """num_images_per_prompt""",
           """num_inference_steps""",
           """generator""",
           """latents""",
           """guidance_scale""",
           """frame_size""",
           """output_type""",
           """return_dict""",
       ]
       __a       :  Union[str, Any]            =      False
       @property
       def 					A   (			self		:			Optional[Any]       ):



           '''simple docstring'''

           return 32
       @property
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           return 32
       @property
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           return self.time_input_dim * 4
       @property
       def 					A   (			self		:			Dict       ):



           '''simple docstring'''

           return 8
       @property
       def 					A   (			self		:			Optional[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''       )
           return tokenizer
       @property
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           torch.manual_seed(0       )
           UpperCAmelCase     =	CLIPTextConfig(
               bos_token_id=0     ,       eos_token_id=2     ,       hidden_size=self.text_embedder_hidden_size     ,       projection_dim=self.text_embedder_hidden_size     ,       intermediate_size=37     ,       layer_norm_eps=1E-05     ,       num_attention_heads=4     ,       num_hidden_layers=5     ,       pad_token_id=1     ,       vocab_size=1_000     ,       )
           return CLIPTextModelWithProjection(lowercase       )
       @property
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           torch.manual_seed(0       )

           UpperCAmelCase     =	{
               '''num_attention_heads''': 2,
               '''attention_head_dim''': 16,
               '''embedding_dim''': self.time_input_dim,
               '''num_embeddings''': 32,
               '''embedding_proj_dim''': self.text_embedder_hidden_size,
               '''time_embed_dim''': self.time_embed_dim,
               '''num_layers''': 1,
               '''clip_embed_dim''': self.time_input_dim * 2,
               '''additional_embeddings''': 0,
               '''time_embed_act_fn''': '''gelu''',
               '''norm_in_type''': '''layer''',
               '''encoder_hid_proj_type''': None,
               '''added_emb_type''': None,
           }

           UpperCAmelCase     =	PriorTransformer(**lowercase       )
           return model
       @property
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           torch.manual_seed(0       )

           UpperCAmelCase     =	{
               '''param_shapes''': (
                   (self.renderer_dim, 93),
                   (self.renderer_dim, 8),
                   (self.renderer_dim, 8),
                   (self.renderer_dim, 8),
               ),
               '''d_latent''': self.time_input_dim,
               '''d_hidden''': self.renderer_dim,
               '''n_output''': 12,
               '''background''': (
                   0.1,
                   0.1,
                   0.1,
               ),
           }
           UpperCAmelCase     =	ShapERenderer(**lowercase       )
           return model
       def 					A   (			self		:			Tuple       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.dummy_prior
           UpperCAmelCase     =	self.dummy_text_encoder
           UpperCAmelCase     =	self.dummy_tokenizer
           UpperCAmelCase     =	self.dummy_renderer

           UpperCAmelCase     =	HeunDiscreteScheduler(
               beta_schedule='''exp'''     ,       num_train_timesteps=1_024     ,       prediction_type='''sample'''     ,       use_karras_sigmas=lowercase     ,       clip_sample=lowercase     ,       clip_sample_range=1.0     ,       )
           UpperCAmelCase     =	{
               '''prior''': prior,
               '''text_encoder''': text_encoder,
               '''tokenizer''': tokenizer,
               '''renderer''': renderer,
               '''scheduler''': scheduler,
           }

           return components
       def 					A   (			self		:			Dict     ,       lowercase		:			Optional[int]     ,       lowercase		:			Optional[int]=0       ):



           '''simple docstring'''

           if str(lowercase       ).startswith('''mps'''       ):
               UpperCAmelCase     =	torch.manual_seed(lowercase       )
           else:
               UpperCAmelCase     =	torch.Generator(device=lowercase       ).manual_seed(lowercase       )
           UpperCAmelCase     =	{
               '''prompt''': '''horse''',
               '''generator''': generator,
               '''num_inference_steps''': 1,
               '''frame_size''': 32,
               '''output_type''': '''np''',
           }
           return inputs
       def 					A   (			self		:			str       ):



           '''simple docstring'''

           UpperCAmelCase     =	'''cpu'''

           UpperCAmelCase     =	self.get_dummy_components()

           UpperCAmelCase     =	self.pipeline_class(**lowercase       )
           UpperCAmelCase     =	pipe.to(lowercase       )

           pipe.set_progress_bar_config(disable=lowercase       )

           UpperCAmelCase     =	pipe(**self.get_dummy_inputs(lowercase       )       )
           UpperCAmelCase     =	output.images[0]
           UpperCAmelCase     =	image[0, -3:, -3:, -1]

           assert image.shape == (20, 32, 32, 3)

           UpperCAmelCase     =	np.array(
               [
                   0.0003_9216,
                   0.0003_9216,
                   0.0003_9216,
                   0.0003_9216,
                   0.0003_9216,
                   0.0003_9216,
                   0.0003_9216,
                   0.0003_9216,
                   0.0003_9216,
               ]       )

           assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2
       def 					A   (			self		:			int       ):



           '''simple docstring'''

           self._test_inference_batch_consistent(batch_sizes=[1, 2]       )
       def 					A   (			self		:			Dict       ):



           '''simple docstring'''

           UpperCAmelCase     =	torch_device == '''cpu'''
           UpperCAmelCase     =	True

           self._test_inference_batch_single_identical(
               batch_size=2     ,       test_max_difference=lowercase     ,       relax_max_difference=lowercase     ,       )







       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.get_dummy_components()
           UpperCAmelCase     =	self.pipeline_class(**lowercase       )
           UpperCAmelCase     =	pipe.to(lowercase       )
           pipe.set_progress_bar_config(disable=lowercase       )

           UpperCAmelCase     =	1
           UpperCAmelCase     =	2

           UpperCAmelCase     =	self.get_dummy_inputs(lowercase       )

           for key in inputs.keys():
               if key in self.batch_params:
                   UpperCAmelCase     =	batch_size * [inputs[key]]

           UpperCAmelCase     =	pipe(**lowercase     ,       num_images_per_prompt=lowercase       )[0]

           assert images.shape[0] == batch_size * num_images_per_prompt






@slow
@require_torch_gpu
class     _a						(			unittest.TestCase				):
       def 					A   (			self		:			str       ):



           '''simple docstring'''

           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()







       def 					A   (			self		:			int       ):



           '''simple docstring'''

           UpperCAmelCase     =	load_numpy(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
               '''/shap_e/test_shap_e_np_out.npy'''       )
           UpperCAmelCase     =	ShapEPipeline.from_pretrained('''openai/shap-e'''       )
           UpperCAmelCase     =	pipe.to(lowercase       )
           pipe.set_progress_bar_config(disable=lowercase       )

           UpperCAmelCase     =	torch.Generator(device=lowercase       ).manual_seed(0       )

           UpperCAmelCase     =	pipe(
               '''a shark'''     ,       generator=lowercase     ,       guidance_scale=15.0     ,       num_inference_steps=64     ,       frame_size=64     ,       output_type='''np'''     ,       ).images[0]

           assert images.shape == (20, 64, 64, 3)

           assert_mean_pixel_difference(lowercase     ,       lowercase       )




