


'''simple docstring'''
import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


lowerCAmelCase						=					[
    {"""dataset""": """wikipedia""", """config_name""": """20220301.de"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.en"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.fr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.frr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.it"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.simple"""},
    {"""dataset""": """snli""", """config_name""": """plain_text"""},
    {"""dataset""": """eli5""", """config_name""": """LFQA_reddit"""},
    {"""dataset""": """wiki40b""", """config_name""": """en"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.compressed"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.no_index"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.multiset.no_index"""},
    {"""dataset""": """natural_questions""", """config_name""": """default"""},
]

def 					lowerCamelCase_ ( __UpperCamelCase					:       List[str]=True		)					->	str:







     """simple docstring"""
     if with_config:
          return [
              {
                  "testcase_name": d["dataset"] + "/" + d["config_name"],
                  "dataset": d["dataset"],
                  "config_name": d["config_name"],
              }
              for d in DATASETS_ON_HF_GCP
          ]
     else:
          return [
              {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
          ]






@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=UpperCAmelCase	)	)
class        lowerCAmelCase_  (     UpperCAmelCase	):
       __UpperCAmelCase					=None
       __UpperCAmelCase					=None






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase   )->      Any:
            with TemporaryDirectory() as tmp_dir:
                 _A				=							dataset_module_factory(_UpperCamelCase			,     cache_dir=_UpperCamelCase   )

                 _A				=							import_main_class(dataset_module.module_path			,     dataset=_UpperCamelCase   )

                 _A				=							builder_cls(
                     cache_dir=_UpperCamelCase			,     config_name=_UpperCamelCase			,     hash=dataset_module.hash			,     )

                 _A				=							'/'.join(
                     [
                         HF_GCP_BASE_URL,
                         builder_instance._relative_data_dir(with_hash=_UpperCamelCase   ).replace(os.sep			,     '/'   ),
                         config.DATASET_INFO_FILENAME,
                     ]   )
                 _A				=							cached_path(_UpperCamelCase			,     cache_dir=_UpperCamelCase   )
                 self.assertTrue(os.path.exists(_UpperCamelCase   )   )






@pytest.mark.integration
def 					lowerCamelCase_ ( __UpperCamelCase					:       Dict		)					->	int:







     """simple docstring"""
     _A				=							tmp_path_factory.mktemp('test_hf_gcp'		) / 'test_wikipedia_simple'
     _A				=							dataset_module_factory('wikipedia'							,    cache_dir=__UpperCamelCase		)
     _A				=							import_main_class(dataset_module.module_path		)
     _A				=							builder_cls(
         cache_dir=__UpperCamelCase							,    config_name='20220301.frr'							,    hash=dataset_module.hash							,    )
     # use the HF cloud storage, not the original download_and_prepare that uses apache-beam
     _A				=							None
     builder_instance.download_and_prepare()
     _A				=							builder_instance.as_dataset()
     assert ds

@pytest.mark.integration
def 					lowerCamelCase_ ( __UpperCamelCase					:       Optional[int]		)					->	int:







     """simple docstring"""
     _A				=							dataset_module_factory('wikipedia'							,    cache_dir=__UpperCamelCase		)
     _A				=							import_main_class(dataset_module.module_path							,    dataset=__UpperCamelCase		)
     _A				=							builder_cls(
         cache_dir=__UpperCamelCase							,    config_name='20220301.frr'							,    hash=dataset_module.hash							,    )
     _A				=							builder_instance.as_streaming_dataset()
     assert ds
     assert isinstance(__UpperCamelCase							,    __UpperCamelCase		)
     assert "train" in ds
     assert isinstance(ds['train']							,    __UpperCamelCase		)
     assert next(iter(ds['train']		)		)









'''simple docstring'''
import gc
import unittest

import numpy as np
import torch

from diffusers import StableDiffusionKDiffusionPipeline
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()






@slow
@require_torch_gpu
class        lowerCAmelCase_  (     unittest.TestCase	):






       def   UpperCamelCase						( self   )->      Optional[int]:
            # clean up the VRAM after each test
            super().tearDown()
            gc.collect()
            torch.cuda.empty_cache()






       def   UpperCamelCase						( self   )->      Union[str, Any]:
            _A				=							StableDiffusionKDiffusionPipeline.from_pretrained('CompVis/stable-diffusion-v1-4'   )
            _A				=							sd_pipe.to(_UpperCamelCase   )
            sd_pipe.set_progress_bar_config(disable=_UpperCamelCase   )

            sd_pipe.set_scheduler('sample_euler'   )

            _A				=							'A painting of a squirrel eating a burger'
            _A				=							torch.manual_seed(0   )
            _A				=							sd_pipe([prompt]			,     generator=_UpperCamelCase			,     guidance_scale=9.0			,     num_inference_steps=20			,     output_type='np'   )

            _A				=							output.images

            _A				=							image[0, -3:, -3:, -1]

            assert image.shape == (1, 512, 512, 3)
            _A				=							np.array([0.0447, 0.0492, 0.0468, 0.0408, 0.0383, 0.0408, 0.0354, 0.0380, 0.0339]   )

            assert np.abs(image_slice.flatten() - expected_slice   ).max() < 1e-2






       def   UpperCamelCase						( self   )->      Optional[Any]:
            _A				=							StableDiffusionKDiffusionPipeline.from_pretrained('stabilityai/stable-diffusion-2-1-base'   )
            _A				=							sd_pipe.to(_UpperCamelCase   )
            sd_pipe.set_progress_bar_config(disable=_UpperCamelCase   )

            sd_pipe.set_scheduler('sample_euler'   )

            _A				=							'A painting of a squirrel eating a burger'
            _A				=							torch.manual_seed(0   )
            _A				=							sd_pipe([prompt]			,     generator=_UpperCamelCase			,     guidance_scale=9.0			,     num_inference_steps=20			,     output_type='np'   )

            _A				=							output.images

            _A				=							image[0, -3:, -3:, -1]

            assert image.shape == (1, 512, 512, 3)
            _A				=							np.array([0.1237, 0.1320, 0.1438, 0.1359, 0.1390, 0.1132, 0.1277, 0.1175, 0.1112]   )

            assert np.abs(image_slice.flatten() - expected_slice   ).max() < 5e-1






       def   UpperCamelCase						( self   )->      Optional[int]:
            _A				=							StableDiffusionKDiffusionPipeline.from_pretrained('stabilityai/stable-diffusion-2-1-base'   )
            _A				=							sd_pipe.to(_UpperCamelCase   )
            sd_pipe.set_progress_bar_config(disable=_UpperCamelCase   )

            sd_pipe.set_scheduler('sample_dpmpp_2m'   )

            _A				=							'A painting of a squirrel eating a burger'
            _A				=							torch.manual_seed(0   )
            _A				=							sd_pipe(
                [prompt]			,     generator=_UpperCamelCase			,     guidance_scale=7.5			,     num_inference_steps=15			,     output_type='np'			,     use_karras_sigmas=_UpperCamelCase			,     )

            _A				=							output.images

            _A				=							image[0, -3:, -3:, -1]

            assert image.shape == (1, 512, 512, 3)
            _A				=							np.array(
                [0.1138_1689, 0.1211_2921, 0.138_9457, 0.1254_9606, 0.124_4964, 0.1083_1517, 0.1156_2866, 0.1086_7816, 0.1049_9048]   )

            assert np.abs(image_slice.flatten() - expected_slice   ).max() < 1e-2





