



'''simple docstring'''






import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def    lowerCamelCase__ (		_A				,    _A       ):
  assert isinstance(_A				,    _A       )
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('keep_in_memory'				,    [False, True]       )
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  a		:      Union[str, Any]       =  tmp_path / 'cache'
  a		:      List[Any]       =  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    a		:      Dict       =  JsonDatasetReader(_A				,    cache_dir=_A				,    keep_in_memory=_A       ).read()
  _check_json_dataset(_A				,    _A       )
@pytest.mark.parametrize(
    'features'				,    [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]				,    )
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  a		:      str       =  tmp_path / 'cache'
  a		:      Dict       =  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
  a		:      Dict       =  features.copy() if features else default_expected_features
  a		:      Union[str, Any]       =  (
      Features({feature: Value(_A       ) for feature, dtype in features.items()}       ) if features is not None else None
  )
  a		:      Optional[int]       =  JsonDatasetReader(_A				,    features=_A				,    cache_dir=_A       ).read()
  _check_json_dataset(_A				,    _A       )
@pytest.mark.parametrize(
    'features'				,    [
        None,
        {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'},
    ]				,    )
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  a		:      Optional[Any]       =  tmp_path / 'cache'
  a		:      List[Any]       =  {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'}
  a		:      int       =  features.copy() if features else default_expected_features
  a		:      Optional[Any]       =  (
      Features({feature: Value(_A       ) for feature, dtype in features.items()}       ) if features is not None else None
  )
  a		:      Optional[Any]       =  JsonDatasetReader(_A				,    features=_A				,    cache_dir=_A       ).read()
  assert isinstance(_A				,    _A       )
  assert dataset.num_rows == 2
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_3", "col_1", "col_2"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype
def    lowerCamelCase__ (		_A				,    _A       ):
  # jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
  a		:      Optional[Any]       =  {'col_2': 'int64', 'col_3': 'float64', 'col_1': 'string'}
  a		:      str       =  features.copy()
  a		:      int       =  (
      Features({feature: Value(_A       ) for feature, dtype in features.items()}       ) if features is not None else None
  )
  a		:      Union[str, Any]       =  tmp_path / 'cache'
  a		:      Dict       =  JsonDatasetReader(_A				,    features=_A				,    cache_dir=_A       ).read()
  assert isinstance(_A				,    _A       )
  assert dataset.num_rows == 2
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_2", "col_3", "col_1"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('split'				,    [None, NamedSplit('train'       ), 'train', 'test']       )
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  a		:      int       =  tmp_path / 'cache'
  a		:      List[str]       =  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
  a		:      str       =  JsonDatasetReader(_A				,    cache_dir=_A				,    split=_A       ).read()
  _check_json_dataset(_A				,    _A       )
  assert dataset.split == split if split else "train"
@pytest.mark.parametrize('path_type'				,    [str, list]       )
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  if issubclass(_A				,    _A       ):
    a		:      int       =  jsonl_path
  elif issubclass(_A				,    _A       ):
    a		:      str       =  [jsonl_path]
  a		:      Optional[Any]       =  tmp_path / 'cache'
  a		:      Dict       =  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
  a		:      Any       =  JsonDatasetReader(_A				,    cache_dir=_A       ).read()
  _check_json_dataset(_A				,    _A       )
def    lowerCamelCase__ (		_A				,    _A				,    _A=("train",)       ):
  assert isinstance(_A				,    _A       )
  for split in splits:
    a		:      Union[str, Any]       =  dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('keep_in_memory'				,    [False, True]       )
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  a		:      Dict       =  tmp_path / 'cache'
  a		:      Optional[Any]       =  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    a		:      Optional[Any]       =  JsonDatasetReader({'train': jsonl_path}				,    cache_dir=_A				,    keep_in_memory=_A       ).read()
  _check_json_datasetdict(_A				,    _A       )
@pytest.mark.parametrize(
    'features'				,    [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]				,    )
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  a		:      Tuple       =  tmp_path / 'cache'
  a		:      Optional[Any]       =  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
  a		:      str       =  features.copy() if features else default_expected_features
  a		:      Tuple       =  (
      Features({feature: Value(_A       ) for feature, dtype in features.items()}       ) if features is not None else None
  )
  a		:      Union[str, Any]       =  JsonDatasetReader({'train': jsonl_path}				,    features=_A				,    cache_dir=_A       ).read()
  _check_json_datasetdict(_A				,    _A       )
@pytest.mark.parametrize('split'				,    [None, NamedSplit('train'       ), 'train', 'test']       )
def    lowerCamelCase__ (		_A				,    _A				,    _A       ):
  if split:
    a		:      Dict       =  {split: jsonl_path}
  else:
    a		:      Union[str, Any]       =  'train'
    a		:      int       =  {'train': jsonl_path, 'test': jsonl_path}
  a		:      Any       =  tmp_path / 'cache'
  a		:      List[str]       =  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
  a		:      List[str]       =  JsonDatasetReader(_A				,    cache_dir=_A       ).read()
  _check_json_datasetdict(_A				,    _A				,    splits=list(path.keys()       )       )
  assert all(dataset[split].split == split for split in path.keys()       )
def    lowerCamelCase__ (		_A       ):
  return json.load(_A       )
def    lowerCamelCase__ (		_A       ):
  return [json.loads(_A       ) for line in buffer]

class 							a__:







    @pytest.mark.parametrize('lines, load_json_function'		,    [(True, load_json_lines), (False, load_json)]			)
    def     lowercase_ (   self				:						int		,    __snake_case				:						Any		,    __snake_case				:						int		,    __snake_case				:						Dict			):
      with io.BytesIO() as buffer:
        JsonDatasetWriter(__snake_case		,    __snake_case		,    lines=__snake_case			).write()
        buffer.seek(0			)
        a		:      List[Any]       =  load_json_function(__snake_case			)
      assert isinstance(__snake_case		,    __snake_case			)
      assert isinstance(exported_content[0]		,    __snake_case			)
      assert len(__snake_case			) == 10







    @pytest.mark.parametrize(
        'orient, container, keys, len_at'		,    [
            ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
            ('split', dict, {'columns', 'data'}, 'data'),
            ('index', dict, set('0123456789'			), None),
            ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
            ('values', list, None, None),
            ('table', dict, {'schema', 'data'}, 'data'),
        ]		,    )
    def     lowercase_ (   self				:						Union[str, Any]		,    __snake_case				:						Union[str, Any]		,    __snake_case				:						Optional[Any]		,    __snake_case				:						Optional[Any]		,    __snake_case				:						Dict		,    __snake_case				:						str			):
      with io.BytesIO() as buffer:
        JsonDatasetWriter(__snake_case		,    __snake_case		,    lines=__snake_case		,    orient=__snake_case			).write()
        buffer.seek(0			)
        a		:      str       =  load_json(__snake_case			)
      assert isinstance(__snake_case		,    __snake_case			)
      if keys:
        if container is dict:
          assert exported_content.keys() == keys
        else:
          assert exported_content[0].keys() == keys
      else:
        assert not hasattr(__snake_case		,    'keys'			) and not hasattr(exported_content[0]		,    'keys'			)
      if len_at:
        assert len(exported_content[len_at]			) == 10
      else:
        assert len(__snake_case			) == 10







    @pytest.mark.parametrize('lines, load_json_function'		,    [(True, load_json_lines), (False, load_json)]			)
    def     lowercase_ (   self				:						Optional[int]		,    __snake_case				:						List[Any]		,    __snake_case				:						List[str]		,    __snake_case				:						List[str]			):
      with io.BytesIO() as buffer:
        JsonDatasetWriter(__snake_case		,    __snake_case		,    lines=__snake_case		,    num_proc=2			).write()
        buffer.seek(0			)
        a		:      Union[str, Any]       =  load_json_function(__snake_case			)
      assert isinstance(__snake_case		,    __snake_case			)
      assert isinstance(exported_content[0]		,    __snake_case			)
      assert len(__snake_case			) == 10







    @pytest.mark.parametrize(
        'orient, container, keys, len_at'		,    [
            ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
            ('split', dict, {'columns', 'data'}, 'data'),
            ('index', dict, set('0123456789'			), None),
            ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
            ('values', list, None, None),
            ('table', dict, {'schema', 'data'}, 'data'),
        ]		,    )
    def     lowercase_ (   self				:						Optional[int]		,    __snake_case				:						Optional[Any]		,    __snake_case				:						Optional[Any]		,    __snake_case				:						Optional[Any]		,    __snake_case				:						int		,    __snake_case				:						str			):
      with io.BytesIO() as buffer:
        JsonDatasetWriter(__snake_case		,    __snake_case		,    lines=__snake_case		,    orient=__snake_case		,    num_proc=2			).write()
        buffer.seek(0			)
        a		:      Tuple       =  load_json(__snake_case			)
      assert isinstance(__snake_case		,    __snake_case			)
      if keys:
        if container is dict:
          assert exported_content.keys() == keys
        else:
          assert exported_content[0].keys() == keys
      else:
        assert not hasattr(__snake_case		,    'keys'			) and not hasattr(exported_content[0]		,    'keys'			)
      if len_at:
        assert len(exported_content[len_at]			) == 10
      else:
        assert len(__snake_case			) == 10







    def     lowercase_ (   self				:						Optional[int]		,    __snake_case				:						str			):
      with pytest.raises(__snake_case			):
        with io.BytesIO() as buffer:
          JsonDatasetWriter(__snake_case		,    __snake_case		,    num_proc=0			)







    @pytest.mark.parametrize('compression, extension'		,    [('gzip', 'gz'), ('bz2', 'bz2'), ('xz', 'xz')]			)
    def     lowercase_ (   self				:						Optional[int]		,    __snake_case				:						List[str]		,    __snake_case				:						List[str]		,    __snake_case				:						Any		,    __snake_case				:						Optional[Any]		,    __snake_case				:						List[str]			):
      a		:      List[str]       =  tmp_path_factory.mktemp('data'			) / F"""test.json.{extension}"""
      a		:      int       =  str(shared_datadir / F"""test_file.json.{extension}"""			)
      JsonDatasetWriter(__snake_case		,    __snake_case		,    compression=__snake_case			).write()

      with fsspec.open(__snake_case		,    'rb'		,    compression='infer'			) as f:
        a		:      Optional[int]       =  f.read()
      with fsspec.open(__snake_case		,    'rb'		,    compression='infer'			) as f:
        a		:      List[Any]       =  f.read()
      assert exported_content == original_content




'''simple docstring'''






import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionAttendAndExcitePipeline,
    UNetaDConditionModel,
)
from diffusers.utils import load_numpy, skip_mps, slow
from diffusers.utils.testing_utils import require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


lowerCAmelCase:      Any		 =		False

@skip_mps
class 							a__(      lowerCamelCase__	,				lowerCamelCase__	,				lowerCamelCase__	,				unittest.TestCase			):
    lowercase__				=					StableDiffusionAttendAndExcitePipeline
    lowercase__				=					False
    lowercase__				=					TEXT_TO_IMAGE_PARAMS
    lowercase__				=					TEXT_TO_IMAGE_BATCH_PARAMS.union({"""token_indices"""}			)
    lowercase__				=					TEXT_TO_IMAGE_IMAGE_PARAMS
    lowercase__				=					TEXT_TO_IMAGE_IMAGE_PARAMS







    @classmethod
    def     lowercase_ (   cls				:						Union[str, Any]			):
      super().setUpClass()
      torch.use_deterministic_algorithms(__snake_case			)







    @classmethod
    def     lowercase_ (   cls				:						Optional[Any]			):
      super().tearDownClass()
      torch.use_deterministic_algorithms(__snake_case			)







    def     lowercase_ (   self				:						List[Any]			):
      torch.manual_seed(0			)
      a		:      List[Any]       =  UNetaDConditionModel(
          block_out_channels=(32, 64)		,    layers_per_block=1		,    sample_size=32		,    in_channels=4		,    out_channels=4		,    down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')		,    up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')		,    cross_attention_dim=32		,    attention_head_dim=(2, 4)		,    use_linear_projection=__snake_case		,    )
      a		:      Optional[int]       =  DDIMScheduler(
          beta_start=0.00085		,    beta_end=0.012		,    beta_schedule='scaled_linear'		,    clip_sample=__snake_case		,    set_alpha_to_one=__snake_case		,    )
      torch.manual_seed(0			)
      a		:      str       =  AutoencoderKL(
          block_out_channels=[32, 64]		,    in_channels=3		,    out_channels=3		,    down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']		,    up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']		,    latent_channels=4		,    sample_size=1_28		,    )
      torch.manual_seed(0			)
      a		:      str       =  CLIPTextConfig(
          bos_token_id=0		,    eos_token_id=2		,    hidden_size=32		,    intermediate_size=37		,    layer_norm_eps=1e-0_5		,    num_attention_heads=4		,    num_hidden_layers=5		,    pad_token_id=1		,    vocab_size=10_00		,    hidden_act='gelu'		,    projection_dim=5_12		,    )
      a		:      Any       =  CLIPTextModel(__snake_case			)
      a		:      Tuple       =  CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'			)

      a		:      List[str]       =  {
          'unet': unet,
          'scheduler': scheduler,
          'vae': vae,
          'text_encoder': text_encoder,
          'tokenizer': tokenizer,
          'safety_checker': None,
          'feature_extractor': None,
      }

      return components







    def     lowercase_ (   self				:						str		,    __snake_case				:						Tuple		,    __snake_case				:						Optional[int]=0			):
      if str(__snake_case			).startswith('mps'			):
        a		:      Any       =  torch.manual_seed(__snake_case			)
      else:
        a		:      Any       =  torch.Generator(device=__snake_case			).manual_seed(__snake_case			)
      a		:      int       =  {
          'prompt': 'a cat and a frog',
          'token_indices': [2, 5],
          'generator': generator,
          'num_inference_steps': 1,
          'guidance_scale': 6.0,
          'output_type': 'numpy',
          'max_iter_to_alter': 2,
          'thresholds': {0: 0.7},
      }
      return inputs







    def     lowercase_ (   self				:						List[Any]			):
      a		:      Union[str, Any]       =  'cpu'

      a		:      Any       =  self.get_dummy_components()
      a		:      List[str]       =  self.pipeline_class(**__snake_case			)
      pipe.to(__snake_case			)
      pipe.set_progress_bar_config(disable=__snake_case			)

      a		:      Any       =  self.get_dummy_inputs(__snake_case			)
      a		:      int       =  pipe(**__snake_case			).images
      a		:      int       =  image[0, -3:, -3:, -1]

      self.assertEqual(image.shape		,    (1, 64, 64, 3)			)
      a		:      str       =  np.array(
          [0.63905364, 0.62897307, 0.48599017, 0.5133624, 0.5550048, 0.45769516, 0.50326973, 0.5023139, 0.45384496]			)
      a		:      int       =  np.abs(image_slice.flatten() - expected_slice			).max()
      self.assertLessEqual(__snake_case		,    1e-3			)







    def     lowercase_ (   self				:						Dict			):
      super().test_cpu_offload_forward_pass(expected_max_diff=5e-4			)







    def     lowercase_ (   self				:						Tuple			):
      # NOTE: Larger batch sizes cause this test to timeout, only test on smaller batches
      self._test_inference_batch_consistent(batch_sizes=[1, 2]			)







    def     lowercase_ (   self				:						Union[str, Any]			):
      self._test_inference_batch_single_identical(batch_size=2		,    expected_max_diff=7e-4			)







    def     lowercase_ (   self				:						Tuple			):
      super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3			)







    def     lowercase_ (   self				:						str			):
      super().test_pt_np_pil_outputs_equivalent(expected_max_diff=5e-4			)







    def     lowercase_ (   self				:						Any			):
      super().test_save_load_local(expected_max_difference=5e-4			)







    def     lowercase_ (   self				:						List[Any]			):
      super().test_save_load_optional_components(expected_max_difference=4e-4			)

@require_torch_gpu
@slow
class 							a__(      unittest.TestCase			):







    @classmethod
    def     lowercase_ (   cls				:						Union[str, Any]			):
      super().setUpClass()
      torch.use_deterministic_algorithms(__snake_case			)







    @classmethod
    def     lowercase_ (   cls				:						Union[str, Any]			):
      super().tearDownClass()
      torch.use_deterministic_algorithms(__snake_case			)







    def     lowercase_ (   self				:						Union[str, Any]			):
      super().tearDown()
      gc.collect()
      torch.cuda.empty_cache()







    def     lowercase_ (   self				:						Optional[Any]			):
      a		:      List[Any]       =  torch.manual_seed(51			)

      a		:      Dict       =  StableDiffusionAttendAndExcitePipeline.from_pretrained(
          'CompVis/stable-diffusion-v1-4'		,    safety_checker=__snake_case		,    torch_dtype=torch.floataa			)
      pipe.to('cuda'			)

      a		:      Optional[Any]       =  'a painting of an elephant with glasses'
      a		:      Any       =  [5, 7]

      a		:      Tuple       =  pipe(
          prompt=__snake_case		,    token_indices=__snake_case		,    guidance_scale=7.5		,    generator=__snake_case		,    num_inference_steps=5		,    max_iter_to_alter=5		,    output_type='numpy'		,    ).images[0]

      a		:      str       =  load_numpy(
          'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/attend-and-excite/elephant_glasses.npy'			)
      assert np.abs((expected_image - image).max()			) < 5e-1