


'''simple docstring'''





import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


UpperCAmelCase__  			=				datasets.utils.logging.get_logger(__name__)

UpperCAmelCase__  			=				['''names''', '''prefix''']
UpperCAmelCase__  			=				['''warn_bad_lines''', '''error_bad_lines''', '''mangle_dupe_cols''']
UpperCAmelCase__  			=				['''encoding_errors''', '''on_bad_lines''']
UpperCAmelCase__  			=				['''date_format''']
@dataclass
class 						a__					(					datasets.BuilderConfig   ):





			'''simple docstring'''

			A						:				str					      =			","
			A						:				Optional[str]					      =			None
			A						:				Optional[Union[int, List[int], str]]					      =			"infer"
			A						:				Optional[List[str]]					      =			None
			A						:				Optional[List[str]]					      =			None
			A						:				Optional[Union[int, str, List[int], List[str]]]					      =			None
			A						:				Optional[Union[List[int], List[str]]]					      =			None
			A						:				Optional[str]					      =			None
			A						:				bool					      =			True
			A						:				Optional[Literal["c", "python", "pyarrow"]]					      =			None
			A						:				Dict[Union[int, str], Callable[[Any], Any]]					      =			None
			A						:				Optional[list]					      =			None
			A						:				Optional[list]					      =			None
			A						:				bool					      =			False
			A						:				Optional[Union[int, List[int]]]					      =			None
			A						:				Optional[int]					      =			None
			A						:				Optional[Union[str, List[str]]]					      =			None
			A						:				bool					      =			True
			A						:				bool					      =			True
			A						:				bool					      =			False
			A						:				bool					      =			True
			A						:				Optional[str]					      =			None
			A						:				str					      =			"."
			A						:				Optional[str]					      =			None
			A						:				str					      =			'"'
			A						:				int					      =			0
			A						:				Optional[str]					      =			None
			A						:				Optional[str]					      =			None
			A						:				Optional[str]					      =			None
			A						:				Optional[str]					      =			None
			A						:				bool					      =			True
			A						:				bool					      =			True
			A						:				int					      =			0
			A						:				bool					      =			True
			A						:				bool					      =			False
			A						:				Optional[str]					      =			None
			A						:				int					      =			1_0000
			A						:				Optional[datasets.Features]					      =			None
			A						:				Optional[str]					      =			"strict"
			A						:				Literal["error", "warn", "skip"]					      =			"error"
			A						:				Optional[str]					      =			None
			def 					lowerCAmelCase     (  self							:			str	)      ->		Any:
						if self.delimiter is not None:
									__A=	self.delimiter
						if self.column_names is not None:
									__A=	self.column_names

			@property
			def 					lowerCAmelCase     (  self							:			Any	)      ->		Any:
						__A=	{
						    'sep': self.sep,
						    'header': self.header,
						    'names': self.names,
						    'index_col': self.index_col,
						    'usecols': self.usecols,
						    'prefix': self.prefix,
						    'mangle_dupe_cols': self.mangle_dupe_cols,
						    'engine': self.engine,
						    'converters': self.converters,
						    'true_values': self.true_values,
						    'false_values': self.false_values,
						    'skipinitialspace': self.skipinitialspace,
						    'skiprows': self.skiprows,
						    'nrows': self.nrows,
						    'na_values': self.na_values,
						    'keep_default_na': self.keep_default_na,
						    'na_filter': self.na_filter,
						    'verbose': self.verbose,
						    'skip_blank_lines': self.skip_blank_lines,
						    'thousands': self.thousands,
						    'decimal': self.decimal,
						    'lineterminator': self.lineterminator,
						    'quotechar': self.quotechar,
						    'quoting': self.quoting,
						    'escapechar': self.escapechar,
						    'comment': self.comment,
						    'encoding': self.encoding,
						    'dialect': self.dialect,
						    'error_bad_lines': self.error_bad_lines,
						    'warn_bad_lines': self.warn_bad_lines,
						    'skipfooter': self.skipfooter,
						    'doublequote': self.doublequote,
						    'memory_map': self.memory_map,
						    'float_precision': self.float_precision,
						    'chunksize': self.chunksize,
						    'encoding_errors': self.encoding_errors,
						    'on_bad_lines': self.on_bad_lines,
						    'date_format': self.date_format,
						}

						# some kwargs must not be passed if they don't have a default value
						# some others are deprecated and we can also not pass them if they are the default value
						for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
									if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()			,    lowerCAmelCase_	):
												del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
						if not (datasets.config.PANDAS_VERSION.major >= 2):
									for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
												del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
						if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
									for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
												del pd_read_csv_kwargs[pd_read_csv_parameter]

						return pd_read_csv_kwargs


class 						a__					(					datasets.ArrowBasedBuilder   ):





			'''simple docstring'''

			A						:				List[Any]					      =			CsvConfig
			def 					lowerCAmelCase     (  self							:			List[Any]	)      ->		Union[str, Any]:
						return datasets.DatasetInfo(features=self.config.features	)
			def 					lowerCAmelCase     (  self							:			Tuple			,    lowerCAmelCase_							:			List[str]	)      ->		str:
						if not self.config.data_files:
									raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""	)
						__A=	dl_manager.download_and_extract(self.config.data_files	)
						if isinstance(lowerCAmelCase_			,    (str, list, tuple)	):
									__A=	data_files
									if isinstance(lowerCAmelCase_			,    lowerCAmelCase_	):
												__A=	[files]
									__A=	[dl_manager.iter_files(lowerCAmelCase_	) for file in files]
									return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,    gen_kwargs={'files': files}	)]
						__A=	[]
						for split_name, files in data_files.items():
									if isinstance(lowerCAmelCase_			,    lowerCAmelCase_	):
												__A=	[files]
									__A=	[dl_manager.iter_files(lowerCAmelCase_	) for file in files]
									splits.append(datasets.SplitGenerator(name=lowerCAmelCase_			,    gen_kwargs={'files': files}	)	)
						return splits
			def 					lowerCAmelCase     (  self							:			Dict			,    lowerCAmelCase_							:			pa.Table	)      ->		pa.Table:
						if self.config.features is not None:
									__A=	self.config.features.arrow_schema
									if all(not require_storage_cast(lowerCAmelCase_	) for feature in self.config.features.values()	):
												# cheaper cast
												__A=	pa.Table.from_arrays([pa_table[field.name] for field in schema]			,    schema=lowerCAmelCase_	)
									else:
												# more expensive cast; allows str <-> int/float or str to Audio for example
												__A=	table_cast(lowerCAmelCase_			,    lowerCAmelCase_	)
						return pa_table

			def 					lowerCAmelCase     (  self							:			Any			,    lowerCAmelCase_							:			str	)      ->		Dict:
						__A=	self.config.features.arrow_schema if self.config.features else None
						# dtype allows reading an int column as str
						__A=	(
						    {
						        name: dtype.to_pandas_dtype() if not require_storage_cast(lowerCAmelCase_	) else object
						        for name, dtype, feature in zip(schema.names			,    schema.types			,    self.config.features.values()	)
						    }
						    if schema is not None
						    else None
						)
						for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase_	)	):
									__A=	pd.read_csv(lowerCAmelCase_			,    iterator=lowerCAmelCase_			,    dtype=lowerCAmelCase_			,    **self.config.pd_read_csv_kwargs	)
									try:
												for batch_idx, df in enumerate(lowerCAmelCase_	):
															__A=	pa.Table.from_pandas(lowerCAmelCase_	)
															# Uncomment for debugging (will print the Arrow table size and elements)
															# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
															# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
															yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase_	)
									except ValueError as e:
												logger.error(F"""Failed to read file '{file}' with error {type(lowerCAmelCase_	)}: {e}"""	)
												raise







'''simple docstring'''





from typing import Callable, List, Optional, Tuple, Union

import torch
from transformers import CLIPTextModel, CLIPTokenizer

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin, TransformeraDModel, VQModel
from ...schedulers import VQDiffusionScheduler
from ...utils import logging
from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput


UpperCAmelCase__  			=				logging.get_logger(__name__)  # pylint: disable=invalid-name
class 						a__					(					a_      ,  a_   ):





			'''simple docstring'''

			@register_to_config
			def __init__(  self							:			Any			,    lowerCAmelCase_							:			bool			,    lowerCAmelCase_							:			Optional[int] = None			,    lowerCAmelCase_							:			Optional[int] = None	)      ->		str:
						super().__init__()

						__A=	learnable

						if self.learnable:
									assert hidden_size is not None, "learnable=True requires `hidden_size` to be set"
									assert length is not None, "learnable=True requires `length` to be set"

									__A=	torch.zeros(lowerCAmelCase_			,    lowerCAmelCase_	)
						else:
									__A=	None

						__A=	torch.nn.Parameter(lowerCAmelCase_	)


class 						a__					(					a_   ):





			'''simple docstring'''

			A						:				VQModel
			A						:				CLIPTextModel
			A						:				CLIPTokenizer
			A						:				TransformeraDModel
			A						:				LearnedClassifierFreeSamplingEmbeddings
			A						:				VQDiffusionScheduler
			def __init__(  self							:			Optional[int]			,    lowerCAmelCase_							:			VQModel			,    lowerCAmelCase_							:			CLIPTextModel			,    lowerCAmelCase_							:			CLIPTokenizer			,    lowerCAmelCase_							:			TransformeraDModel			,    lowerCAmelCase_							:			VQDiffusionScheduler			,    lowerCAmelCase_							:			LearnedClassifierFreeSamplingEmbeddings			,    )      ->		Tuple:
						super().__init__()

						self.register_modules(
						    vqvae=lowerCAmelCase_			,    transformer=lowerCAmelCase_			,    text_encoder=lowerCAmelCase_			,    tokenizer=lowerCAmelCase_			,    scheduler=lowerCAmelCase_			,    learned_classifier_free_sampling_embeddings=lowerCAmelCase_			,    )
			def 					lowerCAmelCase     (  self							:			int			,    lowerCAmelCase_							:			List[str]			,    lowerCAmelCase_							:			Optional[Any]			,    lowerCAmelCase_							:			List[str]	)      ->		Union[str, Any]:
						__A=	len(lowerCAmelCase_	) if isinstance(lowerCAmelCase_			,    lowerCAmelCase_	) else 1

						# get prompt text embeddings
						__A=	self.tokenizer(
						    lowerCAmelCase_			,    padding='max_length'			,    max_length=self.tokenizer.model_max_length			,    return_tensors='pt'			,    )
						__A=	text_inputs.input_ids

						if text_input_ids.shape[-1] > self.tokenizer.model_max_length:
									__A=	self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]	)
									logger.warning(
									    'The following part of your input was truncated because CLIP can only handle sequences up to'
									    F""" {self.tokenizer.model_max_length} tokens: {removed_text}"""	)
									__A=	text_input_ids[:, : self.tokenizer.model_max_length]
						__A=	self.text_encoder(text_input_ids.to(self.device	)	)[0]

						# NOTE: This additional step of normalizing the text embeddings is from VQ-Diffusion.
						# While CLIP does normalize the pooled output of the text transformer when combining
						# the image and text embeddings, CLIP does not directly normalize the last hidden state.
						#
						# CLIP normalizing the pooled output.
						# https://github.com/huggingface/transformers/blob/d92e22d1f28324f513f3080e5c47c071a3916721/src/transformers/models/clip/modeling_clip.py#L1052-L1053
						__A=	prompt_embeds / prompt_embeds.norm(dim=-1			,    keepdim=lowerCAmelCase_	)

						# duplicate text embeddings for each generation per prompt
						__A=	prompt_embeds.repeat_interleave(lowerCAmelCase_			,    dim=0	)

						if do_classifier_free_guidance:
									if self.learned_classifier_free_sampling_embeddings.learnable:
												__A=	self.learned_classifier_free_sampling_embeddings.embeddings
												__A=	negative_prompt_embeds.unsqueeze(0	).repeat(lowerCAmelCase_			,    1			,    1	)
									else:
												__A=	[''] * batch_size

												__A=	text_input_ids.shape[-1]
												__A=	self.tokenizer(
												    lowerCAmelCase_			,    padding='max_length'			,    max_length=lowerCAmelCase_			,    truncation=lowerCAmelCase_			,    return_tensors='pt'			,    )
												__A=	self.text_encoder(uncond_input.input_ids.to(self.device	)	)[0]
												# See comment for normalizing text embeddings
												__A=	negative_prompt_embeds / negative_prompt_embeds.norm(dim=-1			,    keepdim=lowerCAmelCase_	)

									# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
									__A=	negative_prompt_embeds.shape[1]
									__A=	negative_prompt_embeds.repeat(1			,    lowerCAmelCase_			,    1	)
									__A=	negative_prompt_embeds.view(batch_size * num_images_per_prompt			,    lowerCAmelCase_			,    -1	)

									# For classifier free guidance, we need to do two forward passes.
									# Here we concatenate the unconditional and text embeddings into a single batch
									# to avoid doing two forward passes
									__A=	torch.cat([negative_prompt_embeds, prompt_embeds]	)

						return prompt_embeds
			@torch.no_grad()
			def __call__(  self							:			int			,    lowerCAmelCase_							:			Union[str, List[str]]			,    lowerCAmelCase_							:			int = 100			,    lowerCAmelCase_							:			float = 5.0			,    lowerCAmelCase_							:			float = 1.0			,    lowerCAmelCase_							:			int = 1			,    lowerCAmelCase_							:			Optional[Union[torch.Generator, List[torch.Generator]]] = None			,    lowerCAmelCase_							:			Optional[torch.FloatTensor] = None			,    lowerCAmelCase_							:			Optional[str] = "pil"			,    lowerCAmelCase_							:			bool = True			,    lowerCAmelCase_							:			Optional[Callable[[int, int, torch.FloatTensor], None]] = None			,    lowerCAmelCase_							:			int = 1			,    )      ->		Union[ImagePipelineOutput, Tuple]:
						if isinstance(lowerCAmelCase_			,    lowerCAmelCase_	):
									__A=	1
						elif isinstance(lowerCAmelCase_			,    lowerCAmelCase_	):
									__A=	len(lowerCAmelCase_	)
						else:
									raise ValueError(F"""`prompt` has to be of type `str` or `list` but is {type(lowerCAmelCase_	)}"""	)

						__A=	batch_size * num_images_per_prompt

						__A=	guidance_scale > 1.0

						__A=	self._encode_prompt(lowerCAmelCase_			,    lowerCAmelCase_			,    lowerCAmelCase_	)

						if (callback_steps is None) or (
						    callback_steps is not None and (not isinstance(lowerCAmelCase_			,    lowerCAmelCase_	) or callback_steps <= 0)
						):
									raise ValueError(
									    F"""`callback_steps` has to be a positive integer but is {callback_steps} of type"""
									    F""" {type(lowerCAmelCase_	)}."""	)

						# get the initial completely masked latents unless the user supplied it

						__A=	(batch_size, self.transformer.num_latent_pixels)
						if latents is None:
									__A=	self.transformer.num_vector_embeds - 1
									__A=	torch.full(lowerCAmelCase_			,    lowerCAmelCase_	).to(self.device	)
						else:
									if latents.shape != latents_shape:
												raise ValueError(F"""Unexpected latents shape, got {latents.shape}, expected {latents_shape}"""	)
									if (latents < 0).any() or (latents >= self.transformer.num_vector_embeds).any():
												raise ValueError(
												    'Unexpected latents value(s). All latents be valid embedding indices i.e. in the range 0,'
												    F""" {self.transformer.num_vector_embeds - 1} (inclusive)."""	)
									__A=	latents.to(self.device	)

						# set timesteps
						self.scheduler.set_timesteps(lowerCAmelCase_			,    device=self.device	)

						__A=	self.scheduler.timesteps.to(self.device	)

						__A=	latents

						for i, t in enumerate(self.progress_bar(lowerCAmelCase_	)	):
									# expand the sample if we are doing classifier free guidance
									__A=	torch.cat([sample] * 2	) if do_classifier_free_guidance else sample

									# predict the un-noised image
									# model_output == `log_p_x_0`
									__A=	self.transformer(lowerCAmelCase_			,    encoder_hidden_states=lowerCAmelCase_			,    timestep=lowerCAmelCase_	).sample

									if do_classifier_free_guidance:
												__A,	__A=	model_output.chunk(2	)
												__A=	model_output_uncond + guidance_scale * (model_output_text - model_output_uncond)
												model_output -= torch.logsumexp(lowerCAmelCase_			,    dim=1			,    keepdim=lowerCAmelCase_	)

									__A=	self.truncate(lowerCAmelCase_			,    lowerCAmelCase_	)

									# remove `log(0)`'s (`-inf`s)
									__A=	model_output.clamp(-70	)

									# compute the previous noisy sample x_t -> x_t-1
									__A=	self.scheduler.step(lowerCAmelCase_			,    timestep=lowerCAmelCase_			,    sample=lowerCAmelCase_			,    generator=lowerCAmelCase_	).prev_sample

									# call the callback, if provided
									if callback is not None and i % callback_steps == 0:
												callback(lowerCAmelCase_			,    lowerCAmelCase_			,    lowerCAmelCase_	)

						__A=	self.vqvae.config.vq_embed_dim
						__A=	(batch_size, self.transformer.height, self.transformer.width, embedding_channels)
						__A=	self.vqvae.quantize.get_codebook_entry(lowerCAmelCase_			,    shape=lowerCAmelCase_	)
						__A=	self.vqvae.decode(lowerCAmelCase_			,    force_not_quantize=lowerCAmelCase_	).sample

						__A=	(image / 2 + 0.5).clamp(0			,    1	)
						__A=	image.cpu().permute(0			,    2			,    3			,    1	).numpy()

						if output_type == "pil":
									__A=	self.numpy_to_pil(lowerCAmelCase_	)

						if not return_dict:
									return (image,)

						return ImagePipelineOutput(images=lowerCAmelCase_	)

			def 					lowerCAmelCase     (  self							:			Any			,    lowerCAmelCase_							:			torch.FloatTensor			,    lowerCAmelCase_							:			float	)      ->		torch.FloatTensor:
						__A,	__A=	torch.sort(lowerCAmelCase_			,    1			,    descending=lowerCAmelCase_	)
						__A=	torch.exp(lowerCAmelCase_	)
						__A=	sorted_p_x_0.cumsum(dim=1	) < truncation_rate

						# Ensure that at least the largest probability is not zeroed out
						__A=	torch.full_like(keep_mask[:, 0:1, :]			,    lowerCAmelCase_	)
						__A=	torch.cat((all_true, keep_mask)			,    dim=1	)
						__A=	keep_mask[:, :-1, :]

						__A=	keep_mask.gather(1			,    indices.argsort(1	)	)

						__A=	log_p_x_0.clone()

						__A=	-torch.inf  # -inf = log(0)

						return rv



