






'''simple docstring'''





import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, XLMRobertaTokenizer

from diffusers import AltDiffusionPipeline, AutoencoderKL, DDIMScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.alt_diffusion.modeling_roberta_series import (
    RobertaSeriesConfig,
    RobertaSeriesModelWithTransformation,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()





class 		SCREAMING_SNAKE_CASE__ (  snake_case_		,		snake_case_		,		snake_case_		,		unittest.TestCase     ):
		_UpperCAmelCase								=AltDiffusionPipeline
		_UpperCAmelCase								=TEXT_TO_IMAGE_PARAMS
		_UpperCAmelCase								=TEXT_TO_IMAGE_BATCH_PARAMS
		_UpperCAmelCase								=TEXT_TO_IMAGE_IMAGE_PARAMS
		_UpperCAmelCase								=TEXT_TO_IMAGE_IMAGE_PARAMS



		def _lowerCAmelCase  (     self:     Any)	->int:






								'''simple docstring'''


								torch.manual_seed(0)
								a_					  = UNetaDConditionModel(
								    block_out_channels=(32, 64)					, layers_per_block=2					, sample_size=32					, in_channels=4					, out_channels=4					, down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")					, up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")					, cross_attention_dim=32					, )
								a_					  = DDIMScheduler(
								    beta_start=0.0_0085					, beta_end=0.012					, beta_schedule="scaled_linear"					, clip_sample=a					, set_alpha_to_one=a					, )
								torch.manual_seed(0)
								a_					  = AutoencoderKL(
								    block_out_channels=[32, 64]					, in_channels=3					, out_channels=3					, down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]					, up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]					, latent_channels=4					, )

								# TODO: address the non-deterministic text encoder (fails for save-load tests)
								# torch.manual_seed(0)
								# text_encoder_config = RobertaSeriesConfig(
								#     hidden_size=32,
								#     project_dim=32,
								#     intermediate_size=37,
								#     layer_norm_eps=1e-05,
								#     num_attention_heads=4,
								#     num_hidden_layers=5,
								#     vocab_size=5002,
								# )
								# text_encoder = RobertaSeriesModelWithTransformation(text_encoder_config)

								torch.manual_seed(0)
								a_					  = CLIPTextConfig(
								    bos_token_id=0					, eos_token_id=2					, hidden_size=32					, projection_dim=32					, intermediate_size=37					, layer_norm_eps=1e-05					, num_attention_heads=4					, num_hidden_layers=5					, pad_token_id=1					, vocab_size=50_02					, )
								a_					  = CLIPTextModel(a)

								a_					  = XLMRobertaTokenizer.from_pretrained("hf-internal-testing/tiny-xlm-roberta")
								a_					  = 77

								a_					  = {
								    "unet": unet,
								    "scheduler": scheduler,
								    "vae": vae,
								    "text_encoder": text_encoder,
								    "tokenizer": tokenizer,
								    "safety_checker": None,
								    "feature_extractor": None,
								}
								return components



		def _lowerCAmelCase  (     self:     List[Any]					, a:     Any					, a:     List[str]=0)	->Union[str, Any]:






								'''simple docstring'''


								if str(a).startswith("mps"):
														a_					  = torch.manual_seed(a)
								else:
														a_					  = torch.Generator(device=a).manual_seed(a)
								a_					  = {
								    "prompt": "A painting of a squirrel eating a burger",
								    "generator": generator,
								    "num_inference_steps": 2,
								    "guidance_scale": 6.0,
								    "output_type": "numpy",
								}
								return inputs



		def _lowerCAmelCase  (     self:     List[Any])	->List[str]:






								'''simple docstring'''


								super().test_attention_slicing_forward_pass(expected_max_diff=3e-3)



		def _lowerCAmelCase  (     self:     List[Any])	->Dict:






								'''simple docstring'''


								super().test_inference_batch_single_identical(expected_max_diff=3e-3)



		def _lowerCAmelCase  (     self:     Tuple)	->str:






								'''simple docstring'''


								a_					  = "cpu"  # ensure determinism for the device-dependent torch.Generator

								a_					  = self.get_dummy_components()
								torch.manual_seed(0)
								a_					  = RobertaSeriesConfig(
								    hidden_size=32					, project_dim=32					, intermediate_size=37					, layer_norm_eps=1e-05					, num_attention_heads=4					, num_hidden_layers=5					, vocab_size=50_02					, )
								# TODO: remove after fixing the non-deterministic text encoder
								a_					  = RobertaSeriesModelWithTransformation(a)
								a_					  = text_encoder

								a_					  = AltDiffusionPipeline(**a)
								a_					  = alt_pipe.to(a)
								alt_pipe.set_progress_bar_config(disable=a)

								a_					  = self.get_dummy_inputs(a)
								a_					  = "A photo of an astronaut"
								a_					  = alt_pipe(**a)
								a_					  = output.images
								a_					  = image[0, -3:, -3:, -1]

								assert image.shape == (1, 64, 64, 3)
								a_					  = np.array(
								    [0.574_8162, 0.6044_7145, 0.4882_1217, 0.5010_0636, 0.543_1185, 0.4576_3683, 0.4965_7696, 0.4813_2733, 0.4757_3093])

								assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2




		def _lowerCAmelCase  (     self:     List[Any])	->Optional[Any]:






								'''simple docstring'''


								a_					  = "cpu"  # ensure determinism for the device-dependent torch.Generator

								a_					  = self.get_dummy_components()
								a_					  = PNDMScheduler(skip_prk_steps=a)
								torch.manual_seed(0)
								a_					  = RobertaSeriesConfig(
								    hidden_size=32					, project_dim=32					, intermediate_size=37					, layer_norm_eps=1e-05					, num_attention_heads=4					, num_hidden_layers=5					, vocab_size=50_02					, )
								# TODO: remove after fixing the non-deterministic text encoder
								a_					  = RobertaSeriesModelWithTransformation(a)
								a_					  = text_encoder
								a_					  = AltDiffusionPipeline(**a)
								a_					  = alt_pipe.to(a)
								alt_pipe.set_progress_bar_config(disable=a)

								a_					  = self.get_dummy_inputs(a)
								a_					  = alt_pipe(**a)
								a_					  = output.images
								a_					  = image[0, -3:, -3:, -1]

								assert image.shape == (1, 64, 64, 3)
								a_					  = np.array(
								    [0.5160_5093, 0.570_7241, 0.4736_5507, 0.5057_8886, 0.563_3877, 0.464_2503, 0.518_2081, 0.4876_3484, 0.4908_4237])

								assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2





@slow
@require_torch_gpu
class 		SCREAMING_SNAKE_CASE__ (  unittest.TestCase     ):



		def _lowerCAmelCase  (     self:     Union[str, Any])	->int:






								'''simple docstring'''


								super().tearDown()
								gc.collect()
								torch.cuda.empty_cache()



		def _lowerCAmelCase  (     self:     str)	->str:






								'''simple docstring'''


								a_					  = AltDiffusionPipeline.from_pretrained("BAAI/AltDiffusion"					, safety_checker=a)
								a_					  = alt_pipe.to(a)
								alt_pipe.set_progress_bar_config(disable=a)

								a_					  = "A painting of a squirrel eating a burger"
								a_					  = torch.manual_seed(0)
								a_					  = alt_pipe([prompt]					, generator=a					, guidance_scale=6.0					, num_inference_steps=20					, output_type="np")

								a_					  = output.images

								a_					  = image[0, -3:, -3:, -1]

								assert image.shape == (1, 5_12, 5_12, 3)
								a_					  = np.array([0.1010, 0.0800, 0.0794, 0.0885, 0.0843, 0.0762, 0.0769, 0.0729, 0.0586])

								assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2




		def _lowerCAmelCase  (     self:     Dict)	->Union[str, Any]:






								'''simple docstring'''


								a_					  = DDIMScheduler.from_pretrained("BAAI/AltDiffusion"					, subfolder="scheduler")

								a_					  = AltDiffusionPipeline.from_pretrained("BAAI/AltDiffusion"					, scheduler=a					, safety_checker=a)
								a_					  = alt_pipe.to(a)
								alt_pipe.set_progress_bar_config(disable=a)

								a_					  = "A painting of a squirrel eating a burger"
								a_					  = torch.manual_seed(0)

								a_					  = alt_pipe([prompt]					, generator=a					, num_inference_steps=2					, output_type="numpy")
								a_					  = output.images

								a_					  = image[0, -3:, -3:, -1]

								assert image.shape == (1, 5_12, 5_12, 3)
								a_					  = np.array([0.4019, 0.4052, 0.3810, 0.4119, 0.3916, 0.3982, 0.4651, 0.4195, 0.5323])

								assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__A							=logging.getLogger(__name__)





def      lowerCamelCase_					(							):
				lowerCamelCase_         =     argparse.ArgumentParser(
				    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."       )
				parser.add_argument(
				    "--dataset_name"      ,					type=lowerCamelCase__      ,					default="wikitext"      ,					help="Name of the training. Explore datasets at: hf.co/datasets."      ,					)
				parser.add_argument(
				    "--dataset_config"      ,					type=lowerCamelCase__      ,					default="wikitext-103-raw-v1"      ,					help="Configuration name of the dataset."       )
				parser.add_argument(
				    "--tokenizer_name_or_path"      ,					type=lowerCamelCase__      ,					default="sayakpaul/unigram-tokenizer-wikitext"      ,					help="Tokenizer identifier. Can be a local filepath or a Hub identifier."      ,					)
				parser.add_argument(
				    "--shard_size"      ,					type=lowerCamelCase__      ,					default=1_0_0_0      ,					help="Number of entries to go in a single shard."      ,					)
				parser.add_argument("--split"      ,					type=lowerCamelCase__      ,					default="train"      ,					choices=["train", "test", "validation"]       )
				parser.add_argument(
				    "--limit"      ,					default=lowerCamelCase__      ,					type=lowerCamelCase__      ,					help="Limit the number of shards (used for debugging)."      ,					)
				parser.add_argument(
				    "--max_length"      ,					type=lowerCamelCase__      ,					default=5_1_2      ,					help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
				    " sequence length that is a multiple of 8."      ,					)
				parser.add_argument(
				    "--output_dir"      ,					default="tf-tpu"      ,					type=lowerCamelCase__      ,					help="Output directory where the TFRecord shards will be saved. If the"
				    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
				    " shards will be directly saved to a Google Cloud Storage bucket."      ,					)

				lowerCamelCase_         =     parser.parse_args()
				return args





def      lowerCamelCase_					(							lowerCamelCase__       ):
				def fn(lowerCamelCase__       ):
								return tokenizer(examples["text"]       )

				return fn





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     []
				for i in range(len(tokenized_data["input_ids"]       )       ):
								lowerCamelCase_         =     {
								    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]       )       ),
								    "attention_mask": tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]       )       ),
								}
								lowerCamelCase_         =     tf.train.Features(feature=lowerCamelCase__       )
								lowerCamelCase_         =     tf.train.Example(features=lowerCamelCase__       )
								lowerCamelCase_         =     example.SerializeToString()
								records.append(lowerCamelCase__       )
				return records





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     datasets.load_dataset(args.dataset_name      ,					args.dataset_config      ,					split=args.split       )

				if args.limit is not None:
								lowerCamelCase_         =     min(len(lowerCamelCase__       )      ,					args.limit       )
								lowerCamelCase_         =     dataset.select(range(lowerCamelCase__       )       )
								print(F'Limiting the dataset to {args.limit} entries.'       )

				lowerCamelCase_         =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir       ):
												os.makedirs(args.output_dir       )
								lowerCamelCase_         =     os.path.join(args.output_dir      ,					args.split       )
								if not os.path.exists(lowerCamelCase__       ):
												os.makedirs(lowerCamelCase__       )
				else:
								lowerCamelCase_         =     os.path.join(args.output_dir      ,					args.split       )

				# Tokenize the whole dataset at once.
				lowerCamelCase_         =     tokenize_function(lowerCamelCase__       )
				lowerCamelCase_         =     dataset.map(lowerCamelCase__      ,					batched=lowerCamelCase__      ,					num_proc=4      ,					remove_columns=["text"]       )

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(lowerCamelCase__       ):
								# Concatenate all texts.
								lowerCamelCase_         =     {k: sum(examples[k]      ,					[]       ) for k in examples.keys()}
								lowerCamelCase_         =     len(concatenated_examples[list(examples.keys()       )[0]]       )
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								lowerCamelCase_         =     (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								lowerCamelCase_         =     {
								    k: [t[i : i + args.max_length] for i in range(0      ,					lowerCamelCase__      ,					args.max_length       )]
								    for k, t in concatenated_examples.items()
								}
								return result

				lowerCamelCase_         =     dataset_tokenized.map(lowerCamelCase__      ,					batched=lowerCamelCase__      ,					batch_size=1_0_0_0      ,					num_proc=4       )

				lowerCamelCase_         =     0
				lowerCamelCase_         =     0
				for shard in range(0      ,					len(lowerCamelCase__       )      ,					args.shard_size       ):
								lowerCamelCase_         =     grouped_dataset[shard : shard + args.shard_size]
								lowerCamelCase_         =     len(dataset_snapshot["input_ids"]       )
								lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					F'dataset-{shard_count}-{records_containing}.tfrecord'       )
								lowerCamelCase_         =     get_serialized_examples(lowerCamelCase__       )

								with tf.io.TFRecordWriter(lowerCamelCase__       ) as out_file:
												for i in range(len(lowerCamelCase__       )       ):
																lowerCamelCase_         =     serialized_examples[i]
																out_file.write(lowerCamelCase__       )
												print("Wrote file {} containing {} records".format(lowerCamelCase__      ,					lowerCamelCase__       )       )

								shard_count += 1
								total_records += records_containing

				with open(F'split-{args.split}-records-count.txt'      ,					"w"       ) as f:
								print(F'Total {args.split} records: {total_records}'      ,					file=lowerCamelCase__       )


if __name__ == "__main__":
			__A							=parse_args()
			main(args)


