

'''simple docstring'''

import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE							:				Tuple     	=			logging.getLogger(__name__)




def 							_UpperCamelCase				(   )      -> str:
							SCREAMING_SNAKE_CASE_				=				argparse.ArgumentParser(
							    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'   )
							parser.add_argument(
							    '--dataset_name'		,type=lowerCAmelCase__		,default='wikitext'		,help='Name of the training. Explore datasets at: hf.co/datasets.'		,)
							parser.add_argument(
							    '--dataset_config'		,type=lowerCAmelCase__		,default='wikitext-103-raw-v1'		,help='Configuration name of the dataset.'   )
							parser.add_argument(
							    '--tokenizer_name_or_path'		,type=lowerCAmelCase__		,default='sayakpaul/unigram-tokenizer-wikitext'		,help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'		,)
							parser.add_argument(
							    '--shard_size'		,type=lowerCAmelCase__		,default=1000		,help='Number of entries to go in a single shard.'		,)
							parser.add_argument('--split'		,type=lowerCAmelCase__		,default='train'		,choices=['train', 'test', 'validation']   )
							parser.add_argument(
							    '--limit'		,default=lowerCAmelCase__		,type=lowerCAmelCase__		,help='Limit the number of shards (used for debugging).'		,)
							parser.add_argument(
							    '--max_length'		,type=lowerCAmelCase__		,default=512		,help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
							    ' sequence length that is a multiple of 8.'		,)
							parser.add_argument(
							    '--output_dir'		,default='tf-tpu'		,type=lowerCAmelCase__		,help='Output directory where the TFRecord shards will be saved. If the'
							    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
							    ' shards will be directly saved to a Google Cloud Storage bucket.'		,)

							SCREAMING_SNAKE_CASE_				=				parser.parse_args()
							return args




def 							_UpperCamelCase				(   lowerCAmelCase__:					Optional[Any]   )      -> Optional[Any]:
							def fn(lowerCAmelCase__:					Union[str, Any]   ):
														return tokenizer(examples['text']   )

							return fn




def 							_UpperCamelCase				(   lowerCAmelCase__:					Tuple   )      -> int:
							SCREAMING_SNAKE_CASE_				=				[]
							for i in range(len(tokenized_data['input_ids']   )   ):
														SCREAMING_SNAKE_CASE_				=				{
														    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]   )   ),
														    'attention_mask': tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]   )   ),
														}
														SCREAMING_SNAKE_CASE_				=				tf.train.Features(feature=lowerCAmelCase__   )
														SCREAMING_SNAKE_CASE_				=				tf.train.Example(features=lowerCAmelCase__   )
														SCREAMING_SNAKE_CASE_				=				example.SerializeToString()
														records.append(lowerCAmelCase__   )
							return records




def 							_UpperCamelCase				(   lowerCAmelCase__:					int   )      -> Optional[Any]:
							SCREAMING_SNAKE_CASE_				=				datasets.load_dataset(args.dataset_name		,args.dataset_config		,split=args.split   )

							if args.limit is not None:
														SCREAMING_SNAKE_CASE_				=				min(len(lowerCAmelCase__   )		,args.limit   )
														SCREAMING_SNAKE_CASE_				=				dataset.select(range(lowerCAmelCase__   )   )
														print(F"""Limiting the dataset to {args.limit} entries."""   )

							SCREAMING_SNAKE_CASE_				=				AutoTokenizer.from_pretrained(args.tokenizer_name_or_path   )

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir   ):
																					os.makedirs(args.output_dir   )
														SCREAMING_SNAKE_CASE_				=				os.path.join(args.output_dir		,args.split   )
														if not os.path.exists(lowerCAmelCase__   ):
																					os.makedirs(lowerCAmelCase__   )
							else:
														SCREAMING_SNAKE_CASE_				=				os.path.join(args.output_dir		,args.split   )

							# Tokenize the whole dataset at once.
							SCREAMING_SNAKE_CASE_				=				tokenize_function(lowerCAmelCase__   )
							SCREAMING_SNAKE_CASE_				=				dataset.map(lowerCAmelCase__		,batched=lowerCAmelCase__		,num_proc=4		,remove_columns=['text']   )

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(lowerCAmelCase__:					Optional[int]   ):
														# Concatenate all texts.
														SCREAMING_SNAKE_CASE_				=				{k: sum(examples[k]		,[]   ) for k in examples.keys()}
														SCREAMING_SNAKE_CASE_				=				len(concatenated_examples[list(examples.keys()   )[0]]   )
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														SCREAMING_SNAKE_CASE_				=				(total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														SCREAMING_SNAKE_CASE_				=				{
														    k: [t[i : i + args.max_length] for i in range(0		,lowerCAmelCase__		,args.max_length   )]
														    for k, t in concatenated_examples.items()
														}
														return result

							SCREAMING_SNAKE_CASE_				=				dataset_tokenized.map(lowerCAmelCase__		,batched=lowerCAmelCase__		,batch_size=1000		,num_proc=4   )

							SCREAMING_SNAKE_CASE_				=				0
							SCREAMING_SNAKE_CASE_				=				0
							for shard in range(0		,len(lowerCAmelCase__   )		,args.shard_size   ):
														SCREAMING_SNAKE_CASE_				=				grouped_dataset[shard : shard + args.shard_size]
														SCREAMING_SNAKE_CASE_				=				len(dataset_snapshot['input_ids']   )
														SCREAMING_SNAKE_CASE_				=				os.path.join(lowerCAmelCase__		,F"""dataset-{shard_count}-{records_containing}.tfrecord"""   )
														SCREAMING_SNAKE_CASE_				=				get_serialized_examples(lowerCAmelCase__   )

														with tf.io.TFRecordWriter(lowerCAmelCase__   ) as out_file:
																					for i in range(len(lowerCAmelCase__   )   ):
																												SCREAMING_SNAKE_CASE_				=				serialized_examples[i]
																												out_file.write(lowerCAmelCase__   )
																					print('Wrote file {} containing {} records'.format(lowerCAmelCase__		,lowerCAmelCase__   )   )

														shard_count += 1
														total_records += records_containing

							with open(F"""split-{args.split}-records-count.txt"""		,'w'   ) as f:
														print(F"""Total {args.split} records: {total_records}"""		,file=lowerCAmelCase__   )


if __name__ == "__main__":
							SCREAMING_SNAKE_CASE							:				Union[str, Any]     	=			parse_args()
							main(args)








'''simple docstring'''

import gc
import random
import unittest

import numpy as np
import torch
from transformers import XLMRobertaTokenizer

from diffusers import (
    AltDiffusionImgaImgPipeline,
    AutoencoderKL,
    PNDMScheduler,
    UNetaDConditionModel,
)
from diffusers.image_processor import VaeImageProcessor
from diffusers.pipelines.alt_diffusion.modeling_roberta_series import (
    RobertaSeriesConfig,
    RobertaSeriesModelWithTransformation,
)
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()






class    snake_case     (      unittest.TestCase		):






				"""simple docstring"""



				def   a__					(    self       )     ->							int:
											# clean up the VRAM after each test
											super().tearDown()
											gc.collect()
											torch.cuda.empty_cache()



				@property
				def   a__					(    self       )     ->							Optional[int]:
											SCREAMING_SNAKE_CASE_				=				1
											SCREAMING_SNAKE_CASE_				=				3
											SCREAMING_SNAKE_CASE_				=				(32, 32)

											SCREAMING_SNAKE_CASE_				=				floats_tensor((batch_size, num_channels) + sizes,     rng=random.Random(0       )       ).to(_lowercase       )
											return image



				@property
				def   a__					(    self       )     ->							Any:
											torch.manual_seed(0       )
											SCREAMING_SNAKE_CASE_				=				UNetaDConditionModel(
											    block_out_channels=(32, 64),     layers_per_block=2,     sample_size=32,     in_channels=4,     out_channels=4,     down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D'),     up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D'),     cross_attention_dim=32,     )
											return model



				@property
				def   a__					(    self       )     ->							Tuple:
											torch.manual_seed(0       )
											SCREAMING_SNAKE_CASE_				=				AutoencoderKL(
											    block_out_channels=[32, 64],     in_channels=3,     out_channels=3,     down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D'],     up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D'],     latent_channels=4,     )
											return model



				@property
				def   a__					(    self       )     ->							Tuple:
											torch.manual_seed(0       )
											SCREAMING_SNAKE_CASE_				=				RobertaSeriesConfig(
											    hidden_size=32,     project_dim=32,     intermediate_size=37,     layer_norm_eps=1E-05,     num_attention_heads=4,     num_hidden_layers=5,     pad_token_id=1,     vocab_size=5006,     )
											return RobertaSeriesModelWithTransformation(_lowercase       )



				@property
				def   a__					(    self       )     ->							List[str]:
											def extract(*_lowercase,     **_lowercase       ):






																		class    snake_case     :






																						"""simple docstring"""



																						def __init__(    self       )     ->							Dict:
																													SCREAMING_SNAKE_CASE_				=				torch.ones([0]       )





																						def   a__					(    self,     _lowercase       )     ->							List[str]:
																													self.pixel_values.to(_lowercase       )
																													return self


																		return Out()

											return extract



				def   a__					(    self       )     ->							Tuple:
											SCREAMING_SNAKE_CASE_				=				'cpu'  # ensure determinism for the device-dependent torch.Generator
											SCREAMING_SNAKE_CASE_				=				self.dummy_cond_unet
											SCREAMING_SNAKE_CASE_				=				PNDMScheduler(skip_prk_steps=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				self.dummy_vae
											SCREAMING_SNAKE_CASE_				=				self.dummy_text_encoder
											SCREAMING_SNAKE_CASE_				=				XLMRobertaTokenizer.from_pretrained('hf-internal-testing/tiny-xlm-roberta'       )
											SCREAMING_SNAKE_CASE_				=				77

											SCREAMING_SNAKE_CASE_				=				self.dummy_image.to(_lowercase       )
											SCREAMING_SNAKE_CASE_				=				init_image / 2 + 0.5

											# make sure here that pndm scheduler skips prk
											SCREAMING_SNAKE_CASE_				=				AltDiffusionImgaImgPipeline(
											    unet=_lowercase,     scheduler=_lowercase,     vae=_lowercase,     text_encoder=_lowercase,     tokenizer=_lowercase,     safety_checker=_lowercase,     feature_extractor=self.dummy_extractor,     )
											SCREAMING_SNAKE_CASE_				=				VaeImageProcessor(vae_scale_factor=alt_pipe.vae_scale_factor,     do_normalize=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				alt_pipe.to(_lowercase       )
											alt_pipe.set_progress_bar_config(disable=_lowercase       )

											SCREAMING_SNAKE_CASE_				=				'A painting of a squirrel eating a burger'
											SCREAMING_SNAKE_CASE_				=				torch.Generator(device=_lowercase       ).manual_seed(0       )
											SCREAMING_SNAKE_CASE_				=				alt_pipe(
											    [prompt],     generator=_lowercase,     guidance_scale=6.0,     num_inference_steps=2,     output_type='np',     image=_lowercase,     )

											SCREAMING_SNAKE_CASE_				=				output.images

											SCREAMING_SNAKE_CASE_				=				torch.Generator(device=_lowercase       ).manual_seed(0       )
											SCREAMING_SNAKE_CASE_				=				alt_pipe(
											    [prompt],     generator=_lowercase,     guidance_scale=6.0,     num_inference_steps=2,     output_type='np',     image=_lowercase,     return_dict=_lowercase,     )[0]

											SCREAMING_SNAKE_CASE_				=				image[0, -3:, -3:, -1]
											SCREAMING_SNAKE_CASE_				=				image_from_tuple[0, -3:, -3:, -1]

											assert image.shape == (1, 32, 32, 3)
											SCREAMING_SNAKE_CASE_				=				np.array([0.4_427, 0.3_731, 0.4_249, 0.4_941, 0.4_546, 0.4_148, 0.4_193, 0.4_666, 0.4_499]       )

											assert np.abs(image_slice.flatten() - expected_slice       ).max() < 5E-3
											assert np.abs(image_from_tuple_slice.flatten() - expected_slice       ).max() < 5E-3



				@unittest.skipIf(torch_device != 'cuda',     'This test requires a GPU'       )
				def   a__					(    self       )     ->							str:
											SCREAMING_SNAKE_CASE_				=				self.dummy_cond_unet
											SCREAMING_SNAKE_CASE_				=				PNDMScheduler(skip_prk_steps=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				self.dummy_vae
											SCREAMING_SNAKE_CASE_				=				self.dummy_text_encoder
											SCREAMING_SNAKE_CASE_				=				XLMRobertaTokenizer.from_pretrained('hf-internal-testing/tiny-xlm-roberta'       )
											SCREAMING_SNAKE_CASE_				=				77

											SCREAMING_SNAKE_CASE_				=				self.dummy_image.to(_lowercase       )

											# put models in fp16
											SCREAMING_SNAKE_CASE_				=				unet.half()
											SCREAMING_SNAKE_CASE_				=				vae.half()
											SCREAMING_SNAKE_CASE_				=				bert.half()

											# make sure here that pndm scheduler skips prk
											SCREAMING_SNAKE_CASE_				=				AltDiffusionImgaImgPipeline(
											    unet=_lowercase,     scheduler=_lowercase,     vae=_lowercase,     text_encoder=_lowercase,     tokenizer=_lowercase,     safety_checker=_lowercase,     feature_extractor=self.dummy_extractor,     )
											SCREAMING_SNAKE_CASE_				=				VaeImageProcessor(vae_scale_factor=alt_pipe.vae_scale_factor,     do_normalize=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				alt_pipe.to(_lowercase       )
											alt_pipe.set_progress_bar_config(disable=_lowercase       )

											SCREAMING_SNAKE_CASE_				=				'A painting of a squirrel eating a burger'
											SCREAMING_SNAKE_CASE_				=				torch.manual_seed(0       )
											SCREAMING_SNAKE_CASE_				=				alt_pipe(
											    [prompt],     generator=_lowercase,     num_inference_steps=2,     output_type='np',     image=_lowercase,     ).images

											assert image.shape == (1, 32, 32, 3)





				@unittest.skipIf(torch_device != 'cuda',     'This test requires a GPU'       )
				def   a__					(    self       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				load_image(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
											    '/img2img/sketch-mountains-input.jpg'       )
											# resize to resolution that is divisible by 8 but not 16 or 32
											SCREAMING_SNAKE_CASE_				=				init_image.resize((760, 504)       )

											SCREAMING_SNAKE_CASE_				=				'BAAI/AltDiffusion'
											SCREAMING_SNAKE_CASE_				=				AltDiffusionImgaImgPipeline.from_pretrained(
											    _lowercase,     safety_checker=_lowercase,     )
											pipe.to(_lowercase       )
											pipe.set_progress_bar_config(disable=_lowercase       )
											pipe.enable_attention_slicing()

											SCREAMING_SNAKE_CASE_				=				'A fantasy landscape, trending on artstation'

											SCREAMING_SNAKE_CASE_				=				torch.manual_seed(0       )
											SCREAMING_SNAKE_CASE_				=				pipe(
											    prompt=_lowercase,     image=_lowercase,     strength=0.75,     guidance_scale=7.5,     generator=_lowercase,     output_type='np',     )
											SCREAMING_SNAKE_CASE_				=				output.images[0]

											SCREAMING_SNAKE_CASE_				=				image[255:258, 383:386, -1]

											assert image.shape == (504, 760, 3)
											SCREAMING_SNAKE_CASE_				=				np.array([0.9_358, 0.9_397, 0.9_599, 0.9_901, 1.0_000, 1.0_000, 0.9_882, 1.0_000, 1.0_000]       )

											assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2






@slow
@require_torch_gpu
class    snake_case     (      unittest.TestCase		):






				"""simple docstring"""



				def   a__					(    self       )     ->							List[Any]:
											# clean up the VRAM after each test
											super().tearDown()
											gc.collect()
											torch.cuda.empty_cache()





				def   a__					(    self       )     ->							Optional[int]:
											SCREAMING_SNAKE_CASE_				=				load_image(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
											    '/img2img/sketch-mountains-input.jpg'       )
											SCREAMING_SNAKE_CASE_				=				init_image.resize((768, 512)       )
											SCREAMING_SNAKE_CASE_				=				load_numpy(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/fantasy_landscape_alt.npy'       )

											SCREAMING_SNAKE_CASE_				=				'BAAI/AltDiffusion'
											SCREAMING_SNAKE_CASE_				=				AltDiffusionImgaImgPipeline.from_pretrained(
											    _lowercase,     safety_checker=_lowercase,     )
											pipe.to(_lowercase       )
											pipe.set_progress_bar_config(disable=_lowercase       )
											pipe.enable_attention_slicing()

											SCREAMING_SNAKE_CASE_				=				'A fantasy landscape, trending on artstation'

											SCREAMING_SNAKE_CASE_				=				torch.manual_seed(0       )
											SCREAMING_SNAKE_CASE_				=				pipe(
											    prompt=_lowercase,     image=_lowercase,     strength=0.75,     guidance_scale=7.5,     generator=_lowercase,     output_type='np',     )
											SCREAMING_SNAKE_CASE_				=				output.images[0]

											assert image.shape == (512, 768, 3)
											# img2img is flaky across GPUs even in fp32, so using MAE here
											assert np.abs(expected_image - image       ).max() < 1E-2





