


"""simple docstring"""



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__snake_case         =					logging.getLogger(__name__)







def 						__lowerCAmelCase		(		)			->		List[Any]:


    """simple docstring"""





    snake_case					:   List[Any]         =					argparse.ArgumentParser(
        description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."				)
    parser.add_argument(
        "--dataset_name"		,							type=lowercase		,							default="wikitext"		,							help="Name of the training. Explore datasets at: hf.co/datasets."		,							)
    parser.add_argument(
        "--dataset_config"		,							type=lowercase		,							default="wikitext-103-raw-v1"		,							help="Configuration name of the dataset."				)
    parser.add_argument(
        "--tokenizer_name_or_path"		,							type=lowercase		,							default="sayakpaul/unigram-tokenizer-wikitext"		,							help="Tokenizer identifier. Can be a local filepath or a Hub identifier."		,							)
    parser.add_argument(
        "--shard_size"		,							type=lowercase		,							default=1000		,							help="Number of entries to go in a single shard."		,							)
    parser.add_argument("--split"		,							type=lowercase		,							default="train"		,							choices=["train", "test", "validation"]				)
    parser.add_argument(
        "--limit"		,							default=lowercase		,							type=lowercase		,							help="Limit the number of shards (used for debugging)."		,							)
    parser.add_argument(
        "--max_length"		,							type=lowercase		,							default=512		,							help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
        " sequence length that is a multiple of 8."		,							)
    parser.add_argument(
        "--output_dir"		,							default="tf-tpu"		,							type=lowercase		,							help="Output directory where the TFRecord shards will be saved. If the"
        " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
        " shards will be directly saved to a Google Cloud Storage bucket."		,							)

    snake_case					:   Union[str, Any]         =					parser.parse_args()
    return args







def 						__lowerCAmelCase		(		lowercase						:						List[Any]				)			->		Any:


    """simple docstring"""





    def fn(lowercase						:						List[str]				):
        return tokenizer(examples["text"]				)

    return fn







def 						__lowerCAmelCase		(		lowercase						:						str				)			->		List[Any]:


    """simple docstring"""





    snake_case					:   Dict         =					[]
    for i in range(len(tokenized_data["input_ids"]				)				):
        snake_case					:   Optional[Any]         =					{
            "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]				)				),
            "attention_mask": tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]				)				),
        }
        snake_case					:   Optional[Any]         =					tf.train.Features(feature=lowercase				)
        snake_case					:   Optional[int]         =					tf.train.Example(features=lowercase				)
        snake_case					:   str         =					example.SerializeToString()
        records.append(lowercase				)
    return records







def 						__lowerCAmelCase		(		lowercase						:						Dict				)			->		Optional[Any]:


    """simple docstring"""





    snake_case					:   int         =					datasets.load_dataset(args.dataset_name		,							args.dataset_config		,							split=args.split				)

    if args.limit is not None:
        snake_case					:   Dict         =					min(len(lowercase				)		,							args.limit				)
        snake_case					:   Dict         =					dataset.select(range(lowercase				)				)
        print(F'Limiting the dataset to {args.limit} entries.'				)

    snake_case					:   List[str]         =					AutoTokenizer.from_pretrained(args.tokenizer_name_or_path				)

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir				):
            os.makedirs(args.output_dir				)
        snake_case					:   str         =					os.path.join(args.output_dir		,							args.split				)
        if not os.path.exists(lowercase				):
            os.makedirs(lowercase				)
    else:
        snake_case					:   Tuple         =					os.path.join(args.output_dir		,							args.split				)

    # Tokenize the whole dataset at once.
    snake_case					:   Any         =					tokenize_function(lowercase				)
    snake_case					:   Optional[Any]         =					dataset.map(lowercase		,							batched=lowercase		,							num_proc=4		,							remove_columns=["text"]				)

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(lowercase						:						int				):
        # Concatenate all texts.
        snake_case					:   List[str]         =					{k: sum(examples[k]		,							[]				) for k in examples.keys()}
        snake_case					:   List[Any]         =					len(concatenated_examples[list(examples.keys()				)[0]]				)
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        snake_case					:   Union[str, Any]         =					(total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        snake_case					:   List[str]         =					{
            k: [t[i : i + args.max_length] for i in range(0		,							lowercase		,							args.max_length				)]
            for k, t in concatenated_examples.items()
        }
        return result

    snake_case					:   Any         =					dataset_tokenized.map(lowercase		,							batched=lowercase		,							batch_size=1000		,							num_proc=4				)

    snake_case					:   str         =					0
    snake_case					:   Dict         =					0
    for shard in range(0		,							len(lowercase				)		,							args.shard_size				):
        snake_case					:   Dict         =					grouped_dataset[shard : shard + args.shard_size]
        snake_case					:   List[str]         =					len(dataset_snapshot["input_ids"]				)
        snake_case					:   Union[str, Any]         =					os.path.join(lowercase		,							F'dataset-{shard_count}-{records_containing}.tfrecord'				)
        snake_case					:   Any         =					get_serialized_examples(lowercase				)

        with tf.io.TFRecordWriter(lowercase				) as out_file:
            for i in range(len(lowercase				)				):
                snake_case					:   Tuple         =					serialized_examples[i]
                out_file.write(lowercase				)
            print("Wrote file {} containing {} records".format(lowercase		,							lowercase				)				)

        shard_count += 1
        total_records += records_containing

    with open(F'split-{args.split}-records-count.txt'		,							"w"				) as f:
        print(F'Total {args.split} records: {total_records}'		,							file=lowercase				)


if __name__ == "__main__":
  __snake_case         =					parse_args()
  main(args)









"""simple docstring"""



import unittest

import numpy as np
import torch

from diffusers import PNDMPipeline, PNDMScheduler, UNetaDModel
from diffusers.utils.testing_utils import enable_full_determinism, require_torch, slow, torch_device


enable_full_determinism()







class _lowerCAmelCase       (   unittest.TestCase			):







   @property
   def 				lowerCamelCase	(			self		)			->	Any:

       '''simple docstring'''




       torch.manual_seed(0		)
       snake_case					:   Tuple         =					UNetaDModel(
           block_out_channels=(32, 64)		,			layers_per_block=2		,			sample_size=32		,			in_channels=3		,			out_channels=3		,			down_block_types=("DownBlock2D", "AttnDownBlock2D")		,			up_block_types=("AttnUpBlock2D", "UpBlock2D")		,			)
       return model







   def 				lowerCamelCase	(			self		)			->	Optional[Any]:

       '''simple docstring'''




       snake_case					:   Optional[Any]         =					self.dummy_uncond_unet
       snake_case					:   str         =					PNDMScheduler()

       snake_case					:   List[Any]         =					PNDMPipeline(unet=UpperCamelCase__		,			scheduler=UpperCamelCase__		)
       pndm.to(UpperCamelCase__		)
       pndm.set_progress_bar_config(disable=UpperCamelCase__		)

       snake_case					:   str         =					torch.manual_seed(0		)
       snake_case					:   Union[str, Any]         =					pndm(generator=UpperCamelCase__		,			num_inference_steps=20		,			output_type="numpy"		).images

       snake_case					:   Optional[int]         =					torch.manual_seed(0		)
       snake_case					:   Any         =					pndm(generator=UpperCamelCase__		,			num_inference_steps=20		,			output_type="numpy"		,			return_dict=UpperCamelCase__		)[0]

       snake_case					:   Optional[Any]         =					image[0, -3:, -3:, -1]
       snake_case					:   List[str]         =					image_from_tuple[0, -3:, -3:, -1]

       assert image.shape == (1, 32, 32, 3)
       snake_case					:   int         =					np.array([1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0]		)

       assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2
       assert np.abs(image_from_tuple_slice.flatten() - expected_slice		).max() < 1e-2







@slow
@require_torch
class _lowerCAmelCase       (   unittest.TestCase			):







   def 				lowerCamelCase	(			self		)			->	Optional[int]:

       '''simple docstring'''




       snake_case					:   Dict         =					"google/ddpm-cifar10-32"

       snake_case					:   Optional[int]         =					UNetaDModel.from_pretrained(UpperCamelCase__		)
       snake_case					:   List[str]         =					PNDMScheduler()

       snake_case					:   Union[str, Any]         =					PNDMPipeline(unet=UpperCamelCase__		,			scheduler=UpperCamelCase__		)
       pndm.to(UpperCamelCase__		)
       pndm.set_progress_bar_config(disable=UpperCamelCase__		)
       snake_case					:   int         =					torch.manual_seed(0		)
       snake_case					:   str         =					pndm(generator=UpperCamelCase__		,			output_type="numpy"		).images

       snake_case					:   Union[str, Any]         =					image[0, -3:, -3:, -1]

       assert image.shape == (1, 32, 32, 3)
       snake_case					:   Optional[Any]         =					np.array([0.1564, 0.14645, 0.1406, 0.14715, 0.12425, 0.14045, 0.13115, 0.12175, 0.125]		)

       assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2





