






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase__							  =						logging.getLogger(__name__)
def 		_a       ( )				->     Any:
 a						    = argparse.ArgumentParser(
     description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''					)
 parser.add_argument(
     '''--dataset_name'''      ,				type=a      ,				default='''wikitext'''      ,				help='''Name of the training. Explore datasets at: hf.co/datasets.'''      ,				)
 parser.add_argument(
     '''--dataset_config'''      ,				type=a      ,				default='''wikitext-103-raw-v1'''      ,				help='''Configuration name of the dataset.'''					)
 parser.add_argument(
     '''--tokenizer_name_or_path'''      ,				type=a      ,				default='''sayakpaul/unigram-tokenizer-wikitext'''      ,				help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''      ,				)
 parser.add_argument(
     '''--shard_size'''      ,				type=a      ,				default=1_000      ,				help='''Number of entries to go in a single shard.'''      ,				)
 parser.add_argument('''--split'''      ,				type=a      ,				default='''train'''      ,				choices=['''train''', '''test''', '''validation''']					)
 parser.add_argument(
     '''--limit'''      ,				default=a      ,				type=a      ,				help='''Limit the number of shards (used for debugging).'''      ,				)
 parser.add_argument(
     '''--max_length'''      ,				type=a      ,				default=512      ,				help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
     ''' sequence length that is a multiple of 8.'''      ,				)
 parser.add_argument(
     '''--output_dir'''      ,				default='''tf-tpu'''      ,				type=a      ,				help='''Output directory where the TFRecord shards will be saved. If the'''
     ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
     ''' shards will be directly saved to a Google Cloud Storage bucket.'''      ,				)

 a						    = parser.parse_args()
 return args
def 		_a       ( a					:Dict					)				->     str:
 def fn(a					:int					):
  return tokenizer(examples['''text''']					)

 return fn
def 		_a       ( a					:int					)				->     Optional[Any]:
 a						    = []
 for i in range(len(tokenized_data['''input_ids''']					)					):
  a						    = {
      '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]					)					),
      '''attention_mask''': tf.train.Feature(
          intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]					)					),
  }
  a						    = tf.train.Features(feature=a					)
  a						    = tf.train.Example(features=a					)
  a						    = example.SerializeToString()
  records.append(a					)
 return records
def 		_a       ( a					:str					)				->     Optional[int]:
 a						    = datasets.load_dataset(args.dataset_name      ,				args.dataset_config      ,				split=args.split					)

 if args.limit is not None:
  a						    = min(len(a					)      ,				args.limit					)
  a						    = dataset.select(range(a					)					)
  print(F"""Limiting the dataset to {args.limit} entries."""					)

 a						    = AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

 # Handle output directory creation.
 # For serializing into a Google Cloud Storage Bucket, one needs to first
 # create a bucket.
 if "gs" not in args.output_dir:
  if not os.path.exists(args.output_dir					):
   os.makedirs(args.output_dir					)
  a						    = os.path.join(args.output_dir      ,				args.split					)
  if not os.path.exists(a					):
   os.makedirs(a					)
 else:
  a						    = os.path.join(args.output_dir      ,				args.split					)

 # Tokenize the whole dataset at once.
 a						    = tokenize_function(a					)
 a						    = dataset.map(a      ,				batched=a      ,				num_proc=4      ,				remove_columns=['''text''']					)

 # We need to concatenate all our texts together, and then split the result
 # into chunks of a fixed size, which we will call block_size. To do this, we
 # will use the map method again, with the option batched=True. When we use batched=True,
 # the function we pass to map() will be passed multiple inputs at once, allowing us
 # to group them into more or fewer examples than we had in the input.
 # This allows us to create our new fixed-length samples. The advantage of this
 # method is that we don't lose a whole lot of content from the dataset compared to the
 # case where we simply tokenize with a pre-defined max_length.

 def group_texts(a					:int					):
  # Concatenate all texts.
  a						    = {k: sum(examples[k]      ,				[]					) for k in examples.keys()}
  a						    = len(concatenated_examples[list(examples.keys()					)[0]]					)
  # We drop the small remainder, though you could add padding instead if the model supports it
  # In this, as in all things, we advise you to follow your heart 🫀
  a						    = (total_length // args.max_length) * args.max_length
  # Split by chunks of max_len.
  a						    = {
      k: [t[i : i + args.max_length] for i in range(0      ,				a      ,				args.max_length					)]
      for k, t in concatenated_examples.items()
  }
  return result

 a						    = dataset_tokenized.map(a      ,				batched=a      ,				batch_size=1_000      ,				num_proc=4					)

 a						    = 0
 a						    = 0
 for shard in range(0      ,				len(a					)      ,				args.shard_size					):
  a						    = grouped_dataset[shard : shard + args.shard_size]
  a						    = len(dataset_snapshot['''input_ids''']					)
  a						    = os.path.join(a      ,				F"""dataset-{shard_count}-{records_containing}.tfrecord"""					)
  a						    = get_serialized_examples(a					)

  with tf.io.TFRecordWriter(a					) as out_file:
   for i in range(len(a					)					):
    a						    = serialized_examples[i]
    out_file.write(a					)
   print('''Wrote file {} containing {} records'''.format(a      ,				a					)					)

  shard_count += 1
  total_records += records_containing

 with open(F"""split-{args.split}-records-count.txt"""      ,				'''w'''					) as f:
  print(F"""Total {args.split} records: {total_records}"""      ,				file=a					)


if __name__ == "__main__":
     UpperCAmelCase__							  =						parse_args()
     main(args)










import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import TransformeraDModel, VQDiffusionPipeline, VQDiffusionScheduler, VQModel
from diffusers.pipelines.vq_diffusion.pipeline_vq_diffusion import LearnedClassifierFreeSamplingEmbeddings
from diffusers.utils import load_numpy, slow, torch_device
from diffusers.utils.testing_utils import require_torch_gpu


UpperCAmelCase__							  =						False






class 						lowercase_   (      unittest.TestCase  ):






      '''simple docstring'''




      def  __lowerCAmelCase   ( self :		Dict    )       ->str:




       """simple docstring"""


       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()







      @property
      def  __lowerCAmelCase   ( self :		List[str]    )       ->Any:




       """simple docstring"""


       return 12







      @property
      def  __lowerCAmelCase   ( self :		Optional[Any]    )       ->Dict:




       """simple docstring"""


       return 12







      @property
      def  __lowerCAmelCase   ( self :		Any    )       ->Any:




       """simple docstring"""


       return 32







      @property
      def  __lowerCAmelCase   ( self :		int    )       ->List[str]:




       """simple docstring"""


       torch.manual_seed(0    )
       a						    = VQModel(
           block_out_channels=[32, 64]					,   in_channels=3					,   out_channels=3					,   down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']					,   up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']					,   latent_channels=3					,   num_vq_embeddings=self.num_embed					,   vq_embed_dim=3					,   )
       return model







      @property
      def  __lowerCAmelCase   ( self :		str    )       ->str:




       """simple docstring"""


       a						    = CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''    )
       return tokenizer







      @property
      def  __lowerCAmelCase   ( self :		str    )       ->Union[str, Any]:




       """simple docstring"""


       torch.manual_seed(0    )
       a						    = CLIPTextConfig(
           bos_token_id=0					,   eos_token_id=2					,   hidden_size=self.text_embedder_hidden_size					,   intermediate_size=37					,   layer_norm_eps=1e-0_5					,   num_attention_heads=4					,   num_hidden_layers=5					,   pad_token_id=1					,   vocab_size=1_000					,   )
       return CLIPTextModel(__UpperCAmelCase    )







      @property
      def  __lowerCAmelCase   ( self :		Any    )       ->Tuple:




       """simple docstring"""


       torch.manual_seed(0    )

       a						    = 12
       a						    = 12

       a						    = {
           '''attention_bias''': True,
           '''cross_attention_dim''': 32,
           '''attention_head_dim''': height * width,
           '''num_attention_heads''': 1,
           '''num_vector_embeds''': self.num_embed,
           '''num_embeds_ada_norm''': self.num_embeds_ada_norm,
           '''norm_num_groups''': 32,
           '''sample_size''': width,
           '''activation_fn''': '''geglu-approximate''',
       }

       a						    = TransformeraDModel(**__UpperCAmelCase    )
       return model







      def  __lowerCAmelCase   ( self :		Dict    )       ->Any:




       """simple docstring"""


       a						    = '''cpu'''

       a						    = self.dummy_vqvae
       a						    = self.dummy_text_encoder
       a						    = self.dummy_tokenizer
       a						    = self.dummy_transformer
       a						    = VQDiffusionScheduler(self.num_embed    )
       a						    = LearnedClassifierFreeSamplingEmbeddings(learnable=__UpperCAmelCase    )

       a						    = VQDiffusionPipeline(
           vqvae=__UpperCAmelCase					,   text_encoder=__UpperCAmelCase					,   tokenizer=__UpperCAmelCase					,   transformer=__UpperCAmelCase					,   scheduler=__UpperCAmelCase					,   learned_classifier_free_sampling_embeddings=__UpperCAmelCase					,   )
       a						    = pipe.to(__UpperCAmelCase    )
       pipe.set_progress_bar_config(disable=__UpperCAmelCase    )

       a						    = '''teddy bear playing in the pool'''

       a						    = torch.Generator(device=__UpperCAmelCase    ).manual_seed(0    )
       a						    = pipe([prompt]					,   generator=__UpperCAmelCase					,   num_inference_steps=2					,   output_type='''np'''    )
       a						    = output.images

       a						    = torch.Generator(device=__UpperCAmelCase    ).manual_seed(0    )
       a						    = pipe(
           [prompt]					,   generator=__UpperCAmelCase					,   output_type='''np'''					,   return_dict=__UpperCAmelCase					,   num_inference_steps=2    )[0]

       a						    = image[0, -3:, -3:, -1]
       a						    = image_from_tuple[0, -3:, -3:, -1]

       assert image.shape == (1, 24, 24, 3)

       a						    = np.array([0.6551, 0.6168, 0.5008, 0.5676, 0.5659, 0.4295, 0.6073, 0.5599, 0.4992]    )

       assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1e-2
       assert np.abs(image_from_tuple_slice.flatten() - expected_slice    ).max() < 1e-2







      def  __lowerCAmelCase   ( self :		Tuple    )       ->str:




       """simple docstring"""


       a						    = '''cpu'''

       a						    = self.dummy_vqvae
       a						    = self.dummy_text_encoder
       a						    = self.dummy_tokenizer
       a						    = self.dummy_transformer
       a						    = VQDiffusionScheduler(self.num_embed    )
       a						    = LearnedClassifierFreeSamplingEmbeddings(
           learnable=__UpperCAmelCase					,   hidden_size=self.text_embedder_hidden_size					,   length=tokenizer.model_max_length    )

       a						    = VQDiffusionPipeline(
           vqvae=__UpperCAmelCase					,   text_encoder=__UpperCAmelCase					,   tokenizer=__UpperCAmelCase					,   transformer=__UpperCAmelCase					,   scheduler=__UpperCAmelCase					,   learned_classifier_free_sampling_embeddings=__UpperCAmelCase					,   )
       a						    = pipe.to(__UpperCAmelCase    )
       pipe.set_progress_bar_config(disable=__UpperCAmelCase    )

       a						    = '''teddy bear playing in the pool'''

       a						    = torch.Generator(device=__UpperCAmelCase    ).manual_seed(0    )
       a						    = pipe([prompt]					,   generator=__UpperCAmelCase					,   num_inference_steps=2					,   output_type='''np'''    )
       a						    = output.images

       a						    = torch.Generator(device=__UpperCAmelCase    ).manual_seed(0    )
       a						    = pipe(
           [prompt]					,   generator=__UpperCAmelCase					,   output_type='''np'''					,   return_dict=__UpperCAmelCase					,   num_inference_steps=2    )[0]

       a						    = image[0, -3:, -3:, -1]
       a						    = image_from_tuple[0, -3:, -3:, -1]

       assert image.shape == (1, 24, 24, 3)

       a						    = np.array([0.6693, 0.6075, 0.4959, 0.5701, 0.5583, 0.4333, 0.6171, 0.5684, 0.4988]    )

       assert np.abs(image_slice.flatten() - expected_slice    ).max() < 2.0
       assert np.abs(image_from_tuple_slice.flatten() - expected_slice    ).max() < 1e-2






@slow
@require_torch_gpu
class 						lowercase_   (      unittest.TestCase  ):






      '''simple docstring'''




      def  __lowerCAmelCase   ( self :		Union[str, Any]    )       ->Union[str, Any]:




       """simple docstring"""


       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()







      def  __lowerCAmelCase   ( self :		Dict    )       ->int:




       """simple docstring"""


       a						    = load_numpy(
           '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
           '''/vq_diffusion/teddy_bear_pool_classifier_free_sampling.npy'''    )

       a						    = VQDiffusionPipeline.from_pretrained('''microsoft/vq-diffusion-ithq'''    )
       a						    = pipeline.to(__UpperCAmelCase    )
       pipeline.set_progress_bar_config(disable=__UpperCAmelCase    )

       # requires GPU generator for gumbel softmax
       # don't use GPU generator in tests though
       a						    = torch.Generator(device=__UpperCAmelCase    ).manual_seed(0    )
       a						    = pipeline(
           '''teddy bear playing in the pool'''					,   num_images_per_prompt=1					,   generator=__UpperCAmelCase					,   output_type='''np'''					,   )

       a						    = output.images[0]

       assert image.shape == (256, 256, 3)
       assert np.abs(expected_image - image    ).max() < 2.0


