
"""simple docstring"""





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_a								=						logging.getLogger(__name__)



def  lowerCamelCase__						(     )				->						str:


					"""simple docstring"""





					_UpperCamelCase									=    argparse.ArgumentParser(
					    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''      )
					parser.add_argument(
					    '''--dataset_name''',	type=__snake_case,	default='''wikitext''',	help='''Name of the training. Explore datasets at: hf.co/datasets.''',	)
					parser.add_argument(
					    '''--dataset_config''',	type=__snake_case,	default='''wikitext-103-raw-v1''',	help='''Configuration name of the dataset.'''      )
					parser.add_argument(
					    '''--tokenizer_name_or_path''',	type=__snake_case,	default='''sayakpaul/unigram-tokenizer-wikitext''',	help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.''',	)
					parser.add_argument(
					    '''--shard_size''',	type=__snake_case,	default=10_00,	help='''Number of entries to go in a single shard.''',	)
					parser.add_argument('''--split''',	type=__snake_case,	default='''train''',	choices=['''train''', '''test''', '''validation''']      )
					parser.add_argument(
					    '''--limit''',	default=__snake_case,	type=__snake_case,	help='''Limit the number of shards (used for debugging).''',	)
					parser.add_argument(
					    '''--max_length''',	type=__snake_case,	default=5_12,	help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
					    ''' sequence length that is a multiple of 8.''',	)
					parser.add_argument(
					    '''--output_dir''',	default='''tf-tpu''',	type=__snake_case,	help='''Output directory where the TFRecord shards will be saved. If the'''
					    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
					    ''' shards will be directly saved to a Google Cloud Storage bucket.''',	)

					_UpperCamelCase									=    parser.parse_args()
					return args



def  lowerCamelCase__						(     __snake_case      )				->						Union[str, Any]:


					"""simple docstring"""





					def fn(__snake_case      ):
										return tokenizer(examples['''text''']      )

					return fn



def  lowerCamelCase__						(     __snake_case      )				->						str:


					"""simple docstring"""





					_UpperCamelCase									=    []
					for i in range(len(tokenized_data['''input_ids''']      )      ):
										_UpperCamelCase									=    {
										    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]      )      ),
										    '''attention_mask''': tf.train.Feature(
										        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]      )      ),
										}
										_UpperCamelCase									=    tf.train.Features(feature=__snake_case      )
										_UpperCamelCase									=    tf.train.Example(features=__snake_case      )
										_UpperCamelCase									=    example.SerializeToString()
										records.append(__snake_case      )
					return records



def  lowerCamelCase__						(     __snake_case      )				->						str:


					"""simple docstring"""





					_UpperCamelCase									=    datasets.load_dataset(args.dataset_name,	args.dataset_config,	split=args.split      )

					if args.limit is not None:
										_UpperCamelCase									=    min(len(__snake_case      ),	args.limit      )
										_UpperCamelCase									=    dataset.select(range(__snake_case      )      )
										print(F'''Limiting the dataset to {args.limit} entries.'''      )

					_UpperCamelCase									=    AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

					# Handle output directory creation.
					# For serializing into a Google Cloud Storage Bucket, one needs to first
					# create a bucket.
					if "gs" not in args.output_dir:
										if not os.path.exists(args.output_dir      ):
															os.makedirs(args.output_dir      )
										_UpperCamelCase									=    os.path.join(args.output_dir,	args.split      )
										if not os.path.exists(__snake_case      ):
															os.makedirs(__snake_case      )
					else:
										_UpperCamelCase									=    os.path.join(args.output_dir,	args.split      )

					# Tokenize the whole dataset at once.
					_UpperCamelCase									=    tokenize_function(__snake_case      )
					_UpperCamelCase									=    dataset.map(__snake_case,	batched=__snake_case,	num_proc=4,	remove_columns=['''text''']      )

					# We need to concatenate all our texts together, and then split the result
					# into chunks of a fixed size, which we will call block_size. To do this, we
					# will use the map method again, with the option batched=True. When we use batched=True,
					# the function we pass to map() will be passed multiple inputs at once, allowing us
					# to group them into more or fewer examples than we had in the input.
					# This allows us to create our new fixed-length samples. The advantage of this
					# method is that we don't lose a whole lot of content from the dataset compared to the
					# case where we simply tokenize with a pre-defined max_length.

					def group_texts(__snake_case      ):
										# Concatenate all texts.
										_UpperCamelCase									=    {k: sum(examples[k],	[]      ) for k in examples.keys()}
										_UpperCamelCase									=    len(concatenated_examples[list(examples.keys()      )[0]]      )
										# We drop the small remainder, though you could add padding instead if the model supports it
										# In this, as in all things, we advise you to follow your heart 🫀
										_UpperCamelCase									=    (total_length // args.max_length) * args.max_length
										# Split by chunks of max_len.
										_UpperCamelCase									=    {
										    k: [t[i : i + args.max_length] for i in range(0,	__snake_case,	args.max_length      )]
										    for k, t in concatenated_examples.items()
										}
										return result

					_UpperCamelCase									=    dataset_tokenized.map(__snake_case,	batched=__snake_case,	batch_size=10_00,	num_proc=4      )

					_UpperCamelCase									=    0
					_UpperCamelCase									=    0
					for shard in range(0,	len(__snake_case      ),	args.shard_size      ):
										_UpperCamelCase									=    grouped_dataset[shard : shard + args.shard_size]
										_UpperCamelCase									=    len(dataset_snapshot['''input_ids''']      )
										_UpperCamelCase									=    os.path.join(__snake_case,	F'''dataset-{shard_count}-{records_containing}.tfrecord'''      )
										_UpperCamelCase									=    get_serialized_examples(__snake_case      )

										with tf.io.TFRecordWriter(__snake_case      ) as out_file:
															for i in range(len(__snake_case      )      ):
																				_UpperCamelCase									=    serialized_examples[i]
																				out_file.write(__snake_case      )
															print('''Wrote file {} containing {} records'''.format(__snake_case,	__snake_case      )      )

										shard_count += 1
										total_records += records_containing

					with open(F'''split-{args.split}-records-count.txt''',	'''w'''      ) as f:
										print(F'''Total {args.split} records: {total_records}''',	file=__snake_case      )


if __name__ == "__main__":
				_a								=						parse_args()
				main(args)








"""simple docstring"""





import torch

from diffusers import DPMSolverSDEScheduler
from diffusers.utils import torch_device
from diffusers.utils.testing_utils import require_torchsde

from .test_schedulers import SchedulerCommonTest






@require_torchsde
class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = (DPMSolverSDEScheduler,)
		lowercase__         = 10



		def 		UpperCAmelCase				(   self							,						**__a)			->			int:






							'''simple docstring'''
							_UpperCamelCase									=    {
							    '''num_train_timesteps''': 11_00,
							    '''beta_start''': 0.0001,
							    '''beta_end''': 0.02,
							    '''beta_schedule''': '''linear''',
							    '''noise_sampler_seed''': 0,
							}

							config.update(**__a)
							return config



		def 		UpperCAmelCase				(   self)			->			Union[str, Any]:






							'''simple docstring'''
							for timesteps in [10, 50, 1_00, 10_00]:
												self.check_over_configs(num_train_timesteps=__a)



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							for beta_start, beta_end in zip([0.0_0001, 0.0001, 0.001]							,						[0.0002, 0.002, 0.02]):
												self.check_over_configs(beta_start=__a							,						beta_end=__a)



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							for schedule in ["linear", "scaled_linear"]:
												self.check_over_configs(beta_schedule=__a)



		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							for prediction_type in ["epsilon", "v_prediction"]:
												self.check_over_configs(prediction_type=__a)



		def 		UpperCAmelCase				(   self)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.scheduler_classes[0]
							_UpperCamelCase									=    self.get_scheduler_config()
							_UpperCamelCase									=    scheduler_class(**__a)

							scheduler.set_timesteps(self.num_inference_steps)

							_UpperCamelCase									=    self.dummy_model()
							_UpperCamelCase									=    self.dummy_sample_deter * scheduler.init_noise_sigma
							_UpperCamelCase									=    sample.to(__a)

							for i, t in enumerate(scheduler.timesteps):
												_UpperCamelCase									=    scheduler.scale_model_input(__a							,						__a)

												_UpperCamelCase									=    model(__a							,						__a)

												_UpperCamelCase									=    scheduler.step(__a							,						__a							,						__a)
												_UpperCamelCase									=    output.prev_sample

							_UpperCamelCase									=    torch.sum(torch.abs(__a))
							_UpperCamelCase									=    torch.mean(torch.abs(__a))

							if torch_device in ["mps"]:
												assert abs(result_sum.item() - 167.47_8210_4492_1875) < 1e-2
												assert abs(result_mean.item() - 0.2178_7059_6456_5277) < 1e-3
							elif torch_device in ["cuda"]:
												assert abs(result_sum.item() - 171.59_3521_1181_6406) < 1e-2
												assert abs(result_mean.item() - 0.2_2342_9068_9229_9652) < 1e-3
							else:
												assert abs(result_sum.item() - 162.52_3834_2285_1562) < 1e-2
												assert abs(result_mean.item() - 0.211_6195_7085_1326) < 1e-3



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							_UpperCamelCase									=    self.scheduler_classes[0]
							_UpperCamelCase									=    self.get_scheduler_config(prediction_type='''v_prediction''')
							_UpperCamelCase									=    scheduler_class(**__a)

							scheduler.set_timesteps(self.num_inference_steps)

							_UpperCamelCase									=    self.dummy_model()
							_UpperCamelCase									=    self.dummy_sample_deter * scheduler.init_noise_sigma
							_UpperCamelCase									=    sample.to(__a)

							for i, t in enumerate(scheduler.timesteps):
												_UpperCamelCase									=    scheduler.scale_model_input(__a							,						__a)

												_UpperCamelCase									=    model(__a							,						__a)

												_UpperCamelCase									=    scheduler.step(__a							,						__a							,						__a)
												_UpperCamelCase									=    output.prev_sample

							_UpperCamelCase									=    torch.sum(torch.abs(__a))
							_UpperCamelCase									=    torch.mean(torch.abs(__a))

							if torch_device in ["mps"]:
												assert abs(result_sum.item() - 124.77_1492_0043_9453) < 1e-2
												assert abs(result_mean.item() - 0.1_6226_2890_1481_6284) < 1e-3
							elif torch_device in ["cuda"]:
												assert abs(result_sum.item() - 128.1_6633_6059_5703) < 1e-2
												assert abs(result_mean.item() - 0.1_6688_3260_0116_7297) < 1e-3
							else:
												assert abs(result_sum.item() - 119.8_4875_4882_8125) < 1e-2
												assert abs(result_mean.item() - 0.1560_5306_6253_6621) < 1e-3



		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    self.scheduler_classes[0]
							_UpperCamelCase									=    self.get_scheduler_config()
							_UpperCamelCase									=    scheduler_class(**__a)

							scheduler.set_timesteps(self.num_inference_steps							,						device=__a)

							_UpperCamelCase									=    self.dummy_model()
							_UpperCamelCase									=    self.dummy_sample_deter.to(__a) * scheduler.init_noise_sigma

							for t in scheduler.timesteps:
												_UpperCamelCase									=    scheduler.scale_model_input(__a							,						__a)

												_UpperCamelCase									=    model(__a							,						__a)

												_UpperCamelCase									=    scheduler.step(__a							,						__a							,						__a)
												_UpperCamelCase									=    output.prev_sample

							_UpperCamelCase									=    torch.sum(torch.abs(__a))
							_UpperCamelCase									=    torch.mean(torch.abs(__a))

							if torch_device in ["mps"]:
												assert abs(result_sum.item() - 167.46_9573_9746_0938) < 1e-2
												assert abs(result_mean.item() - 0.2_1805_9346_0798_2635) < 1e-3
							elif torch_device in ["cuda"]:
												assert abs(result_sum.item() - 171.59_3536_3769_5312) < 1e-2
												assert abs(result_mean.item() - 0.2_2342_9083_8241_5771) < 1e-3
							else:
												assert abs(result_sum.item() - 162.52_3834_2285_1562) < 1e-2
												assert abs(result_mean.item() - 0.211_6195_7085_1326) < 1e-3




		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    self.scheduler_classes[0]
							_UpperCamelCase									=    self.get_scheduler_config()
							_UpperCamelCase									=    scheduler_class(**__a							,						use_karras_sigmas=__a)

							scheduler.set_timesteps(self.num_inference_steps							,						device=__a)

							_UpperCamelCase									=    self.dummy_model()
							_UpperCamelCase									=    self.dummy_sample_deter.to(__a) * scheduler.init_noise_sigma
							_UpperCamelCase									=    sample.to(__a)

							for t in scheduler.timesteps:
												_UpperCamelCase									=    scheduler.scale_model_input(__a							,						__a)

												_UpperCamelCase									=    model(__a							,						__a)

												_UpperCamelCase									=    scheduler.step(__a							,						__a							,						__a)
												_UpperCamelCase									=    output.prev_sample

							_UpperCamelCase									=    torch.sum(torch.abs(__a))
							_UpperCamelCase									=    torch.mean(torch.abs(__a))

							if torch_device in ["mps"]:
												assert abs(result_sum.item() - 176.66_9741_3574_2188) < 1e-2
												assert abs(result_mean.item() - 0.2_3003_8727_3098_1811) < 1e-2
							elif torch_device in ["cuda"]:
												assert abs(result_sum.item() - 177.63_6535_6445_3125) < 1e-2
												assert abs(result_mean.item() - 0.2_3003_8727_3098_1811) < 1e-2
							else:
												assert abs(result_sum.item() - 170.3_1352_2338_8672) < 1e-2
												assert abs(result_mean.item() - 0.2_3003_8727_3098_1811) < 1e-2






