



'''simple docstring'''






import copy
import random

from transformers import CLIPTokenizer

class        SCREAMING_SNAKE_CASE(							A__    ):







	"""simple docstring"""






	def __init__(		self    :  Optional[Any]	,    *__snake_case    :  str	,    **__snake_case    :  Optional[Any]	)    ->	Optional[Any]:
								super().__init__(*__snake_case	,    **__snake_case	)
								UpperCAmelCase				: Optional[Any]											=	{}







	def   A			(		self    :  Dict	,    __snake_case    :  Optional[Any]	,    *__snake_case    :  str	,    **__snake_case    :  Union[str, Any]	)    ->	List[str]:
								UpperCAmelCase				: List[Any]											=	super().add_tokens(__snake_case	,    *__snake_case	,    **__snake_case	)
								if num_added_tokens == 0:
															raise ValueError(
															    F"""The tokenizer already contains the token {placeholder_token}. Please pass a different"""
															    ''' `placeholder_token` that is not already in the tokenizer.'''	)







	def   A			(		self    :  Dict	,    __snake_case    :  Tuple	,    *__snake_case    :  Dict	,    __snake_case    :  List[str]=1	,    **__snake_case    :  Optional[int]	)    ->	Tuple:
								UpperCAmelCase				: List[Any]											=	[]
								if num_vec_per_token == 1:
															self.try_adding_tokens(__snake_case	,    *__snake_case	,    **__snake_case	)
															output.append(__snake_case	)
								else:
															UpperCAmelCase				: List[Any]											=	[]
															for i in range(__snake_case	):
																						UpperCAmelCase				: Union[str, Any]											=	placeholder_token + F"""_{i}"""
																						self.try_adding_tokens(__snake_case	,    *__snake_case	,    **__snake_case	)
																						output.append(__snake_case	)
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
								for token in self.token_map:
															if token in placeholder_token:
																						raise ValueError(
																						    F"""The tokenizer already has placeholder token {token} that can get confused with"""
																						    F""" {placeholder_token}keep placeholder tokens independent"""	)
								UpperCAmelCase				: Tuple											=	output







	def   A			(		self    :  Tuple	,    __snake_case    :  Any	,    __snake_case    :  Optional[int]=False	,    __snake_case    :  Optional[int]=1.0	)    ->	List[str]:
								if isinstance(__snake_case	,    __snake_case	):
															UpperCAmelCase				: Any											=	[]
															for i in range(len(__snake_case	)	):
																						output.append(self.replace_placeholder_tokens_in_text(text[i]	,    vector_shuffle=__snake_case	)	)
															return output
								for placeholder_token in self.token_map:
															if placeholder_token in text:
																						UpperCAmelCase				: str											=	self.token_map[placeholder_token]
																						UpperCAmelCase				: Union[str, Any]											=	tokens[: 1 + int(len(__snake_case	) * prop_tokens_to_load	)]
																						if vector_shuffle:
																													UpperCAmelCase				: List[str]											=	copy.copy(__snake_case	)
																													random.shuffle(__snake_case	)
																						UpperCAmelCase				: List[str]											=	text.replace(__snake_case	,    ''' '''.join(__snake_case	)	)
								return text







	def __call__(		self    :  Any	,    __snake_case    :  Any	,    *__snake_case    :  List[Any]	,    __snake_case    :  int=False	,    __snake_case    :  str=1.0	,    **__snake_case    :  Any	)    ->	List[Any]:
								return super().__call__(
								    self.replace_placeholder_tokens_in_text(
								        __snake_case	,    vector_shuffle=__snake_case	,    prop_tokens_to_load=__snake_case	)	,    *__snake_case	,    **__snake_case	,    )







	def   A			(		self    :  List[str]	,    __snake_case    :  Optional[int]	,    *__snake_case    :  List[Any]	,    __snake_case    :  List[str]=False	,    __snake_case    :  Optional[int]=1.0	,    **__snake_case    :  List[str]	)    ->	List[Any]:
								return super().encode(
								    self.replace_placeholder_tokens_in_text(
								        __snake_case	,    vector_shuffle=__snake_case	,    prop_tokens_to_load=__snake_case	)	,    *__snake_case	,    **__snake_case	,    )










'''simple docstring'''






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase__:     List[Any]										=    logging.getLogger(__name__)


def 				snake_case_  (      )   ->     int:
							UpperCAmelCase				: int											=	argparse.ArgumentParser(
							    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''						)
							parser.add_argument(
							    '''--dataset_name'''     ,   type=_lowerCAmelCase     ,   default='''wikitext'''     ,   help='''Name of the training. Explore datasets at: hf.co/datasets.'''     ,   )
							parser.add_argument(
							    '''--dataset_config'''     ,   type=_lowerCAmelCase     ,   default='''wikitext-103-raw-v1'''     ,   help='''Configuration name of the dataset.'''						)
							parser.add_argument(
							    '''--tokenizer_name_or_path'''     ,   type=_lowerCAmelCase     ,   default='''sayakpaul/unigram-tokenizer-wikitext'''     ,   help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''     ,   )
							parser.add_argument(
							    '''--shard_size'''     ,   type=_lowerCAmelCase     ,   default=1000     ,   help='''Number of entries to go in a single shard.'''     ,   )
							parser.add_argument('''--split'''     ,   type=_lowerCAmelCase     ,   default='''train'''     ,   choices=['''train''', '''test''', '''validation''']						)
							parser.add_argument(
							    '''--limit'''     ,   default=_lowerCAmelCase     ,   type=_lowerCAmelCase     ,   help='''Limit the number of shards (used for debugging).'''     ,   )
							parser.add_argument(
							    '''--max_length'''     ,   type=_lowerCAmelCase     ,   default=512     ,   help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
							    ''' sequence length that is a multiple of 8.'''     ,   )
							parser.add_argument(
							    '''--output_dir'''     ,   default='''tf-tpu'''     ,   type=_lowerCAmelCase     ,   help='''Output directory where the TFRecord shards will be saved. If the'''
							    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
							    ''' shards will be directly saved to a Google Cloud Storage bucket.'''     ,   )

							UpperCAmelCase				: List[str]											=	parser.parse_args()
							return args


def 				snake_case_  (      _lowerCAmelCase						:		Union[str, Any]						)   ->     Optional[int]:
							def fn(_lowerCAmelCase						:		Tuple						):
														return tokenizer(examples['''text''']						)

							return fn


def 				snake_case_  (      _lowerCAmelCase						:		int						)   ->     Dict:
							UpperCAmelCase				: Optional[Any]											=	[]
							for i in range(len(tokenized_data['''input_ids''']						)						):
														UpperCAmelCase				: Optional[Any]											=	{
														    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]						)						),
														    '''attention_mask''': tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]						)						),
														}
														UpperCAmelCase				: Optional[Any]											=	tf.train.Features(feature=_lowerCAmelCase						)
														UpperCAmelCase				: Union[str, Any]											=	tf.train.Example(features=_lowerCAmelCase						)
														UpperCAmelCase				: Any											=	example.SerializeToString()
														records.append(_lowerCAmelCase						)
							return records


def 				snake_case_  (      _lowerCAmelCase						:		List[str]						)   ->     List[str]:
							UpperCAmelCase				: List[str]											=	datasets.load_dataset(args.dataset_name     ,   args.dataset_config     ,   split=args.split						)

							if args.limit is not None:
														UpperCAmelCase				: List[Any]											=	min(len(_lowerCAmelCase						)     ,   args.limit						)
														UpperCAmelCase				: Dict											=	dataset.select(range(_lowerCAmelCase						)						)
														print(f"""Limiting the dataset to {args.limit} entries."""						)

							UpperCAmelCase				: List[Any]											=	AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir						):
																					os.makedirs(args.output_dir						)
														UpperCAmelCase				: List[Any]											=	os.path.join(args.output_dir     ,   args.split						)
														if not os.path.exists(_lowerCAmelCase						):
																					os.makedirs(_lowerCAmelCase						)
							else:
														UpperCAmelCase				: Tuple											=	os.path.join(args.output_dir     ,   args.split						)

							# Tokenize the whole dataset at once.
							UpperCAmelCase				: Optional[Any]											=	tokenize_function(_lowerCAmelCase						)
							UpperCAmelCase				: int											=	dataset.map(_lowerCAmelCase     ,   batched=_lowerCAmelCase     ,   num_proc=4     ,   remove_columns=['''text''']						)

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(_lowerCAmelCase						:		List[str]						):
														# Concatenate all texts.
														UpperCAmelCase				: Optional[Any]											=	{k: sum(examples[k]     ,   []						) for k in examples.keys()}
														UpperCAmelCase				: Optional[Any]											=	len(concatenated_examples[list(examples.keys()						)[0]]						)
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														UpperCAmelCase				: Any											=	(total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														UpperCAmelCase				: List[str]											=	{
														    k: [t[i : i + args.max_length] for i in range(0     ,   _lowerCAmelCase     ,   args.max_length						)]
														    for k, t in concatenated_examples.items()
														}
														return result

							UpperCAmelCase				: str											=	dataset_tokenized.map(_lowerCAmelCase     ,   batched=_lowerCAmelCase     ,   batch_size=1000     ,   num_proc=4						)

							UpperCAmelCase				: Union[str, Any]											=	0
							UpperCAmelCase				: Dict											=	0
							for shard in range(0     ,   len(_lowerCAmelCase						)     ,   args.shard_size						):
														UpperCAmelCase				: Union[str, Any]											=	grouped_dataset[shard : shard + args.shard_size]
														UpperCAmelCase				: Optional[int]											=	len(dataset_snapshot['''input_ids''']						)
														UpperCAmelCase				: Dict											=	os.path.join(_lowerCAmelCase     ,   f"""dataset-{shard_count}-{records_containing}.tfrecord"""						)
														UpperCAmelCase				: List[Any]											=	get_serialized_examples(_lowerCAmelCase						)

														with tf.io.TFRecordWriter(_lowerCAmelCase						) as out_file:
																					for i in range(len(_lowerCAmelCase						)						):
																												UpperCAmelCase				: Any											=	serialized_examples[i]
																												out_file.write(_lowerCAmelCase						)
																					print('''Wrote file {} containing {} records'''.format(_lowerCAmelCase     ,   _lowerCAmelCase						)						)

														shard_count += 1
														total_records += records_containing

							with open(f"""split-{args.split}-records-count.txt"""     ,   '''w'''						) as f:
														print(f"""Total {args.split} records: {total_records}"""     ,   file=_lowerCAmelCase						)


if __name__ == "__main__":
						UpperCamelCase__:     List[Any]										=    parse_args()
						main(args)





