



'''simple docstring'''






import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


UpperCamelCase__:     Tuple										=    logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
						import torch_xla.core.xla_model as xm
						import torch_xla.debug.metrics as met

class        SCREAMING_SNAKE_CASE(							A__    ):







	"""simple docstring"""






	def __init__(		self    :  str	,    *__snake_case    :  Union[str, Any]	,    __snake_case    :  List[Any]=None	,    __snake_case    :  Union[str, Any]=None	,    __snake_case    :  Optional[Any]=None	,    **__snake_case    :  str	)    ->	str:
								super().__init__(*__snake_case	,    **__snake_case	)
								UpperCAmelCase				: Optional[int]											=	eval_examples
								UpperCAmelCase				: Dict											=	post_process_function
								UpperCAmelCase				: Union[str, Any]											=	quant_trainer_args
								UpperCAmelCase				: Union[str, Any]											=	128  # default number of calibration samples







	def   A			(		self    :  Tuple	,    __snake_case    :  List[Any]=None	)    ->	Tuple:
								if calib_dataset is None and self.calib_dataset is None:
															raise ValueError('''Trainer: calibration requires an calib_dataset.'''	)
								UpperCAmelCase				: Dict											=	calib_dataset if calib_dataset is not None else self.calib_dataset

								UpperCAmelCase				: Optional[Any]											=	self._remove_unused_columns(__snake_case	,    description='''Calibration'''	)

								return DataLoader(
								    __snake_case	,    batch_size=self.args.eval_batch_size	,    collate_fn=self.data_collator	,    drop_last=self.args.dataloader_drop_last	,    num_workers=self.args.dataloader_num_workers	,    pin_memory=self.args.dataloader_pin_memory	,    shuffle=__snake_case	,    )







	def   A			(		self    :  Optional[int]	,    __snake_case    :  Tuple=None	)    ->	Dict:
								UpperCAmelCase				: Optional[Any]											=	self.train_dataset if calib_dataset is None else calib_dataset
								UpperCAmelCase				: int											=	self.get_calib_dataloader(__snake_case	)

								UpperCAmelCase				: Any											=	self.model
								quant_trainer.configure_model(__snake_case	,    self.quant_trainer_args	,    calib=__snake_case	)
								model.eval()
								quant_trainer.enable_calibration(__snake_case	)

								logger.info('''***** Running calibration *****'''	)
								logger.info(F"""  Num examples = {self.calib_num}"""	)
								logger.info(F"""  Batch size = {calib_dataloader.batch_size}"""	)

								for step, inputs in enumerate(__snake_case	):
															# Prediction step
															UpperCAmelCase							,	UpperCAmelCase							,	UpperCAmelCase				: int											=	self.prediction_step(__snake_case	,    __snake_case	,    prediction_loss_only=__snake_case	)
															if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
																						break

								quant_trainer.finish_calibration(__snake_case	,    self.quant_trainer_args	)
								UpperCAmelCase				: Optional[int]											=	model







	def   A			(		self    :  Optional[Any]	,    __snake_case    :  str=None	,    __snake_case    :  str=None	,    __snake_case    :  str=None	,    __snake_case    :  str = "eval"	)    ->	int:
								UpperCAmelCase				: Any											=	self.eval_dataset if eval_dataset is None else eval_dataset
								UpperCAmelCase				: Optional[int]											=	self.get_eval_dataloader(__snake_case	)
								UpperCAmelCase				: Union[str, Any]											=	self.eval_examples if eval_examples is None else eval_examples

								# Temporarily disable metric computation, we will do it in the loop here.
								UpperCAmelCase				: int											=	self.compute_metrics
								UpperCAmelCase				: Any											=	None
								UpperCAmelCase				: Optional[int]											=	self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
								try:
															UpperCAmelCase				: str											=	eval_loop(
															    __snake_case	,    description='''Evaluation'''	,    prediction_loss_only=True if compute_metrics is None else None	,    ignore_keys=__snake_case	,    )
								finally:
															UpperCAmelCase				: Dict											=	compute_metrics

								if self.post_process_function is not None and self.compute_metrics is not None:
															UpperCAmelCase				: List[Any]											=	self.post_process_function(__snake_case	,    __snake_case	,    output.predictions	)
															UpperCAmelCase				: int											=	self.compute_metrics(__snake_case	)

															# Prefix all keys with metric_key_prefix + '_'
															for key in list(metrics.keys()	):
																						if not key.startswith(F"""{metric_key_prefix}_"""	):
																													UpperCAmelCase				: List[Any]											=	metrics.pop(__snake_case	)

															self.log(__snake_case	)
								else:
															UpperCAmelCase				: int											=	{}

								if self.args.tpu_metrics_debug or self.args.debug:
															# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
															xm.master_print(met.metrics_report()	)

								UpperCAmelCase				: List[Any]											=	self.callback_handler.on_evaluate(self.args	,    self.state	,    self.control	,    __snake_case	)
								return metrics







	def   A			(		self    :  Dict	,    __snake_case    :  List[str]	,    __snake_case    :  Optional[int]	,    __snake_case    :  Tuple=None	,    __snake_case    :  str = "test"	)    ->	Tuple:
								UpperCAmelCase				: List[str]											=	self.get_test_dataloader(__snake_case	)

								# Temporarily disable metric computation, we will do it in the loop here.
								UpperCAmelCase				: str											=	self.compute_metrics
								UpperCAmelCase				: Any											=	None
								UpperCAmelCase				: Tuple											=	self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
								try:
															UpperCAmelCase				: Dict											=	eval_loop(
															    __snake_case	,    description='''Prediction'''	,    prediction_loss_only=True if compute_metrics is None else None	,    ignore_keys=__snake_case	,    )
								finally:
															UpperCAmelCase				: List[str]											=	compute_metrics

								if self.post_process_function is None or self.compute_metrics is None:
															return output

								UpperCAmelCase				: int											=	self.post_process_function(__snake_case	,    __snake_case	,    output.predictions	,    '''predict'''	)
								UpperCAmelCase				: Dict											=	self.compute_metrics(__snake_case	)

								# Prefix all keys with metric_key_prefix + '_'
								for key in list(metrics.keys()	):
															if not key.startswith(F"""{metric_key_prefix}_"""	):
																						UpperCAmelCase				: int											=	metrics.pop(__snake_case	)

								return PredictionOutput(predictions=predictions.predictions	,    label_ids=predictions.label_ids	,    metrics=__snake_case	)







	def   A			(		self    :  str	,    __snake_case    :  Optional[int]="./"	)    ->	Optional[Any]:
								UpperCAmelCase				: Optional[Any]											=	self.eval_dataset
								UpperCAmelCase				: List[Any]											=	self.get_eval_dataloader(__snake_case	)

								UpperCAmelCase				: Optional[Any]											=	next(iter(__snake_case	)	)

								# saving device - to make it consistent
								UpperCAmelCase				: Union[str, Any]											=	torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''	)

								# convert to tuple
								UpperCAmelCase				: str											=	tuple(v.to(__snake_case	) for k, v in batch.items()	)

								logger.info('''Converting model to be onnx compatible'''	)
								from pytorch_quantization.nn import TensorQuantizer

								UpperCAmelCase				: Tuple											=	True

								UpperCAmelCase				: Dict											=	self.model.to(__snake_case	)

								model.eval()
								model.float()

								UpperCAmelCase				: List[Any]											=	model.module if hasattr(__snake_case	,    '''module'''	) else model
								quant_trainer.configure_model(__snake_case	,    self.quant_trainer_args	)

								UpperCAmelCase				: List[Any]											=	os.path.join(__snake_case	,    '''model.onnx'''	)
								logger.info(F"""exporting model to {output_model_file}"""	)

								UpperCAmelCase				: List[str]											=	{0: '''batch_size''', 1: '''seq_len'''}

								torch.onnx.export(
								    __snake_case	,    __snake_case	,    __snake_case	,    export_params=__snake_case	,    opset_version=13	,    do_constant_folding=__snake_case	,    input_names=['''input_ids''', '''attention_mask''', '''token_type_ids''']	,    output_names=['''output_start_logits''', '''output_end_logits''']	,    dynamic_axes={
								        '''input_ids''': axes,
								        '''attention_mask''': axes,
								        '''token_type_ids''': axes,
								        '''output_start_logits''': axes,
								        '''output_end_logits''': axes,
								    }	,    verbose=__snake_case	,    )
								logger.info('''onnx export finished'''	)










'''simple docstring'''






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase__:     List[Any]										=    logging.getLogger(__name__)


def 				snake_case_  (      )   ->     int:
							UpperCAmelCase				: int											=	argparse.ArgumentParser(
							    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''						)
							parser.add_argument(
							    '''--dataset_name'''     ,   type=_lowerCAmelCase     ,   default='''wikitext'''     ,   help='''Name of the training. Explore datasets at: hf.co/datasets.'''     ,   )
							parser.add_argument(
							    '''--dataset_config'''     ,   type=_lowerCAmelCase     ,   default='''wikitext-103-raw-v1'''     ,   help='''Configuration name of the dataset.'''						)
							parser.add_argument(
							    '''--tokenizer_name_or_path'''     ,   type=_lowerCAmelCase     ,   default='''sayakpaul/unigram-tokenizer-wikitext'''     ,   help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''     ,   )
							parser.add_argument(
							    '''--shard_size'''     ,   type=_lowerCAmelCase     ,   default=1000     ,   help='''Number of entries to go in a single shard.'''     ,   )
							parser.add_argument('''--split'''     ,   type=_lowerCAmelCase     ,   default='''train'''     ,   choices=['''train''', '''test''', '''validation''']						)
							parser.add_argument(
							    '''--limit'''     ,   default=_lowerCAmelCase     ,   type=_lowerCAmelCase     ,   help='''Limit the number of shards (used for debugging).'''     ,   )
							parser.add_argument(
							    '''--max_length'''     ,   type=_lowerCAmelCase     ,   default=512     ,   help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
							    ''' sequence length that is a multiple of 8.'''     ,   )
							parser.add_argument(
							    '''--output_dir'''     ,   default='''tf-tpu'''     ,   type=_lowerCAmelCase     ,   help='''Output directory where the TFRecord shards will be saved. If the'''
							    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
							    ''' shards will be directly saved to a Google Cloud Storage bucket.'''     ,   )

							UpperCAmelCase				: List[str]											=	parser.parse_args()
							return args


def 				snake_case_  (      _lowerCAmelCase						:		Union[str, Any]						)   ->     Optional[int]:
							def fn(_lowerCAmelCase						:		Tuple						):
														return tokenizer(examples['''text''']						)

							return fn


def 				snake_case_  (      _lowerCAmelCase						:		int						)   ->     Dict:
							UpperCAmelCase				: Optional[Any]											=	[]
							for i in range(len(tokenized_data['''input_ids''']						)						):
														UpperCAmelCase				: Optional[Any]											=	{
														    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]						)						),
														    '''attention_mask''': tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]						)						),
														}
														UpperCAmelCase				: Optional[Any]											=	tf.train.Features(feature=_lowerCAmelCase						)
														UpperCAmelCase				: Union[str, Any]											=	tf.train.Example(features=_lowerCAmelCase						)
														UpperCAmelCase				: Any											=	example.SerializeToString()
														records.append(_lowerCAmelCase						)
							return records


def 				snake_case_  (      _lowerCAmelCase						:		List[str]						)   ->     List[str]:
							UpperCAmelCase				: List[str]											=	datasets.load_dataset(args.dataset_name     ,   args.dataset_config     ,   split=args.split						)

							if args.limit is not None:
														UpperCAmelCase				: List[Any]											=	min(len(_lowerCAmelCase						)     ,   args.limit						)
														UpperCAmelCase				: Dict											=	dataset.select(range(_lowerCAmelCase						)						)
														print(f"""Limiting the dataset to {args.limit} entries."""						)

							UpperCAmelCase				: List[Any]											=	AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir						):
																					os.makedirs(args.output_dir						)
														UpperCAmelCase				: List[Any]											=	os.path.join(args.output_dir     ,   args.split						)
														if not os.path.exists(_lowerCAmelCase						):
																					os.makedirs(_lowerCAmelCase						)
							else:
														UpperCAmelCase				: Tuple											=	os.path.join(args.output_dir     ,   args.split						)

							# Tokenize the whole dataset at once.
							UpperCAmelCase				: Optional[Any]											=	tokenize_function(_lowerCAmelCase						)
							UpperCAmelCase				: int											=	dataset.map(_lowerCAmelCase     ,   batched=_lowerCAmelCase     ,   num_proc=4     ,   remove_columns=['''text''']						)

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(_lowerCAmelCase						:		List[str]						):
														# Concatenate all texts.
														UpperCAmelCase				: Optional[Any]											=	{k: sum(examples[k]     ,   []						) for k in examples.keys()}
														UpperCAmelCase				: Optional[Any]											=	len(concatenated_examples[list(examples.keys()						)[0]]						)
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														UpperCAmelCase				: Any											=	(total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														UpperCAmelCase				: List[str]											=	{
														    k: [t[i : i + args.max_length] for i in range(0     ,   _lowerCAmelCase     ,   args.max_length						)]
														    for k, t in concatenated_examples.items()
														}
														return result

							UpperCAmelCase				: str											=	dataset_tokenized.map(_lowerCAmelCase     ,   batched=_lowerCAmelCase     ,   batch_size=1000     ,   num_proc=4						)

							UpperCAmelCase				: Union[str, Any]											=	0
							UpperCAmelCase				: Dict											=	0
							for shard in range(0     ,   len(_lowerCAmelCase						)     ,   args.shard_size						):
														UpperCAmelCase				: Union[str, Any]											=	grouped_dataset[shard : shard + args.shard_size]
														UpperCAmelCase				: Optional[int]											=	len(dataset_snapshot['''input_ids''']						)
														UpperCAmelCase				: Dict											=	os.path.join(_lowerCAmelCase     ,   f"""dataset-{shard_count}-{records_containing}.tfrecord"""						)
														UpperCAmelCase				: List[Any]											=	get_serialized_examples(_lowerCAmelCase						)

														with tf.io.TFRecordWriter(_lowerCAmelCase						) as out_file:
																					for i in range(len(_lowerCAmelCase						)						):
																												UpperCAmelCase				: Any											=	serialized_examples[i]
																												out_file.write(_lowerCAmelCase						)
																					print('''Wrote file {} containing {} records'''.format(_lowerCAmelCase     ,   _lowerCAmelCase						)						)

														shard_count += 1
														total_records += records_containing

							with open(f"""split-{args.split}-records-count.txt"""     ,   '''w'''						) as f:
														print(f"""Total {args.split} records: {total_records}"""     ,   file=_lowerCAmelCase						)


if __name__ == "__main__":
						UpperCamelCase__:     List[Any]										=    parse_args()
						main(args)





