"""simple docstring"""



import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


SCREAMING_SNAKE_CASE__:Any									=			logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
   import torch_xla.core.xla_model as xm
   import torch_xla.debug.metrics as met






class  snake_case__      (       snake_case_       ):


 def __init__( self    ,		*lowerCamelCase    ,		lowerCamelCase=None    ,		lowerCamelCase=None    ,		lowerCamelCase=None    ,		**lowerCamelCase						):
        super().__init__(*lowerCamelCase    ,		**lowerCamelCase						)
        __a       =    eval_examples
        __a       =    post_process_function
        __a       =    quant_trainer_args
        __a       =    128  # default number of calibration samples


 def 						a__   ( self    ,		lowerCamelCase=None						):
        if calib_dataset is None and self.calib_dataset is None:
               raise ValueError("Trainer: calibration requires an calib_dataset."						)
        __a       =    calib_dataset if calib_dataset is not None else self.calib_dataset

        __a       =    self._remove_unused_columns(lowerCamelCase    ,		description="Calibration"						)

        return DataLoader(
            lowerCamelCase    ,		batch_size=self.args.eval_batch_size    ,		collate_fn=self.data_collator    ,		drop_last=self.args.dataloader_drop_last    ,		num_workers=self.args.dataloader_num_workers    ,		pin_memory=self.args.dataloader_pin_memory    ,		shuffle=lowerCamelCase    ,		)


 def 						a__   ( self    ,		lowerCamelCase=None						):
        __a       =    self.train_dataset if calib_dataset is None else calib_dataset
        __a       =    self.get_calib_dataloader(lowerCamelCase						)

        __a       =    self.model
        quant_trainer.configure_model(lowerCamelCase    ,		self.quant_trainer_args    ,		calib=lowerCamelCase						)
        model.eval()
        quant_trainer.enable_calibration(lowerCamelCase						)

        logger.info("***** Running calibration *****"						)
        logger.info(F"  Num examples = {self.calib_num}"						)
        logger.info(F"  Batch size = {calib_dataloader.batch_size}"						)

        for step, inputs in enumerate(lowerCamelCase						):
               # Prediction step
               __a ,   __a ,   __a       =    self.prediction_step(lowerCamelCase    ,		lowerCamelCase    ,		prediction_loss_only=lowerCamelCase						)
               if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
                      break

        quant_trainer.finish_calibration(lowerCamelCase    ,		self.quant_trainer_args						)
        __a       =    model


 def 						a__   ( self    ,		lowerCamelCase=None    ,		lowerCamelCase=None    ,		lowerCamelCase=None    ,		lowerCamelCase = "eval"						):
        __a       =    self.eval_dataset if eval_dataset is None else eval_dataset
        __a       =    self.get_eval_dataloader(lowerCamelCase						)
        __a       =    self.eval_examples if eval_examples is None else eval_examples

        # Temporarily disable metric computation, we will do it in the loop here.
        __a       =    self.compute_metrics
        __a       =    None
        __a       =    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
               __a       =    eval_loop(
                   lowerCamelCase    ,		description="Evaluation"    ,		prediction_loss_only=True if compute_metrics is None else None    ,		ignore_keys=lowerCamelCase    ,		)
        finally:
               __a       =    compute_metrics

        if self.post_process_function is not None and self.compute_metrics is not None:
               __a       =    self.post_process_function(lowerCamelCase    ,		lowerCamelCase    ,		output.predictions						)
               __a       =    self.compute_metrics(lowerCamelCase						)

               # Prefix all keys with metric_key_prefix + '_'
               for key in list(metrics.keys()						):
                      if not key.startswith(F"{metric_key_prefix}_"						):
                             __a       =    metrics.pop(lowerCamelCase						)

               self.log(lowerCamelCase						)
        else:
               __a       =    {}

        if self.args.tpu_metrics_debug or self.args.debug:
               # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
               xm.master_print(met.metrics_report()						)

        __a       =    self.callback_handler.on_evaluate(self.args    ,		self.state    ,		self.control    ,		lowerCamelCase						)
        return metrics


 def 						a__   ( self    ,		lowerCamelCase    ,		lowerCamelCase    ,		lowerCamelCase=None    ,		lowerCamelCase = "test"						):
        __a       =    self.get_test_dataloader(lowerCamelCase						)

        # Temporarily disable metric computation, we will do it in the loop here.
        __a       =    self.compute_metrics
        __a       =    None
        __a       =    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
               __a       =    eval_loop(
                   lowerCamelCase    ,		description="Prediction"    ,		prediction_loss_only=True if compute_metrics is None else None    ,		ignore_keys=lowerCamelCase    ,		)
        finally:
               __a       =    compute_metrics

        if self.post_process_function is None or self.compute_metrics is None:
               return output

        __a       =    self.post_process_function(lowerCamelCase    ,		lowerCamelCase    ,		output.predictions    ,		"predict"						)
        __a       =    self.compute_metrics(lowerCamelCase						)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()						):
               if not key.startswith(F"{metric_key_prefix}_"						):
                      __a       =    metrics.pop(lowerCamelCase						)

        return PredictionOutput(predictions=predictions.predictions    ,		label_ids=predictions.label_ids    ,		metrics=lowerCamelCase						)






 def 						a__   ( self    ,		lowerCamelCase="./"						):
        __a       =    self.eval_dataset
        __a       =    self.get_eval_dataloader(lowerCamelCase						)

        __a       =    next(iter(lowerCamelCase						)						)

        # saving device - to make it consistent
        __a       =    torch.device("cuda" if torch.cuda.is_available() else "cpu"						)

        # convert to tuple
        __a       =    tuple(v.to(lowerCamelCase						) for k, v in batch.items()						)

        logger.info("Converting model to be onnx compatible"						)
        from pytorch_quantization.nn import TensorQuantizer

        __a       =    True

        __a       =    self.model.to(lowerCamelCase						)

        model.eval()
        model.float()

        __a       =    model.module if hasattr(lowerCamelCase    ,		"module"						) else model
        quant_trainer.configure_model(lowerCamelCase    ,		self.quant_trainer_args						)

        __a       =    os.path.join(lowerCamelCase    ,		"model.onnx"						)
        logger.info(F"exporting model to {output_model_file}"						)

        __a       =    {0: "batch_size", 1: "seq_len"}

        torch.onnx.export(
            lowerCamelCase    ,		lowerCamelCase    ,		lowerCamelCase    ,		export_params=lowerCamelCase    ,		opset_version=13    ,		do_constant_folding=lowerCamelCase    ,		input_names=["input_ids", "attention_mask", "token_type_ids"]    ,		output_names=["output_start_logits", "output_end_logits"]    ,		dynamic_axes={
                "input_ids": axes,
                "attention_mask": axes,
                "token_type_ids": axes,
                "output_start_logits": axes,
                "output_end_logits": axes,
            }    ,		verbose=lowerCamelCase    ,		)
        logger.info("onnx export finished"						)

"""simple docstring"""



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE__:List[str]									=			logging.getLogger(__name__)



def 	_lowerCamelCase(	):
       __a       =    argparse.ArgumentParser(
           description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."				)
       parser.add_argument(
           "--dataset_name"   ,    type=a   ,    default="wikitext"   ,    help="Name of the training. Explore datasets at: hf.co/datasets."   ,    )
       parser.add_argument(
           "--dataset_config"   ,    type=a   ,    default="wikitext-103-raw-v1"   ,    help="Configuration name of the dataset."				)
       parser.add_argument(
           "--tokenizer_name_or_path"   ,    type=a   ,    default="sayakpaul/unigram-tokenizer-wikitext"   ,    help="Tokenizer identifier. Can be a local filepath or a Hub identifier."   ,    )
       parser.add_argument(
           "--shard_size"   ,    type=a   ,    default=1_0_0_0   ,    help="Number of entries to go in a single shard."   ,    )
       parser.add_argument("--split"   ,    type=a   ,    default="train"   ,    choices=["train", "test", "validation"]				)
       parser.add_argument(
           "--limit"   ,    default=a   ,    type=a   ,    help="Limit the number of shards (used for debugging)."   ,    )
       parser.add_argument(
           "--max_length"   ,    type=a   ,    default=5_1_2   ,    help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
           " sequence length that is a multiple of 8."   ,    )
       parser.add_argument(
           "--output_dir"   ,    default="tf-tpu"   ,    type=a   ,    help="Output directory where the TFRecord shards will be saved. If the"
           " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
           " shards will be directly saved to a Google Cloud Storage bucket."   ,    )

       __a       =    parser.parse_args()
       return args



def 	_lowerCamelCase(	a				):
       def fn(a				):
              return tokenizer(examples["text"]				)

       return fn



def 	_lowerCamelCase(	a				):
       __a       =    []
       for i in range(len(tokenized_data["input_ids"]				)				):
              __a       =    {
                  "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]				)				),
                  "attention_mask": tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]				)				),
              }
              __a       =    tf.train.Features(feature=a				)
              __a       =    tf.train.Example(features=a				)
              __a       =    example.SerializeToString()
              records.append(a				)
       return records



def 	_lowerCamelCase(	a				):
       __a       =    datasets.load_dataset(args.dataset_name   ,    args.dataset_config   ,    split=args.split				)

       if args.limit is not None:
              __a       =    min(len(a				)   ,    args.limit				)
              __a       =    dataset.select(range(a				)				)
              print(F"Limiting the dataset to {args.limit} entries."				)

       __a       =    AutoTokenizer.from_pretrained(args.tokenizer_name_or_path				)

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir				):
                     os.makedirs(args.output_dir				)
              __a       =    os.path.join(args.output_dir   ,    args.split				)
              if not os.path.exists(a				):
                     os.makedirs(a				)
       else:
              __a       =    os.path.join(args.output_dir   ,    args.split				)

       # Tokenize the whole dataset at once.
       __a       =    tokenize_function(a				)
       __a       =    dataset.map(a   ,    batched=a   ,    num_proc=4   ,    remove_columns=["text"]				)

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(a				):
              # Concatenate all texts.
              __a       =    {k: sum(examples[k]   ,    []				) for k in examples.keys()}
              __a       =    len(concatenated_examples[list(examples.keys()				)[0]]				)
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              __a       =    (total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              __a       =    {
                  k: [t[i : i + args.max_length] for i in range(0   ,    a   ,    args.max_length				)]
                  for k, t in concatenated_examples.items()
              }
              return result

       __a       =    dataset_tokenized.map(a   ,    batched=a   ,    batch_size=1_0_0_0   ,    num_proc=4				)

       __a       =    0
       __a       =    0
       for shard in range(0   ,    len(a				)   ,    args.shard_size				):
              __a       =    grouped_dataset[shard : shard + args.shard_size]
              __a       =    len(dataset_snapshot["input_ids"]				)
              __a       =    os.path.join(a   ,    F"dataset-{shard_count}-{records_containing}.tfrecord"				)
              __a       =    get_serialized_examples(a				)

              with tf.io.TFRecordWriter(a				) as out_file:
                     for i in range(len(a				)				):
                            __a       =    serialized_examples[i]
                            out_file.write(a				)
                     print("Wrote file {} containing {} records".format(a   ,    a				)				)

              shard_count += 1
              total_records += records_containing

       with open(F"split-{args.split}-records-count.txt"   ,    "w"				) as f:
              print(F"Total {args.split} records: {total_records}"   ,    file=a				)


if __name__ == "__main__":
   SCREAMING_SNAKE_CASE__:List[Any]									=			parse_args()
   main(args)
