


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__magic_name__            =  logging.getLogger(__name__)





def 	SCREAMING_SNAKE_CASE__    (      ):
    snake_case__						=				argparse.ArgumentParser(
        description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."							)
    parser.add_argument(
        "--dataset_name"    ,		type=__lowerCAmelCase    ,		default="wikitext"    ,		help="Name of the training. Explore datasets at: hf.co/datasets."    ,		)
    parser.add_argument(
        "--dataset_config"    ,		type=__lowerCAmelCase    ,		default="wikitext-103-raw-v1"    ,		help="Configuration name of the dataset."							)
    parser.add_argument(
        "--tokenizer_name_or_path"    ,		type=__lowerCAmelCase    ,		default="sayakpaul/unigram-tokenizer-wikitext"    ,		help="Tokenizer identifier. Can be a local filepath or a Hub identifier."    ,		)
    parser.add_argument(
        "--shard_size"    ,		type=__lowerCAmelCase    ,		default=1_000    ,		help="Number of entries to go in a single shard."    ,		)
    parser.add_argument("--split"    ,		type=__lowerCAmelCase    ,		default="train"    ,		choices=["train", "test", "validation"]							)
    parser.add_argument(
        "--limit"    ,		default=__lowerCAmelCase    ,		type=__lowerCAmelCase    ,		help="Limit the number of shards (used for debugging)."    ,		)
    parser.add_argument(
        "--max_length"    ,		type=__lowerCAmelCase    ,		default=512    ,		help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
        " sequence length that is a multiple of 8."    ,		)
    parser.add_argument(
        "--output_dir"    ,		default="tf-tpu"    ,		type=__lowerCAmelCase    ,		help="Output directory where the TFRecord shards will be saved. If the"
        " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
        " shards will be directly saved to a Google Cloud Storage bucket."    ,		)

    snake_case__						=				parser.parse_args()
    return args





def 	SCREAMING_SNAKE_CASE__    (      __lowerCAmelCase							):
    def fn(__lowerCAmelCase							):
        return tokenizer(examples["text"]							)

    return fn





def 	SCREAMING_SNAKE_CASE__    (      __lowerCAmelCase							):
    snake_case__						=				[]
    for i in range(len(tokenized_data["input_ids"]							)							):
        snake_case__						=				{
            "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]							)							),
            "attention_mask": tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]							)							),
        }
        snake_case__						=				tf.train.Features(feature=__lowerCAmelCase							)
        snake_case__						=				tf.train.Example(features=__lowerCAmelCase							)
        snake_case__						=				example.SerializeToString()
        records.append(__lowerCAmelCase							)
    return records





def 	SCREAMING_SNAKE_CASE__    (      __lowerCAmelCase							):
    snake_case__						=				datasets.load_dataset(args.dataset_name    ,		args.dataset_config    ,		split=args.split							)

    if args.limit is not None:
        snake_case__						=				min(len(__lowerCAmelCase							)    ,		args.limit							)
        snake_case__						=				dataset.select(range(__lowerCAmelCase							)							)
        print(F"""Limiting the dataset to {args.limit} entries."""							)

    snake_case__						=				AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir							):
            os.makedirs(args.output_dir							)
        snake_case__						=				os.path.join(args.output_dir    ,		args.split							)
        if not os.path.exists(__lowerCAmelCase							):
            os.makedirs(__lowerCAmelCase							)
    else:
        snake_case__						=				os.path.join(args.output_dir    ,		args.split							)

    # Tokenize the whole dataset at once.
    snake_case__						=				tokenize_function(__lowerCAmelCase							)
    snake_case__						=				dataset.map(__lowerCAmelCase    ,		batched=__lowerCAmelCase    ,		num_proc=4    ,		remove_columns=["text"]							)

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(__lowerCAmelCase							):
        # Concatenate all texts.
        snake_case__						=				{k: sum(examples[k]    ,		[]							) for k in examples.keys()}
        snake_case__						=				len(concatenated_examples[list(examples.keys()							)[0]]							)
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        snake_case__						=				(total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        snake_case__						=				{
            k: [t[i : i + args.max_length] for i in range(0    ,		__lowerCAmelCase    ,		args.max_length							)]
            for k, t in concatenated_examples.items()
        }
        return result

    snake_case__						=				dataset_tokenized.map(__lowerCAmelCase    ,		batched=__lowerCAmelCase    ,		batch_size=1_000    ,		num_proc=4							)

    snake_case__						=				0
    snake_case__						=				0
    for shard in range(0    ,		len(__lowerCAmelCase							)    ,		args.shard_size							):
        snake_case__						=				grouped_dataset[shard : shard + args.shard_size]
        snake_case__						=				len(dataset_snapshot["input_ids"]							)
        snake_case__						=				os.path.join(__lowerCAmelCase    ,		F"""dataset-{shard_count}-{records_containing}.tfrecord"""							)
        snake_case__						=				get_serialized_examples(__lowerCAmelCase							)

        with tf.io.TFRecordWriter(__lowerCAmelCase							) as out_file:
            for i in range(len(__lowerCAmelCase							)							):
                snake_case__						=				serialized_examples[i]
                out_file.write(__lowerCAmelCase							)
            print("Wrote file {} containing {} records".format(__lowerCAmelCase    ,		__lowerCAmelCase							)							)

        shard_count += 1
        total_records += records_containing

    with open(F"""split-{args.split}-records-count.txt"""    ,		"w"							) as f:
        print(F"""Total {args.split} records: {total_records}"""    ,		file=__lowerCAmelCase							)


if __name__ == "__main__":
     __magic_name__            =  parse_args()
     main(args)





import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow




class 					_SCREAMING_SNAKE_CASE  :
      def A_			(			self		,	lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase ):
          return None





class 					_SCREAMING_SNAKE_CASE  :
      def A_			(			self		,	lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase ):
          return None





class 					_SCREAMING_SNAKE_CASE  (      unittest.TestCase ):
      _A						:		int    	=     [
          # (model_name, model_kwargs)
          ('bert-base-cased', {}),
          ('gpt2', {'use_cache': False}),  # We don't support exporting GPT2 past keys anymore
      ]
      @require_tf
      @slow
      def A_			(			self ):
          for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
              self._test_export(lowerCamelCase		,	"tf"		,	12		,	**lowerCamelCase )
      @require_torch
      @slow
      def A_			(			self ):
          for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
              self._test_export(lowerCamelCase		,	"pt"		,	12		,	**lowerCamelCase )
      @require_torch
      @slow
      def A_			(			self ):
          from transformers import BertModel

          snake_case__						=				["[UNK]", "[SEP]", "[CLS]", "[PAD]", "[MASK]", "some", "other", "words"]
          with NamedTemporaryFile(mode="w+t" ) as vocab_file:
              vocab_file.write("\n".join(lowerCamelCase ) )
              vocab_file.flush()
              snake_case__						=				BertTokenizerFast(vocab_file.name )

          with TemporaryDirectory() as bert_save_dir:
              snake_case__						=				BertModel(BertConfig(vocab_size=len(lowerCamelCase ) ) )
              model.save_pretrained(lowerCamelCase )
              self._test_export(lowerCamelCase		,	"pt"		,	12		,	lowerCamelCase )
      @require_tf
      @slow
      def A_			(			self ):
          for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
              snake_case__						=				self._test_export(lowerCamelCase		,	"tf"		,	12		,	**lowerCamelCase )
              snake_case__						=				quantize(Path(lowerCamelCase ) )

              # Ensure the actual quantized model is not bigger than the original one
              if quantized_path.stat().st_size >= Path(lowerCamelCase ).stat().st_size:
                  self.fail("Quantized model is bigger than initial ONNX model" )
      @require_torch
      @slow
      def A_			(			self ):
          for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
              snake_case__						=				self._test_export(lowerCamelCase		,	"pt"		,	12		,	**lowerCamelCase )
              snake_case__						=				quantize(lowerCamelCase )

              # Ensure the actual quantized model is not bigger than the original one
              if quantized_path.stat().st_size >= Path(lowerCamelCase ).stat().st_size:
                  self.fail("Quantized model is bigger than initial ONNX model" )
      def A_			(			self		,	lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase=None		,	**lowerCamelCase ):
          try:
              # Compute path
              with TemporaryDirectory() as tempdir:
                  snake_case__						=				Path(lowerCamelCase ).joinpath("model.onnx" )

              # Remove folder if exists
              if path.parent.exists():
                  path.parent.rmdir()

              # Export
              convert(lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase		,	**lowerCamelCase )

              return path
          except Exception as e:
              self.fail(lowerCamelCase )
      @require_torch
      @require_tokenizers
      @slow
      def A_			(			self ):
          from transformers import BertModel

          snake_case__						=				BertModel(BertConfig.from_pretrained("lysandre/tiny-bert-random" ) )
          snake_case__						=				BertTokenizerFast.from_pretrained("lysandre/tiny-bert-random" )
          self._test_infer_dynamic_axis(lowerCamelCase		,	lowerCamelCase		,	"pt" )
      @require_tf
      @require_tokenizers
      @slow
      def A_			(			self ):
          from transformers import TFBertModel

          snake_case__						=				TFBertModel(BertConfig.from_pretrained("lysandre/tiny-bert-random" ) )
          snake_case__						=				BertTokenizerFast.from_pretrained("lysandre/tiny-bert-random" )
          self._test_infer_dynamic_axis(lowerCamelCase		,	lowerCamelCase		,	"tf" )
      def A_			(			self		,	lowerCamelCase		,	lowerCamelCase		,	lowerCamelCase ):
          snake_case__						=				FeatureExtractionPipeline(lowerCamelCase		,	lowerCamelCase )

          snake_case__						=				["input_ids", "token_type_ids", "attention_mask", "output_0", "output_1"]
          snake_case__						,					snake_case__						,					snake_case__						,					snake_case__						=				infer_shapes(lowerCamelCase		,	lowerCamelCase )

          # Assert all variables are present
          self.assertEqual(len(lowerCamelCase )		,	len(lowerCamelCase ) )
          self.assertTrue(all(var_name in shapes for var_name in variable_names ) )
          self.assertSequenceEqual(variable_names[:3]		,	lowerCamelCase )
          self.assertSequenceEqual(variable_names[3:]		,	lowerCamelCase )

          # Assert inputs are {0: batch, 1: sequence}
          for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
              self.assertDictEqual(shapes[var_name]		,	{0: "batch", 1: "sequence"} )

          # Assert outputs are {0: batch, 1: sequence} and {0: batch}
          self.assertDictEqual(shapes["output_0"]		,	{0: "batch", 1: "sequence"} )
          self.assertDictEqual(shapes["output_1"]		,	{0: "batch"} )
      def A_			(			self ):
          snake_case__						=				["input_ids", "attention_mask", "token_type_ids"]
          snake_case__						=				{"input_ids": [1, 2, 3, 4], "attention_mask": [0, 0, 0, 0], "token_type_ids": [1, 1, 1, 1]}
          snake_case__						,					snake_case__						=				ensure_valid_input(FuncContiguousArgs()		,	lowerCamelCase		,	lowerCamelCase )

          # Should have exactly the same number of args (all are valid)
          self.assertEqual(len(lowerCamelCase )		,	3 )

          # Should have exactly the same input names
          self.assertEqual(set(lowerCamelCase )		,	set(lowerCamelCase ) )

          # Parameter should be reordered according to their respective place in the function:
          # (input_ids, token_type_ids, attention_mask)
          self.assertEqual(lowerCamelCase		,	(tokens["input_ids"], tokens["token_type_ids"], tokens["attention_mask"]) )

          # Generated args are interleaved with another args (for instance parameter "past" in GPT2)
          snake_case__						,					snake_case__						=				ensure_valid_input(FuncNonContiguousArgs()		,	lowerCamelCase		,	lowerCamelCase )

          # Should have exactly the one arg (all before the one not provided "some_other_args")
          self.assertEqual(len(lowerCamelCase )		,	1 )
          self.assertEqual(len(lowerCamelCase )		,	1 )

          # Should have only "input_ids"
          self.assertEqual(inputs_args[0]		,	tokens["input_ids"] )
          self.assertEqual(ordered_input_names[0]		,	"input_ids" )



      def A_			(			self ):
          snake_case__						=				generate_identified_filename(Path("/home/something/my_fake_model.onnx" )		,	"-test" )
          self.assertEqual("/home/something/my_fake_model-test.onnx"		,	generated.as_posix() )



