import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE :	Optional[Any]							     =		logging.getLogger(__name__)



def   __A		(			):



 """simple docstring"""


 __a       =  argparse.ArgumentParser(
     description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."					)
 parser.add_argument(
     "--dataset_name"							,				type=_A							,				default="wikitext"							,				help="Name of the training. Explore datasets at: hf.co/datasets."							,				)
 parser.add_argument(
     "--dataset_config"							,				type=_A							,				default="wikitext-103-raw-v1"							,				help="Configuration name of the dataset."					)
 parser.add_argument(
     "--tokenizer_name_or_path"							,				type=_A							,				default="sayakpaul/unigram-tokenizer-wikitext"							,				help="Tokenizer identifier. Can be a local filepath or a Hub identifier."							,				)
 parser.add_argument(
     "--shard_size"							,				type=_A							,				default=1000							,				help="Number of entries to go in a single shard."							,				)
 parser.add_argument("--split"							,				type=_A							,				default="train"							,				choices=["train", "test", "validation"]					)
 parser.add_argument(
     "--limit"							,				default=_A							,				type=_A							,				help="Limit the number of shards (used for debugging)."							,				)
 parser.add_argument(
     "--max_length"							,				type=_A							,				default=512							,				help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
     " sequence length that is a multiple of 8."							,				)
 parser.add_argument(
     "--output_dir"							,				default="tf-tpu"							,				type=_A							,				help="Output directory where the TFRecord shards will be saved. If the"
     " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
     " shards will be directly saved to a Google Cloud Storage bucket."							,				)

 __a       =  parser.parse_args()
 return args



def   __A		(			_A					):



 """simple docstring"""


 def fn(_A					):
  return tokenizer(examples["text"]					)

 return fn



def   __A		(			_A					):



 """simple docstring"""


 __a       =  []
 for i in range(len(tokenized_data["input_ids"]					)					):
  __a       =  {
      "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]					)					),
      "attention_mask": tf.train.Feature(
          intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]					)					),
  }
  __a       =  tf.train.Features(feature=_A					)
  __a       =  tf.train.Example(features=_A					)
  __a       =  example.SerializeToString()
  records.append(_A					)
 return records



def   __A		(			_A					):



 """simple docstring"""


 __a       =  datasets.load_dataset(args.dataset_name							,				args.dataset_config							,				split=args.split					)

 if args.limit is not None:
  __a       =  min(len(_A					)							,				args.limit					)
  __a       =  dataset.select(range(_A					)					)
  print(f"""Limiting the dataset to {args.limit} entries."""					)

 __a       =  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

 # Handle output directory creation.
 # For serializing into a Google Cloud Storage Bucket, one needs to first
 # create a bucket.
 if "gs" not in args.output_dir:
  if not os.path.exists(args.output_dir					):
   os.makedirs(args.output_dir					)
  __a       =  os.path.join(args.output_dir							,				args.split					)
  if not os.path.exists(_A					):
   os.makedirs(_A					)
 else:
  __a       =  os.path.join(args.output_dir							,				args.split					)

 # Tokenize the whole dataset at once.
 __a       =  tokenize_function(_A					)
 __a       =  dataset.map(_A							,				batched=_A							,				num_proc=4							,				remove_columns=["text"]					)

 # We need to concatenate all our texts together, and then split the result
 # into chunks of a fixed size, which we will call block_size. To do this, we
 # will use the map method again, with the option batched=True. When we use batched=True,
 # the function we pass to map() will be passed multiple inputs at once, allowing us
 # to group them into more or fewer examples than we had in the input.
 # This allows us to create our new fixed-length samples. The advantage of this
 # method is that we don't lose a whole lot of content from the dataset compared to the
 # case where we simply tokenize with a pre-defined max_length.

 def group_texts(_A					):
  # Concatenate all texts.
  __a       =  {k: sum(examples[k]							,				[]					) for k in examples.keys()}
  __a       =  len(concatenated_examples[list(examples.keys()					)[0]]					)
  # We drop the small remainder, though you could add padding instead if the model supports it
  # In this, as in all things, we advise you to follow your heart 🫀
  __a       =  (total_length // args.max_length) * args.max_length
  # Split by chunks of max_len.
  __a       =  {
      k: [t[i : i + args.max_length] for i in range(0							,				_A							,				args.max_length					)]
      for k, t in concatenated_examples.items()
  }
  return result

 __a       =  dataset_tokenized.map(_A							,				batched=_A							,				batch_size=1000							,				num_proc=4					)

 __a       =  0
 __a       =  0
 for shard in range(0							,				len(_A					)							,				args.shard_size					):
  __a       =  grouped_dataset[shard : shard + args.shard_size]
  __a       =  len(dataset_snapshot["input_ids"]					)
  __a       =  os.path.join(_A							,				f"""dataset-{shard_count}-{records_containing}.tfrecord"""					)
  __a       =  get_serialized_examples(_A					)

  with tf.io.TFRecordWriter(_A					) as out_file:
   for i in range(len(_A					)					):
    __a       =  serialized_examples[i]
    out_file.write(_A					)
   print("Wrote file {} containing {} records".format(_A							,				_A					)					)

  shard_count += 1
  total_records += records_containing

 with open(f"""split-{args.split}-records-count.txt"""							,				"w"					) as f:
  print(f"""Total {args.split} records: {total_records}"""							,				file=_A					)


if __name__ == "__main__":
  SCREAMING_SNAKE_CASE :	List[str]							     =		parse_args()
  main(args)



import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
  import ctypes
  import msvcrt  # noqa
  class    A_ (       ctypes.Structure ):
    # _fields is a specific attr expected by ctypes
    _SCREAMING_SNAKE_CASE    				=			[("""size""", ctypes.c_int), ("""visible""", ctypes.c_byte)]



def   __A		(			):



 """simple docstring"""


 if os.name == "nt":
  __a       =  CursorInfo()
  __a       =  ctypes.windll.kernelaa.GetStdHandle(-11					)
  ctypes.windll.kernelaa.GetConsoleCursorInfo(_A							,				ctypes.byref(_A					)					)
  __a       =  False
  ctypes.windll.kernelaa.SetConsoleCursorInfo(_A							,				ctypes.byref(_A					)					)
 elif os.name == "posix":
  sys.stdout.write("\033[?25l"					)
  sys.stdout.flush()



def   __A		(			):



 """simple docstring"""


 if os.name == "nt":
  __a       =  CursorInfo()
  __a       =  ctypes.windll.kernelaa.GetStdHandle(-11					)
  ctypes.windll.kernelaa.GetConsoleCursorInfo(_A							,				ctypes.byref(_A					)					)
  __a       =  True
  ctypes.windll.kernelaa.SetConsoleCursorInfo(_A							,				ctypes.byref(_A					)					)
 elif os.name == "posix":
  sys.stdout.write("\033[?25h"					)
  sys.stdout.flush()






@contextmanager
def   __A		(			):



 """simple docstring"""


 try:
  hide_cursor()
  yield
 finally:
  show_cursor()


