code: def parse_args():
      parser = argparse.ArgumentParser(
          description="""
      Converts a text dataset into the training data format expected by the model.
  
      Adapted from the script create_tfrecords.py in the gpt-neo repo.
  
      - Your text dataset:
          - can be provided as .txt files, or as an archive (.tar.gz, .xz, jsonl.zst).
          - can be one file or multiple
              - using a single large file may use too much memory and crash - if this occurs, split the file up into a few files
          - the model's end-of-text separator is added between the contents of each file
          - if the string '<|endoftext|>' appears inside a file, it is treated as the model's end-of-text separator (not the actual string '<|endoftext|>')
              - this behavior can be disabled with --treat-eot-as-text
  
      This script creates a single .tfrecords file as output
          - Why: the model's data loader ignores "trailing" data (< 1 batch) at the end of a .tfrecords file
              - this causes data loss if you have many .tfrecords files
          - This is probably not appropriate for very large datasets
      """,
          formatter_class=argparse.RawTextHelpFormatter,
      )
      parser.add_argument(
          "input_path",
          type=str,
          help="Path to an input file, or a directory that contains the input files.",
      )
      parser.add_argument(
          "name",
          type=str,
          help="Name of output file will be {name}_{seqnum}.tfrecords, where seqnum is total sequence count",
      )
      parser.add_argument(
          "--output-dir",
          type=str,
          default="",
          help="Output directory (default: current directory)",
      )
  
      cleaning_args = parser.add_argument_group(
          "data cleaning arguments"
      )
  
      cleaning_args.add_argument(
          "--normalize-with-ftfy",
          action="store_true",
          help="Normalize text with ftfy",
      )
      cleaning_args.add_argument(
          "--normalize-with-wikitext-detokenize",
          action="store_true",
          help="Use wikitext detokenizer",
      )
      minu_help = "Exclude repetitive documents made up of < MIN_UNIQUE_TOKENS unique tokens. These can produce large gradients."
      minu_help += " Set <= 0 to disable. If enabled, 200 is a good default value. (Default: 0)"
      cleaning_args.add_argument(
          "--min-unique-tokens", type=int, default=0, help=minu_help
      )
  
      shuffle_pack_args = parser.add_argument_group(
          "data shuffling/packing arguments"
      )
      repack_ep_help = "Repeat the data N_REPACK_EPOCHS times, shuffled differently in each repetition. Recommended for multi-epoch training (set this to your intended number of epochs)."
      shuffle_pack_args.add_argument(
          "--n-repack-epochs", type=int, default=1, help=repack_ep_help
      )
      shuffle_pack_args.add_argument(
          "--seed",
          type=int,
          default=10,
          help="random seed for shuffling data (default: 10)",
      )
      shuffle_pack_args.add_argument(
          "--preserve-data-order",
          default=False,
          action="store_true",
          help="Disables shuffling, so the input and output data have the same order.",
      )
  
      misc_args = parser.add_argument_group("miscellaneous arguments")
      misc_args.add_argument(
          "--verbose",
          default=False,
          action="store_true",
          help="Prints extra information, such as the text removed by --min-unique-tokens",
      )
  
      args = parser.parse_args()
  
      # convert input_path to pathy
      args.input_path = Path(args.input_path)
  
      return args
  
imports: import argparse
  import os
  import re
  import random
  from pathlib import Path
  from typing import List
  import ftfy
  import tensorflow
  from lm_dataformat import Reader
  from transformers import GPT2TokenizerFast
  from tqdm import tqdm
label_desc: Use image node_id 1 for calling a global function with example usage: parse_args() and returns: args