code: def load_dataset(enc, path, combine):
      paths = []
      if os.path.isfile(path):
          # Simple file
          paths.append(path)
      elif os.path.isdir(path):
          # Directory
          for dirpath, _, fnames in os.walk(path):
              for fname in fnames:
                  paths.append(os.path.join(dirpath, fname))
      else:
          # Assume glob
          paths = glob.glob(path)
  
      token_chunks = []
      raw_text = ""
      for path in tqdm.tqdm(paths):
          if path.endswith(".npz"):
              # Pre-encoded
              with np.load(path) as npz:
                  for item in npz.files:
                      token_chunks.append(npz[item])
          elif path.endswith(".csv"):
              start_token = "<|startoftext|>"
              end_token = "<|endoftext|>"
              with open(
                  path, "r", encoding="utf8", errors="ignore"
              ) as fp:
                  fp.readline()  # skip header
                  reader = csv.reader(fp)
                  for row in reader:
                      raw_text += (
                          start_token + row[0] + end_token + "\n"
                      )
          else:
              # Plain text
              with open(
                  path, "r", encoding="utf8", errors="ignore"
              ) as fp:
                  raw_text += fp.read()
              if len(raw_text) >= combine:
                  tokens = np.stack(enc.encode(raw_text))
                  token_chunks.append(tokens)
                  raw_text = ""
              else:
                  raw_text += "<|endoftext|>"
      if raw_text:
          tokens = np.stack(enc.encode(raw_text))
          token_chunks.append(tokens)
      return token_chunks
  
file_path: gpt-2-simple/gpt_2_simple/src/load_dataset.py
imports: import glob
  import numpy
  import os
  import random
  import tensorflow
  import tqdm
  import csv
label_desc: Use image node_id 1 for calling a global function with example usage: load_dataset(enc, path, combine) and returns: token_chunks