code: def generate(
      tables,
      SCALE_FACTOR,
      folder,
      upload_to_s3,
      validate_dataset,
      num_processes,
  ):
      if upload_to_s3:
          assert (
              "AWS_ACCESS_KEY_ID" in os.environ
          ), "AWS credentials not set"
      else:
          shutil.rmtree(f"{folder}", ignore_errors=True)
          os.mkdir(f"{folder}")
  
      if validate_dataset:
          fs = None
          if upload_to_s3:
              import s3fs
  
              fs = s3fs.S3FileSystem()
  
      for table_name, (
          table_short,
          num_pieces,
          load_func,
      ) in tables.items():
          if upload_to_s3:
              output_prefix = f"s3://{folder}/{table_name}.pq"
          else:
              output_prefix = f"{folder}/{table_name}.pq"
              if num_pieces > 1:
                  os.mkdir(output_prefix)
  
          if num_pieces > 1:
              with Pool(num_processes) as pool:
                  pool.map(
                      to_parquet,
                      [
                          (
                              SCALE_FACTOR,
                              table_name,
                              table_short,
                              load_func,
                              p,
                              num_pieces,
                              output_prefix,
                          )
                          for p in range(num_pieces)
                      ],
                  )
          else:
              dbgen_fname = f"{tpch_dbgen_location}/{table_name}.tbl"
              # generate the whole table for the given scale factor with dbgen
              remove_file_if_exists(dbgen_fname)
              cmd = f"./dbgen -f -s {SCALE_FACTOR} -T {table_short}"
              subprocess.run(
                  cmd.split(), check=True, cwd=tpch_dbgen_location
              )
              # load csv file into pandas dataframe
              df = load_func(dbgen_fname)
              # csv file no longer needed, remove
              os.remove(dbgen_fname)
              # write dataframe to parquet
              df.to_parquet(output_prefix)
  
          if validate_dataset:
              # make sure dataset is correct
              ds = pq.ParquetDataset(output_prefix, filesystem=fs)
              assert len(ds.pieces) == num_pieces
  
file_path: mars/benchmarks/tpch/gen_data.py
imports: import os
  import argparse
  import shutil
  import subprocess
  from multiprocessing import Pool, set_start_method
  import pyarrow.parquet
  from loader import load_customer, load_lineitem_with_date, load_nation, load_orders_with_date, load_part, load_partsupp, load_region, load_supplier
label_desc: Use image node_id 4 for calling a global function with example usage: generate(tables, SCALE_FACTOR, folder, upload_to_s3, validate_dataset, num_processes) without return types