code: def make_and_upload_dataset(dir_path):
      import os
      import random
  
      import pandas as pd
      import sklearn.datasets
  
      NUM_EXAMPLES = 2_000_000
      NUM_FEATURES = 20
      PARQUET_FILE_CHUNK_SIZE = 50_000
      NUM_FILES = NUM_EXAMPLES // PARQUET_FILE_CHUNK_SIZE
  
      def create_data_chunk(n, d, seed, include_label=False):
          X, y = sklearn.datasets.make_classification(
              n_samples=n,
              n_features=d,
              n_informative=10,
              n_redundant=2,
              n_repeated=0,
              n_classes=2,
              n_clusters_per_class=3,
              weights=None,
              flip_y=0.03,
              class_sep=0.8,
              hypercube=True,
              shift=0.0,
              scale=1.0,
              shuffle=False,
              random_state=seed,
          )
  
          # turn into dataframe with column names
          col_names = ["feature_%0d" % i for i in range(1, d + 1, 1)]
          df = pd.DataFrame(X)
          df.columns = col_names
  
          # add some bogus categorical data columns
          options = ["apple", "banana", "orange"]
          df["fruit"] = df.feature_1.map(
              lambda x: random.choice(options)
          )  # bogus, but nice to test categoricals
  
          # add some nullable columns
          options = [None, 1, 2]
          df["nullable_feature"] = df.feature_1.map(
              lambda x: random.choice(options)
          )  # bogus, but nice to test categoricals
  
          # add label column
          if include_label:
              df["label"] = y
          return df
  
      # create data files
      print("Creating synthetic dataset...")
      data_path = os.path.join(dir_path, "data")
      os.makedirs(data_path, exist_ok=True)
      for i in range(NUM_FILES):
          path = os.path.join(data_path, f"data_{i:05d}.parquet.snappy")
          if not os.path.exists(path):
              tmp_df = create_data_chunk(
                  n=PARQUET_FILE_CHUNK_SIZE,
                  d=NUM_FEATURES,
                  seed=i,
                  include_label=True,
              )
              tmp_df.to_parquet(path, compression="snappy", index=False)
          print(f"Wrote {path} to disk...")
          # todo: at large enough scale we might want to upload the rest after
          #  first N files rather than write to disk
          # to simulate a user with local copy of subset of data
  
      print("Creating synthetic inference dataset...")
      inference_path = os.path.join(dir_path, "inference")
      os.makedirs(inference_path, exist_ok=True)
      for i in range(NUM_FILES):
          path = os.path.join(
              inference_path, f"data_{i:05d}.parquet.snappy"
          )
          if not os.path.exists(path):
              tmp_df = create_data_chunk(
                  n=PARQUET_FILE_CHUNK_SIZE,
                  d=NUM_FEATURES,
                  seed=i,
                  include_label=False,
              )
              tmp_df.to_parquet(path, compression="snappy", index=False)
          print(f"Wrote {path} to disk...")
  
imports: import argparse
  import collections
  import os
  import sys
  import time
  from tempfile import TemporaryDirectory
  from typing import Tuple
  import boto3
  import mlflow
  import pandas
  import torch
  import torch.nn
  import torch.optim
  import ray
  from ray import train
  from ray.air.integrations.mlflow import MLflowLoggerCallback
  from ray.data.aggregate import Mean, Std
  from ray.train import Checkpoint, DataConfig, RunConfig, ScalingConfig
  from ray.train.torch.torch_trainer import TorchTrainer
label_desc: Use image node_id 1 for calling a global function with example usage: make_and_upload_dataset(dir_path) and returns: df