submit-fine-tuning.py
def submit() -> None:
    """
    Creates and submits submission script. Uses global variables defined at top
    of this file.
    """
    check_paths()

    # create an output directory
    dataset_output_path = f"{HOME}/GraphINVENT/output_{DATASET}"
    tensorboard_path    = os.path.join(dataset_output_path, "tensorboard")
    if JOBNAME != "":
        dataset_output_path = os.path.join(dataset_output_path, JOBNAME)
        tensorboard_path    = os.path.join(tensorboard_path, JOBNAME)

    os.makedirs(dataset_output_path, exist_ok=True)
    os.makedirs(tensorboard_path, exist_ok=True)
    print(f"* Creating dataset directory {dataset_output_path}/", flush=True)

    # submit `N_JOBS` separate jobs
    jobdir_end_idx = JOBDIR_START_IDX + N_JOBS
    for job_idx in range(JOBDIR_START_IDX, jobdir_end_idx):

        # specify and create the job subdirectory if it does not exist
        params["job_dir"]         = f"{dataset_output_path}/job_{job_idx}/"
        params["tensorboard_dir"] = f"{tensorboard_path}/job_{job_idx}/"

        # create the directory if it does not exist already, otherwise raises an
        # error, which is good because *might* not want to override data our
        # existing directories!
        os.makedirs(params["tensorboard_dir"], exist_ok=True)
        try:
            job_dir_exists_already = bool(
                JOB_TYPE in ["generate", "test"] or FORCE_OVERWRITE
            )
            os.makedirs(params["job_dir"], exist_ok=job_dir_exists_already)
            print(
                f"* Creating model subdirectory {dataset_output_path}/job_{job_idx}/",
                flush=True,
            )
        except FileExistsError:
            print(
                f"-- Model subdirectory {dataset_output_path}/job_{job_idx}/ already exists.",
                flush=True,
            )
            if not RESTART:
                continue

        # write the `input.csv` file
        write_input_csv(params_dict=params, filename="input.csv")

        # write `submit.sh` and submit
        if USE_SLURM:
            print("* Writing submission script.", flush=True)
            write_submission_script(job_dir=params["job_dir"],
                                    job_idx=job_idx,
                                    job_type=params["job_type"],
                                    max_n_nodes=params["max_n_nodes"],
                                    runtime=RUN_TIME,
                                    mem=MEM_GB,
                                    ptn=PARTITION,
                                    cpu_per_task=CPUS_PER_TASK,
                                    python_bin_path=PYTHON_PATH)

            print("* Submitting job to SLURM.", flush=True)
            subprocess.run(["sbatch", params["job_dir"] + "submit.sh"],
                           check=True)
        else:
            print("* Running job as a normal process.", flush=True)
            subprocess.run(["ls", f"{PYTHON_PATH}"], check=True)
            subprocess.run([f"{PYTHON_PATH}",
                            f"{GRAPHINVENT_PATH}main.py",
                            "--job-dir",
                            params["job_dir"]],
                           check=True)

        # sleep a few secs before submitting next job
        print("-- Sleeping 2 seconds.")
        time.sleep(2)

    Creates and submits submission script. Uses global variables defined at top
    of this file.
    
Creates and submits submission script. Uses global variables defined at top
of this file.
Creates and submits submission script . Uses global variables defined at top of this file .
def submit() -> None:
    check_paths()
    dataset_output_path = f"{HOME}/GraphINVENT/output_{DATASET}"
    tensorboard_path    = os.path.join(dataset_output_path, "tensorboard")
    if JOBNAME != "":
        dataset_output_path = os.path.join(dataset_output_path, JOBNAME)
        tensorboard_path    = os.path.join(tensorboard_path, JOBNAME)
    os.makedirs(dataset_output_path, exist_ok=True)
    os.makedirs(tensorboard_path, exist_ok=True)
    print(f"* Creating dataset directory {dataset_output_path}/", flush=True)
    jobdir_end_idx = JOBDIR_START_IDX + N_JOBS
    for job_idx in range(JOBDIR_START_IDX, jobdir_end_idx):
        params["job_dir"]         = f"{dataset_output_path}/job_{job_idx}/"
        params["tensorboard_dir"] = f"{tensorboard_path}/job_{job_idx}/"
        os.makedirs(params["tensorboard_dir"], exist_ok=True)
        try:
            job_dir_exists_already = bool(
                JOB_TYPE in ["generate", "test"] or FORCE_OVERWRITE
            )
            os.makedirs(params["job_dir"], exist_ok=job_dir_exists_already)
            print(
                f"* Creating model subdirectory {dataset_output_path}/job_{job_idx}/",
                flush=True,
            )
        except FileExistsError:
            print(
                f"-- Model subdirectory {dataset_output_path}/job_{job_idx}/ already exists.",
                flush=True,
            )
            if not RESTART:
                continue
        write_input_csv(params_dict=params, filename="input.csv")
        if USE_SLURM:
            print("* Writing submission script.", flush=True)
            write_submission_script(job_dir=params["job_dir"],
                                    job_idx=job_idx,
                                    job_type=params["job_type"],
                                    max_n_nodes=params["max_n_nodes"],
                                    runtime=RUN_TIME,
                                    mem=MEM_GB,
                                    ptn=PARTITION,
                                    cpu_per_task=CPUS_PER_TASK,
                                    python_bin_path=PYTHON_PATH)
            print("* Submitting job to SLURM.", flush=True)
            subprocess.run(["sbatch", params["job_dir"] + "submit.sh"],
                           check=True)
        else:
            print("* Running job as a normal process.", flush=True)
            subprocess.run(["ls", f"{PYTHON_PATH}"], check=True)
            subprocess.run([f"{PYTHON_PATH}",
                            f"{GRAPHINVENT_PATH}main.py",
                            "--job-dir",
                            params["job_dir"]],
                           check=True)
        print("-- Sleeping 2 seconds.")
        time.sleep(2)
def submit ( ) -> None : check_paths ( ) dataset_output_path = f"{HOME}/GraphINVENT/output_{DATASET}" tensorboard_path = os . path . join ( dataset_output_path , "tensorboard" ) if JOBNAME != "" : dataset_output_path = os . path . join ( dataset_output_path , JOBNAME ) tensorboard_path = os . path . join ( tensorboard_path , JOBNAME ) os . makedirs ( dataset_output_path , exist_ok = True ) os . makedirs ( tensorboard_path , exist_ok = True ) print ( f"* Creating dataset directory {dataset_output_path}/" , flush = True ) jobdir_end_idx = JOBDIR_START_IDX + N_JOBS for job_idx in range ( JOBDIR_START_IDX , jobdir_end_idx ) : params [ "job_dir" ] = f"{dataset_output_path}/job_{job_idx}/" params [ "tensorboard_dir" ] = f"{tensorboard_path}/job_{job_idx}/" os . makedirs ( params [ "tensorboard_dir" ] , exist_ok = True ) try : job_dir_exists_already = bool ( JOB_TYPE in [ "generate" , "test" ] or FORCE_OVERWRITE ) os . makedirs ( params [ "job_dir" ] , exist_ok = job_dir_exists_already ) print ( f"* Creating model subdirectory {dataset_output_path}/job_{job_idx}/" , flush = True , ) except FileExistsError : print ( f"-- Model subdirectory {dataset_output_path}/job_{job_idx}/ already exists." , flush = True , ) if not RESTART : continue write_input_csv ( params_dict = params , filename = "input.csv" ) if USE_SLURM : print ( "* Writing submission script." , flush = True ) write_submission_script ( job_dir = params [ "job_dir" ] , job_idx = job_idx , job_type = params [ "job_type" ] , max_n_nodes = params [ "max_n_nodes" ] , runtime = RUN_TIME , mem = MEM_GB , ptn = PARTITION , cpu_per_task = CPUS_PER_TASK , python_bin_path = PYTHON_PATH ) print ( "* Submitting job to SLURM." , flush = True ) subprocess . run ( [ "sbatch" , params [ "job_dir" ] + "submit.sh" ] , check = True ) else : print ( "* Running job as a normal process." , flush = True ) subprocess . run ( [ "ls" , f"{PYTHON_PATH}" ] , check = True ) subprocess . run ( [ f"{PYTHON_PATH}" , f"{GRAPHINVENT_PATH}main.py" , "--job-dir" , params [ "job_dir" ] ] , check = True ) print ( "-- Sleeping 2 seconds." ) time . sleep ( 2 )
Creates and submits submission script.
Creates and submits submission script .
"""
    Creates and submits submission script. Uses global variables defined at top
    of this file.
    """ # create an output directory # submit `N_JOBS` separate jobs # specify and create the job subdirectory if it does not exist # create the directory if it does not exist already, otherwise raises an # error, which is good because *might* not want to override data our # existing directories! # write the `input.csv` file # write `submit.sh` and submit # sleep a few secs before submitting next job