def run_pyspark_yarn_cluster(env_dir, env_name, env_archive, args):
    """
    Initializes the requires spark command line options on order to start a python job with the given python environment.

    Parameters
    ----------
    env_dir : str
    env_name : str
    env_archive : str
    args : list

    Returns
    -------
    This call will spawn a child process and block until that is complete.
    """
    env = dict(os.environ)
    yarn_python = os.path.join(".", "CONDA", env_name, "bin", "python")
    archives = env_archive + "#CONDA"

    prepend_args = [
        "--master", "yarn",
        "--deploy-mode", "cluster",
        "--conf", "spark.yarn.appMasterEnv.PYSPARK_PYTHON={}".format(yarn_python),
        "--archives", archives,
    ]

    env_update = {
        "PYSPARK_PYTHON": yarn_python
    }

    env.update(env_update)
    spark_submit = os.path.join(env["SPARK_HOME"], "bin", "spark-submit")

    log.info("Running spark in YARN-client mode with added arguments")
    log.info("  args: %s", pprint.pprint(prepend_args, indent=4))
    log.info("  env: %s", pprint.pprint(env_update, indent=4))
    # REPLACE our python process with another one
    subprocess.check_call([spark_submit] + prepend_args + args, env=env)
Initializes the requires spark command line options on order to start a python job with the given python environment.

    Parameters
    ----------
    env_dir : str
    env_name : str
    env_archive : str
    args : list

    Returns
    -------
    This call will spawn a child process and block until that is complete.