aztk/spark/client/base/helpers/generate_cluster_start_task.py
def generate_cluster_start_task(
        core_base_operations,
        zip_resource_file: batch_models.ResourceFile,
        cluster_id: str,
        gpu_enabled: bool,
        docker_repo: str = None,
        docker_run_options: str = None,
        file_shares: List[models.FileShare] = None,
        mixed_mode: bool = False,
        worker_on_master: bool = True,
):
    """
        This will return the start task object for the pool to be created.
        :param cluster_id str: Id of the cluster(Used for uploading the resource files)
        :param zip_resource_file: Resource file object pointing to the zip file containing scripts to run on the node
    """

    resource_files = [zip_resource_file]
    spark_web_ui_port = constants.DOCKER_SPARK_WEB_UI_PORT
    spark_worker_ui_port = constants.DOCKER_SPARK_WORKER_UI_PORT
    spark_job_ui_port = constants.DOCKER_SPARK_JOB_UI_PORT

    spark_container_name = constants.DOCKER_SPARK_CONTAINER_NAME
    spark_submit_logs_file = constants.SPARK_SUBMIT_LOGS_FILE

    # TODO use certificate
    environment_settings = (__get_secrets_env(core_base_operations) + [
        batch_models.EnvironmentSetting(name="SPARK_WEB_UI_PORT", value=spark_web_ui_port),
        batch_models.EnvironmentSetting(name="SPARK_WORKER_UI_PORT", value=spark_worker_ui_port),
        batch_models.EnvironmentSetting(name="SPARK_JOB_UI_PORT", value=spark_job_ui_port),
        batch_models.EnvironmentSetting(name="SPARK_CONTAINER_NAME", value=spark_container_name),
        batch_models.EnvironmentSetting(name="SPARK_SUBMIT_LOGS_FILE", value=spark_submit_logs_file),
        batch_models.EnvironmentSetting(name="AZTK_GPU_ENABLED", value=helpers.bool_env(gpu_enabled)),
    ] + __get_docker_credentials(core_base_operations) + _get_aztk_environment(cluster_id, worker_on_master,
                                                                               mixed_mode))

    # start task command
    command = __cluster_install_cmd(zip_resource_file, gpu_enabled, docker_repo, docker_run_options, file_shares)

    return batch_models.StartTask(
        command_line=helpers.wrap_commands_in_shell(command),
        resource_files=resource_files,
        environment_settings=environment_settings,
        user_identity=POOL_ADMIN_USER_IDENTITY,
        wait_for_success=True,
        max_task_retry_count=2,
    )

        This will return the start task object for the pool to be created.
        :param cluster_id str: Id of the cluster(Used for uploading the resource files)
        :param zip_resource_file: Resource file object pointing to the zip file containing scripts to run on the node
    
This will return the start task object for the pool to be created.
This will return the start task object for the pool to be created .
def generate_cluster_start_task(
        core_base_operations,
        zip_resource_file: batch_models.ResourceFile,
        cluster_id: str,
        gpu_enabled: bool,
        docker_repo: str = None,
        docker_run_options: str = None,
        file_shares: List[models.FileShare] = None,
        mixed_mode: bool = False,
        worker_on_master: bool = True,
):
    resource_files = [zip_resource_file]
    spark_web_ui_port = constants.DOCKER_SPARK_WEB_UI_PORT
    spark_worker_ui_port = constants.DOCKER_SPARK_WORKER_UI_PORT
    spark_job_ui_port = constants.DOCKER_SPARK_JOB_UI_PORT
    spark_container_name = constants.DOCKER_SPARK_CONTAINER_NAME
    spark_submit_logs_file = constants.SPARK_SUBMIT_LOGS_FILE
    environment_settings = (__get_secrets_env(core_base_operations) + [
        batch_models.EnvironmentSetting(name="SPARK_WEB_UI_PORT", value=spark_web_ui_port),
        batch_models.EnvironmentSetting(name="SPARK_WORKER_UI_PORT", value=spark_worker_ui_port),
        batch_models.EnvironmentSetting(name="SPARK_JOB_UI_PORT", value=spark_job_ui_port),
        batch_models.EnvironmentSetting(name="SPARK_CONTAINER_NAME", value=spark_container_name),
        batch_models.EnvironmentSetting(name="SPARK_SUBMIT_LOGS_FILE", value=spark_submit_logs_file),
        batch_models.EnvironmentSetting(name="AZTK_GPU_ENABLED", value=helpers.bool_env(gpu_enabled)),
    ] + __get_docker_credentials(core_base_operations) + _get_aztk_environment(cluster_id, worker_on_master,
                                                                               mixed_mode))
    command = __cluster_install_cmd(zip_resource_file, gpu_enabled, docker_repo, docker_run_options, file_shares)
    return batch_models.StartTask(
        command_line=helpers.wrap_commands_in_shell(command),
        resource_files=resource_files,
        environment_settings=environment_settings,
        user_identity=POOL_ADMIN_USER_IDENTITY,
        wait_for_success=True,
        max_task_retry_count=2,
    )
def generate_cluster_start_task ( core_base_operations , zip_resource_file : batch_models . ResourceFile , cluster_id : str , gpu_enabled : bool , docker_repo : str = None , docker_run_options : str = None , file_shares : List [ models . FileShare ] = None , mixed_mode : bool = False , worker_on_master : bool = True , ) : resource_files = [ zip_resource_file ] spark_web_ui_port = constants . DOCKER_SPARK_WEB_UI_PORT spark_worker_ui_port = constants . DOCKER_SPARK_WORKER_UI_PORT spark_job_ui_port = constants . DOCKER_SPARK_JOB_UI_PORT spark_container_name = constants . DOCKER_SPARK_CONTAINER_NAME spark_submit_logs_file = constants . SPARK_SUBMIT_LOGS_FILE environment_settings = ( __get_secrets_env ( core_base_operations ) + [ batch_models . EnvironmentSetting ( name = "SPARK_WEB_UI_PORT" , value = spark_web_ui_port ) , batch_models . EnvironmentSetting ( name = "SPARK_WORKER_UI_PORT" , value = spark_worker_ui_port ) , batch_models . EnvironmentSetting ( name = "SPARK_JOB_UI_PORT" , value = spark_job_ui_port ) , batch_models . EnvironmentSetting ( name = "SPARK_CONTAINER_NAME" , value = spark_container_name ) , batch_models . EnvironmentSetting ( name = "SPARK_SUBMIT_LOGS_FILE" , value = spark_submit_logs_file ) , batch_models . EnvironmentSetting ( name = "AZTK_GPU_ENABLED" , value = helpers . bool_env ( gpu_enabled ) ) , ] + __get_docker_credentials ( core_base_operations ) + _get_aztk_environment ( cluster_id , worker_on_master , mixed_mode ) ) command = __cluster_install_cmd ( zip_resource_file , gpu_enabled , docker_repo , docker_run_options , file_shares ) return batch_models . StartTask ( command_line = helpers . wrap_commands_in_shell ( command ) , resource_files = resource_files , environment_settings = environment_settings , user_identity = POOL_ADMIN_USER_IDENTITY , wait_for_success = True , max_task_retry_count = 2 , )
This will return the start task object for the pool to be created.
This will return the start task object for the pool to be created .
"""
        This will return the start task object for the pool to be created.
        :param cluster_id str: Id of the cluster(Used for uploading the resource files)
        :param zip_resource_file: Resource file object pointing to the zip file containing scripts to run on the node
    """ # TODO use certificate # start task command
core_base_operations
zip_resource_file
batch_models.ResourceFile
Resource file object pointing to the zip file containing scripts to run on the node
Resource file object pointing to the zip file containing scripts to run on the node
cluster_id
str
gpu_enabled
bool
docker_repo
str
docker_run_options
str
file_shares
List[models.FileShare]
mixed_mode
bool
worker_on_master
bool
str
Id of the cluster(Used for uploading the resource files)
Id of the cluster ( Used for uploading the resource files )