aztk_cli/utils.py
def ssh_in_master(
        client,
        cluster_id: str,
        cluster_configuration: models.ClusterConfiguration,
        username: str = None,
        webui: str = None,
        jobui: str = None,
        jobhistoryui: str = None,
        ports=None,
        host: bool = False,
        connect: bool = True,
        internal: bool = False,
):
    """
        SSH into head node of spark-app
        :param cluster_id: Id of the cluster to ssh in
        :param username: Username to use to ssh
        :param webui: Port for the spark master web ui (Local port)
        :param jobui: Port for the job web ui (Local port)
        :param ports: an list of local and remote ports
        :type ports: [[<local-port>, <remote-port>]]
    """
    # check if ssh is available, this throws OSError if ssh is not present
    subprocess.call(["ssh"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    # Get master node id from task (job and task are both named pool_id)
    cluster = client.cluster.get(cluster_id)

    master_node_id = cluster.master_node_id

    if master_node_id is None:
        raise error.ClusterNotReadyError("Master node has not yet been picked!")

    # get remote login settings for the user
    remote_login_settings = client.cluster.get_remote_login_settings(cluster.id, master_node_id)
    master_internal_node_ip = [node.ip_address for node in cluster.nodes if node.id == master_node_id][0]
    master_node_ip = remote_login_settings.ip_address
    master_node_port = remote_login_settings.port

    spark_web_ui_port = utils.constants.DOCKER_SPARK_WEB_UI_PORT
    spark_job_ui_port = utils.constants.DOCKER_SPARK_JOB_UI_PORT
    spark_job_history_ui_port = utils.constants.DOCKER_SPARK_JOB_UI_HISTORY_PORT

    ssh_command = utils.command_builder.CommandBuilder("ssh")

    # get ssh private key path if specified
    ssh_priv_key = client.secrets_configuration.ssh_priv_key
    if ssh_priv_key is not None:
        ssh_command.add_option("-i", ssh_priv_key)

    ssh_command.add_argument("-t")
    ssh_command.add_option("-L", "{0}:localhost:{1}".format(webui, spark_web_ui_port), enable=bool(webui))
    ssh_command.add_option("-L", "{0}:localhost:{1}".format(jobui, spark_job_ui_port), enable=bool(jobui))
    ssh_command.add_option(
        "-L", "{0}:localhost:{1}".format(jobhistoryui, spark_job_history_ui_port), enable=bool(jobui))

    if ports is not None:
        for port in ports:
            ssh_command.add_option("-L", "{0}:localhost:{1}".format(port[0], port[1]))
    if cluster_configuration and cluster_configuration.plugins:
        for plugin in cluster_configuration.plugins:
            for port in plugin.ports:
                if port.expose_publicly:
                    ssh_command.add_option("-L", "{0}:localhost:{1}".format(port.public_port, port.internal))

    user = username if username is not None else "<username>"
    if internal:
        ssh_command.add_argument("{0}@{1}".format(user, master_internal_node_ip))
    else:
        ssh_command.add_argument("{0}@{1} -p {2}".format(user, master_node_ip, master_node_port))

    if host is False:
        ssh_command.add_argument("'sudo docker exec -it spark /bin/bash'")

    command = ssh_command.to_str()

    if connect:
        subprocess.call(command, shell=True)

    return "\n\t{}\n".format(command)

        SSH into head node of spark-app
        :param cluster_id: Id of the cluster to ssh in
        :param username: Username to use to ssh
        :param webui: Port for the spark master web ui (Local port)
        :param jobui: Port for the job web ui (Local port)
        :param ports: an list of local and remote ports
        :type ports: [[<local-port>, <remote-port>]]
    
SSH into head node of spark-app
SSH into head node of spark - app
def ssh_in_master(
        client,
        cluster_id: str,
        cluster_configuration: models.ClusterConfiguration,
        username: str = None,
        webui: str = None,
        jobui: str = None,
        jobhistoryui: str = None,
        ports=None,
        host: bool = False,
        connect: bool = True,
        internal: bool = False,
):
    subprocess.call(["ssh"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    cluster = client.cluster.get(cluster_id)
    master_node_id = cluster.master_node_id
    if master_node_id is None:
        raise error.ClusterNotReadyError("Master node has not yet been picked!")
    remote_login_settings = client.cluster.get_remote_login_settings(cluster.id, master_node_id)
    master_internal_node_ip = [node.ip_address for node in cluster.nodes if node.id == master_node_id][0]
    master_node_ip = remote_login_settings.ip_address
    master_node_port = remote_login_settings.port
    spark_web_ui_port = utils.constants.DOCKER_SPARK_WEB_UI_PORT
    spark_job_ui_port = utils.constants.DOCKER_SPARK_JOB_UI_PORT
    spark_job_history_ui_port = utils.constants.DOCKER_SPARK_JOB_UI_HISTORY_PORT
    ssh_command = utils.command_builder.CommandBuilder("ssh")
    ssh_priv_key = client.secrets_configuration.ssh_priv_key
    if ssh_priv_key is not None:
        ssh_command.add_option("-i", ssh_priv_key)
    ssh_command.add_argument("-t")
    ssh_command.add_option("-L", "{0}:localhost:{1}".format(webui, spark_web_ui_port), enable=bool(webui))
    ssh_command.add_option("-L", "{0}:localhost:{1}".format(jobui, spark_job_ui_port), enable=bool(jobui))
    ssh_command.add_option(
        "-L", "{0}:localhost:{1}".format(jobhistoryui, spark_job_history_ui_port), enable=bool(jobui))
    if ports is not None:
        for port in ports:
            ssh_command.add_option("-L", "{0}:localhost:{1}".format(port[0], port[1]))
    if cluster_configuration and cluster_configuration.plugins:
        for plugin in cluster_configuration.plugins:
            for port in plugin.ports:
                if port.expose_publicly:
                    ssh_command.add_option("-L", "{0}:localhost:{1}".format(port.public_port, port.internal))
    user = username if username is not None else "<username>"
    if internal:
        ssh_command.add_argument("{0}@{1}".format(user, master_internal_node_ip))
    else:
        ssh_command.add_argument("{0}@{1} -p {2}".format(user, master_node_ip, master_node_port))
    if host is False:
        ssh_command.add_argument("'sudo docker exec -it spark /bin/bash'")
    command = ssh_command.to_str()
    if connect:
        subprocess.call(command, shell=True)
    return "\n\t{}\n".format(command)
def ssh_in_master ( client , cluster_id : str , cluster_configuration : models . ClusterConfiguration , username : str = None , webui : str = None , jobui : str = None , jobhistoryui : str = None , ports = None , host : bool = False , connect : bool = True , internal : bool = False , ) : subprocess . call ( [ "ssh" ] , stdout = subprocess . PIPE , stderr = subprocess . PIPE ) cluster = client . cluster . get ( cluster_id ) master_node_id = cluster . master_node_id if master_node_id is None : raise error . ClusterNotReadyError ( "Master node has not yet been picked!" ) remote_login_settings = client . cluster . get_remote_login_settings ( cluster . id , master_node_id ) master_internal_node_ip = [ node . ip_address for node in cluster . nodes if node . id == master_node_id ] [ 0 ] master_node_ip = remote_login_settings . ip_address master_node_port = remote_login_settings . port spark_web_ui_port = utils . constants . DOCKER_SPARK_WEB_UI_PORT spark_job_ui_port = utils . constants . DOCKER_SPARK_JOB_UI_PORT spark_job_history_ui_port = utils . constants . DOCKER_SPARK_JOB_UI_HISTORY_PORT ssh_command = utils . command_builder . CommandBuilder ( "ssh" ) ssh_priv_key = client . secrets_configuration . ssh_priv_key if ssh_priv_key is not None : ssh_command . add_option ( "-i" , ssh_priv_key ) ssh_command . add_argument ( "-t" ) ssh_command . add_option ( "-L" , "{0}:localhost:{1}" . format ( webui , spark_web_ui_port ) , enable = bool ( webui ) ) ssh_command . add_option ( "-L" , "{0}:localhost:{1}" . format ( jobui , spark_job_ui_port ) , enable = bool ( jobui ) ) ssh_command . add_option ( "-L" , "{0}:localhost:{1}" . format ( jobhistoryui , spark_job_history_ui_port ) , enable = bool ( jobui ) ) if ports is not None : for port in ports : ssh_command . add_option ( "-L" , "{0}:localhost:{1}" . format ( port [ 0 ] , port [ 1 ] ) ) if cluster_configuration and cluster_configuration . plugins : for plugin in cluster_configuration . plugins : for port in plugin . ports : if port . expose_publicly : ssh_command . add_option ( "-L" , "{0}:localhost:{1}" . format ( port . public_port , port . internal ) ) user = username if username is not None else "<username>" if internal : ssh_command . add_argument ( "{0}@{1}" . format ( user , master_internal_node_ip ) ) else : ssh_command . add_argument ( "{0}@{1} -p {2}" . format ( user , master_node_ip , master_node_port ) ) if host is False : ssh_command . add_argument ( "'sudo docker exec -it spark /bin/bash'" ) command = ssh_command . to_str ( ) if connect : subprocess . call ( command , shell = True ) return "\n\t{}\n" . format ( command )
SSH into head node of spark-app
SSH into head node of spark - app
"""
        SSH into head node of spark-app
        :param cluster_id: Id of the cluster to ssh in
        :param username: Username to use to ssh
        :param webui: Port for the spark master web ui (Local port)
        :param jobui: Port for the job web ui (Local port)
        :param ports: an list of local and remote ports
        :type ports: [[<local-port>, <remote-port>]]
    """ # check if ssh is available, this throws OSError if ssh is not present # Get master node id from task (job and task are both named pool_id) # get remote login settings for the user # get ssh private key path if specified
client
cluster_id
str
Id of the cluster to ssh in
Id of the cluster to ssh in
cluster_configuration
models.ClusterConfiguration
username
str
Username to use to ssh
Username to use to ssh
webui
str
Port for the spark master web ui (Local port)
Port for the spark master web ui ( Local port )
jobui
str
Port for the job web ui (Local port)
Port for the job web ui ( Local port )
jobhistoryui
str
ports
an list of local and remote ports
an list of local and remote ports
host
bool
connect
bool
internal
bool