aztk/spark/client/cluster/helpers/create.py
def create_cluster(core_cluster_operations,
                   spark_cluster_operations,
                   cluster_conf: models.ClusterConfiguration,
                   wait: bool = False):
    """
    Create a new aztk spark cluster

    Args:
        cluster_conf(aztk.spark.models.models.ClusterConfiguration): Configuration for the the cluster to be created
        wait(bool): If you should wait for the cluster to be ready before returning

    Returns:
        :obj:`aztk.spark.models.Cluster`
    """
    cluster_conf = _apply_default_for_cluster_config(cluster_conf)
    cluster_conf.validate()

    cluster_data = core_cluster_operations.get_cluster_data(cluster_conf.cluster_id)
    try:
        zip_resource_files = None
        node_data = NodeData(cluster_conf).add_core().done()
        zip_resource_files = cluster_data.upload_node_data(node_data).to_resource_file()

        start_task = spark_cluster_operations._generate_cluster_start_task(
            core_cluster_operations,
            zip_resource_files,
            cluster_conf.cluster_id,
            cluster_conf.gpu_enabled(),
            cluster_conf.get_docker_repo(),
            cluster_conf.get_docker_run_options(),
            cluster_conf.file_shares,
            cluster_conf.mixed_mode(),
            cluster_conf.worker_on_master,
        )

        software_metadata_key = base_models.Software.spark

        cluster = core_cluster_operations.create(cluster_conf, software_metadata_key, start_task,
                                                 constants.SPARK_VM_IMAGE)

        # Wait for the master to be ready
        if wait:
            util.wait_for_master_to_be_ready(core_cluster_operations, spark_cluster_operations, cluster.id)
            cluster = spark_cluster_operations.get(cluster.id)

        return cluster

    except BatchErrorException as e:
        raise error.AztkError(helpers.format_batch_exception(e))

    Create a new aztk spark cluster

    Args:
        cluster_conf(aztk.spark.models.models.ClusterConfiguration): Configuration for the the cluster to be created
        wait(bool): If you should wait for the cluster to be ready before returning

    Returns:
        :obj:`aztk.spark.models.Cluster`
    
Create a new aztk spark cluster
Create a new aztk spark cluster
def create_cluster(core_cluster_operations,
                   spark_cluster_operations,
                   cluster_conf: models.ClusterConfiguration,
                   wait: bool = False):
    cluster_conf = _apply_default_for_cluster_config(cluster_conf)
    cluster_conf.validate()
    cluster_data = core_cluster_operations.get_cluster_data(cluster_conf.cluster_id)
    try:
        zip_resource_files = None
        node_data = NodeData(cluster_conf).add_core().done()
        zip_resource_files = cluster_data.upload_node_data(node_data).to_resource_file()
        start_task = spark_cluster_operations._generate_cluster_start_task(
            core_cluster_operations,
            zip_resource_files,
            cluster_conf.cluster_id,
            cluster_conf.gpu_enabled(),
            cluster_conf.get_docker_repo(),
            cluster_conf.get_docker_run_options(),
            cluster_conf.file_shares,
            cluster_conf.mixed_mode(),
            cluster_conf.worker_on_master,
        )
        software_metadata_key = base_models.Software.spark
        cluster = core_cluster_operations.create(cluster_conf, software_metadata_key, start_task,
                                                 constants.SPARK_VM_IMAGE)
        if wait:
            util.wait_for_master_to_be_ready(core_cluster_operations, spark_cluster_operations, cluster.id)
            cluster = spark_cluster_operations.get(cluster.id)
        return cluster
    except BatchErrorException as e:
        raise error.AztkError(helpers.format_batch_exception(e))
def create_cluster ( core_cluster_operations , spark_cluster_operations , cluster_conf : models . ClusterConfiguration , wait : bool = False ) : cluster_conf = _apply_default_for_cluster_config ( cluster_conf ) cluster_conf . validate ( ) cluster_data = core_cluster_operations . get_cluster_data ( cluster_conf . cluster_id ) try : zip_resource_files = None node_data = NodeData ( cluster_conf ) . add_core ( ) . done ( ) zip_resource_files = cluster_data . upload_node_data ( node_data ) . to_resource_file ( ) start_task = spark_cluster_operations . _generate_cluster_start_task ( core_cluster_operations , zip_resource_files , cluster_conf . cluster_id , cluster_conf . gpu_enabled ( ) , cluster_conf . get_docker_repo ( ) , cluster_conf . get_docker_run_options ( ) , cluster_conf . file_shares , cluster_conf . mixed_mode ( ) , cluster_conf . worker_on_master , ) software_metadata_key = base_models . Software . spark cluster = core_cluster_operations . create ( cluster_conf , software_metadata_key , start_task , constants . SPARK_VM_IMAGE ) if wait : util . wait_for_master_to_be_ready ( core_cluster_operations , spark_cluster_operations , cluster . id ) cluster = spark_cluster_operations . get ( cluster . id ) return cluster except BatchErrorException as e : raise error . AztkError ( helpers . format_batch_exception ( e ) )
Create a new aztk spark cluster
Create a new aztk spark cluster
"""
    Create a new aztk spark cluster

    Args:
        cluster_conf(aztk.spark.models.models.ClusterConfiguration): Configuration for the the cluster to be created
        wait(bool): If you should wait for the cluster to be ready before returning

    Returns:
        :obj:`aztk.spark.models.Cluster`
    """ # Wait for the master to be ready
None
core_cluster_operations
spark_cluster_operations
cluster_conf
models.ClusterConfiguration
Configuration for the the cluster to be created
Configuration for the the cluster to be created
wait
bool
If you should wait for the cluster to be ready before returning
If you should wait for the cluster to be ready before returning