import inspect
import json
import os
import random
import subprocess
import time
import requests
import ast
import paramiko
import rancher
from rancher import ApiError
from lib.aws import AmazonWebServices

DEFAULT_TIMEOUT = 120
DEFAULT_MULTI_CLUSTER_APP_TIMEOUT = 300

CATTLE_TEST_URL = os.environ.get('CATTLE_TEST_URL', "http://localhost:80")
ADMIN_TOKEN = os.environ.get('ADMIN_TOKEN', "None")

CATTLE_API_URL = CATTLE_TEST_URL + "/v3"

kube_fname = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                          "k8s_kube_config")
MACHINE_TIMEOUT = float(os.environ.get('RANCHER_MACHINE_TIMEOUT', "1200"))

TEST_IMAGE = "sangeetha/mytestcontainer"
CLUSTER_NAME = os.environ.get("RANCHER_CLUSTER_NAME", "")
RANCHER_CLEANUP_CLUSTER = \
    ast.literal_eval(os.environ.get('RANCHER_CLEANUP_CLUSTER', "True"))
env_file = os.path.join(
    os.path.dirname(os.path.realpath(__file__)),
    "rancher_env.config")
CLUSTER_NAME_2 = ""

def random_str():
    return 'random-{0}-{1}'.format(random_num(), int(time.time()))


def random_num():
    return random.randint(0, 1000000)


def random_int(start, end):
    return random.randint(start, end)


def random_test_name(name="test"):
    return name + "-" + str(random_int(10000, 99999))


def get_admin_client():
    return rancher.Client(url=CATTLE_API_URL, token=ADMIN_TOKEN, verify=False)


def get_client_for_token(token):
    return rancher.Client(url=CATTLE_API_URL, token=token, verify=False)


def get_project_client_for_token(project, token):
    p_url = project.links['self'] + '/schemas'
    p_client = rancher.Client(url=p_url, token=token, verify=False)
    return p_client


def get_cluster_client_for_token(cluster, token):
    c_url = cluster.links['self'] + '/schemas'
    c_client = rancher.Client(url=c_url, token=token, verify=False)
    return c_client


def up(cluster, token):
    c_url = cluster.links['self'] + '/schemas'
    c_client = rancher.Client(url=c_url, token=token, verify=False)
    return c_client


def wait_state(client, obj, state, timeout=DEFAULT_TIMEOUT):
    wait_for(lambda: client.reload(obj).state == state, timeout)
    return client.reload(obj)


def wait_for_condition(client, resource, check_function, fail_handler=None,
                       timeout=DEFAULT_TIMEOUT):
    start = time.time()
    resource = client.reload(resource)
    while not check_function(resource):
        if time.time() - start > timeout:
            exceptionMsg = 'Timeout waiting for ' + resource.baseType + \
                ' to satisfy condition: ' + \
                inspect.getsource(check_function)
            if fail_handler:
                exceptionMsg = exceptionMsg + fail_handler(resource)
            raise Exception(exceptionMsg)
        time.sleep(.5)
        resource = client.reload(resource)
    return resource


def wait_for(callback, timeout=DEFAULT_TIMEOUT, timeout_message=None):
    start = time.time()
    ret = callback()
    while ret is None or ret is False:
        time.sleep(.5)
        if time.time() - start > timeout:
            if timeout_message:
                raise Exception(timeout_message)
            else:
                raise Exception('Timeout waiting for condition')
        ret = callback()
    return ret


def random_name():
    return "test" + "-" + str(random_int(10000, 99999))


def create_project_and_ns(token, cluster, project_name=None, ns_name=None):
    client = get_client_for_token(token)
    p = create_project(client, cluster, project_name)
    c_client = get_cluster_client_for_token(cluster, token)
    ns = create_ns(c_client, cluster, p, ns_name)
    return p, ns


def create_project(client, cluster, project_name=None):
    if project_name is None:
        project_name = random_name()
    p = client.create_project(name=project_name,
                              clusterId=cluster.id)
    time.sleep(5)
    p = wait_until_available(client, p)
    assert p.state == 'active'
    return p


def create_project_with_pspt(client, cluster, pspt):
    p = client.create_project(name=random_name(),
                              clusterId=cluster.id)
    p = wait_until_available(client, p)
    assert p.state == 'active'
    return set_pspt_for_project(p, client, pspt)


def set_pspt_for_project(project, client, pspt):
    project.setpodsecuritypolicytemplate(podSecurityPolicyTemplateId=pspt.id)
    project = wait_until_available(client, project)
    assert project.state == 'active'
    return project


def create_ns(client, cluster, project, ns_name=None):
    if ns_name is None:
        ns_name = random_name()
    ns = client.create_namespace(name=ns_name,
                                 clusterId=cluster.id,
                                 projectId=project.id)
    wait_for_ns_to_become_active(client, ns)
    ns = client.reload(ns)
    assert ns.state == 'active'
    return ns


def assign_members_to_cluster(client, user, cluster, role_template_id):
    crtb = client.create_cluster_role_template_binding(
        clusterId=cluster.id,
        roleTemplateId=role_template_id,
        subjectKind="User",
        userId=user.id)
    return crtb


def assign_members_to_project(client, user, project, role_template_id):
    prtb = client.create_project_role_template_binding(
        projectId=project.id,
        roleTemplateId=role_template_id,
        subjectKind="User",
        userId=user.id)
    return prtb


def change_member_role_in_cluster(client, user, crtb, role_template_id):
    crtb = client.update(
        crtb,
        roleTemplateId=role_template_id,
        userId=user.id)
    return crtb


def change_member_role_in_project(client, user, prtb, role_template_id):
    prtb = client.update(
        prtb,
        roleTemplateId=role_template_id,
        userId=user.id)
    return prtb


def create_kubeconfig(cluster):
    generateKubeConfigOutput = cluster.generateKubeconfig()
    print(generateKubeConfigOutput.config)
    file = open(kube_fname, "w")
    file.write(generateKubeConfigOutput.config)
    file.close()


def validate_psp_error_worklaod(p_client, workload, error_message):
    workload = wait_for_wl_transitioning(p_client, workload)
    assert workload.state == "updating"
    assert workload.transitioning == "error"
    print(workload.transitioningMessage)
    assert error_message in workload.transitioningMessage


def validate_workload(p_client, workload, type, ns_name, pod_count=1,
                      wait_for_cron_pods=60):
    workload = wait_for_wl_to_active(p_client, workload)
    assert workload.state == "active"
    # For cronjob, wait for the first pod to get created after
    # scheduled wait time
    if type == "cronJob":
        time.sleep(wait_for_cron_pods)
    pods = p_client.list_pod(workloadId=workload.id).data
    assert len(pods) == pod_count
    for pod in pods:
        wait_for_pod_to_running(p_client, pod)
    wl_result = execute_kubectl_cmd(
        "get " + type + " " + workload.name + " -n " + ns_name)
    if type == "deployment" or type == "statefulSet":
        assert wl_result["status"]["readyReplicas"] == pod_count
    if type == "daemonSet":
        assert wl_result["status"]["currentNumberScheduled"] == pod_count
    if type == "cronJob":
        assert len(wl_result["status"]["active"]) >= pod_count
        return
    for key, value in workload.workloadLabels.items():
        label = key + "=" + value
    get_pods = "get pods -l" + label + " -n " + ns_name
    pods_result = execute_kubectl_cmd(get_pods)
    assert len(pods_result["items"]) == pod_count
    for pod in pods_result["items"]:
        assert pod["status"]["phase"] == "Running"
    return pods_result["items"]


def validate_workload_with_sidekicks(p_client, workload, type, ns_name,
                                     pod_count=1):
    workload = wait_for_wl_to_active(p_client, workload)
    assert workload.state == "active"
    pods = wait_for_pods_in_workload(p_client, workload, pod_count)
    assert len(pods) == pod_count
    for pod in pods:
        wait_for_pod_to_running(p_client, pod)
    wl_result = execute_kubectl_cmd(
        "get " + type + " " + workload.name + " -n " + ns_name)
    assert wl_result["status"]["readyReplicas"] == pod_count
    for key, value in workload.workloadLabels.items():
        label = key + "=" + value
    get_pods = "get pods -l" + label + " -n " + ns_name
    execute_kubectl_cmd(get_pods)
    pods_result = execute_kubectl_cmd(get_pods)
    assert len(pods_result["items"]) == pod_count
    for pod in pods_result["items"]:
        assert pod["status"]["phase"] == "Running"
        assert len(pod["status"]["containerStatuses"]) == 2
        assert "running" in pod["status"]["containerStatuses"][0]["state"]
        assert "running" in pod["status"]["containerStatuses"][1]["state"]


def validate_workload_paused(p_client, workload, expectedstatus):
    workloadStatus = p_client.list_workload(uuid=workload.uuid).data[0].paused
    assert workloadStatus == expectedstatus


def validate_pod_images(expectedimage, workload, ns_name):
    for key, value in workload.workloadLabels.items():
        label = key + "=" + value
    get_pods = "get pods -l" + label + " -n " + ns_name
    pods = execute_kubectl_cmd(get_pods)

    for pod in pods["items"]:
        assert pod["spec"]["containers"][0]["image"] == expectedimage


def validate_pods_are_running_by_id(expectedpods, workload, ns_name):
    for key, value in workload.workloadLabels.items():
        label = key + "=" + value
    get_pods = "get pods -l" + label + " -n " + ns_name
    pods = execute_kubectl_cmd(get_pods)

    curpodnames = []
    for pod in pods["items"]:
        curpodnames.append(pod["metadata"]["name"])

    for expectedpod in expectedpods["items"]:
        assert expectedpod["metadata"]["name"] in curpodnames


def validate_workload_image(client, workload, expectedImage, ns):
    workload = client.list_workload(uuid=workload.uuid).data[0]
    assert workload.containers[0].image == expectedImage
    validate_pod_images(expectedImage, workload, ns.name)


def execute_kubectl_cmd(cmd, json_out=True, stderr=False):
    command = 'kubectl --kubeconfig {0} {1}'.format(
        kube_fname, cmd)
    if json_out:
        command += ' -o json'
    if stderr:
        result = run_command_with_stderr(command)
    else:
        result = run_command(command)
    if json_out:
        result = json.loads(result)
    print(result)
    return result


def run_command(command):
    return subprocess.check_output(command, shell=True, text=True)


def run_command_with_stderr(command):
    try:
        output = subprocess.check_output(command, shell=True,
                                         stderr=subprocess.PIPE)
        returncode = 0
    except subprocess.CalledProcessError as e:
        output = e.output
        returncode = e.returncode
    print(returncode)
    return (output, returncode)


def wait_for_wl_to_active(client, workload, timeout=DEFAULT_TIMEOUT):
    start = time.time()
    workloads = client.list_workload(uuid=workload.uuid).data
    assert len(workloads) == 1
    wl = workloads[0]
    while wl.state != "active":
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        workloads = client.list_workload(uuid=workload.uuid).data
        assert len(workloads) == 1
        wl = workloads[0]
    return wl


def wait_for_ingress_to_active(client, ingress, timeout=DEFAULT_TIMEOUT):
    start = time.time()
    ingresses = client.list_ingress(uuid=ingress.uuid).data
    assert len(ingresses) == 1
    wl = ingresses[0]
    while wl.state != "active":
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        ingresses = client.list_ingress(uuid=ingress.uuid).data
        assert len(ingresses) == 1
        wl = ingresses[0]
    return wl


def wait_for_wl_transitioning(client, workload, timeout=DEFAULT_TIMEOUT,
                              state="error"):
    start = time.time()
    workloads = client.list_workload(uuid=workload.uuid).data
    assert len(workloads) == 1
    wl = workloads[0]
    while wl.transitioning != state:
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        workloads = client.list_workload(uuid=workload.uuid).data
        assert len(workloads) == 1
        wl = workloads[0]
    return wl


def wait_for_pod_to_running(client, pod, timeout=DEFAULT_TIMEOUT):
    start = time.time()
    pods = client.list_pod(uuid=pod.uuid).data
    assert len(pods) == 1
    p = pods[0]
    while p.state != "running":
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        pods = client.list_pod(uuid=pod.uuid).data
        assert len(pods) == 1
        p = pods[0]
    return p


def get_schedulable_nodes(cluster):
    client = get_admin_client()
    nodes = client.list_node(clusterId=cluster.id).data
    schedulable_nodes = []
    for node in nodes:
        if node.worker:
            schedulable_nodes.append(node)
    return schedulable_nodes


def get_role_nodes(cluster, role):
    etcd_nodes = []
    control_nodes = []
    worker_nodes = []
    node_list = []
    client = get_admin_client()
    nodes = client.list_node(clusterId=cluster.id).data
    for node in nodes:
        if node.etcd:
            etcd_nodes.append(node)
        if node.controlPlane:
            control_nodes.append(node)
        if node.worker:
            worker_nodes.append(node)
    if role == "etcd":
        node_list = etcd_nodes
    if role == "control":
        node_list = control_nodes
    if role == "worker":
        node_list = worker_nodes
    return node_list


def validate_ingress(p_client, cluster, workloads, host, path,
                     insecure_redirect=False):
    time.sleep(10)
    curl_args = " "
    if (insecure_redirect):
        curl_args = " -L --insecure "
    if len(host) > 0:
        curl_args += " --header 'Host: " + host + "'"
    nodes = get_schedulable_nodes(cluster)
    target_name_list = get_target_names(p_client, workloads)
    for node in nodes:
        host_ip = node.externalIpAddress
        cmd = curl_args + " http://" + host_ip + path
        validate_http_response(cmd, target_name_list)


def validate_ingress_using_endpoint(p_client, ingress, workloads,
                                    timeout=300):
    target_name_list = get_target_names(p_client, workloads)
    start = time.time()
    fqdn_available = False
    url = None
    while not fqdn_available:
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for endpoint to be available")
        time.sleep(.5)
        ingress_list = p_client.list_ingress(uuid=ingress.uuid).data
        assert len(ingress_list) == 1
        ingress = ingress_list[0]
        if hasattr(ingress, 'publicEndpoints'):
            for public_endpoint in ingress.publicEndpoints:
                if public_endpoint["hostname"].startswith(ingress.name):
                    fqdn_available = True
                    url = \
                        public_endpoint["protocol"].lower() + "://" + \
                        public_endpoint["hostname"]
                    if "path" in public_endpoint.keys():
                        url += public_endpoint["path"]
    time.sleep(10)
    validate_http_response(url, target_name_list)


def get_target_names(p_client, workloads):
    pods = []
    for workload in workloads:
        pod_list = p_client.list_pod(workloadId=workload.id).data
        pods.extend(pod_list)
    target_name_list = []
    for pod in pods:
        target_name_list.append(pod.name)
    print("target name list:" + str(target_name_list))
    return target_name_list


def get_endpoint_url_for_workload(p_client, workload, timeout=600):
    fqdn_available = False
    url = ""
    start = time.time()
    while not fqdn_available:
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for endpoint to be available")
        time.sleep(.5)
        workload_list = p_client.list_workload(uuid=workload.uuid).data
        assert len(workload_list) == 1
        workload = workload_list[0]
        if hasattr(workload, 'publicEndpoints'):
            assert len(workload.publicEndpoints) > 0
            url = "http://"
            url = url + workload.publicEndpoints[0]["addresses"][0] + ":"
            url = url + str(workload.publicEndpoints[0]["port"])
            fqdn_available = True
    return url


def wait_until_lb_is_active(url, timeout=300):
    start = time.time()
    while check_for_no_access(url):
        time.sleep(.5)
        print("No access yet")
        if time.time() - start > timeout:
            raise Exception('Timed out waiting for LB to become active')
    return


def check_for_no_access(url):
    try:
        requests.get(url)
        return False
    except requests.ConnectionError:
        print("Connection Error - " + url)
        return True


def validate_http_response(cmd, target_name_list, client_pod=None):
    target_hit_list = target_name_list[:]
    count = 5 * len(target_name_list)
    for i in range(1, count):
        if len(target_hit_list) == 0:
            break
        if client_pod is None:
            curl_cmd = "curl " + cmd
            result = run_command(curl_cmd)
        else:
            wget_cmd = "wget -qO- " + cmd
            result = kubectl_pod_exec(client_pod, wget_cmd)
            result = result.decode()
        result = result.rstrip()
        print("cmd: \t" + cmd)
        print("result: \t" + result)
        assert result in target_name_list
        if result in target_hit_list:
            target_hit_list.remove(result)
    print("After removing all, the rest is: ", target_hit_list)
    assert len(target_hit_list) == 0


def validate_cluster(client, cluster, intermediate_state="provisioning",
                     check_intermediate_state=True, skipIngresscheck=True,
                     nodes_not_in_active_state=[], k8s_version=""):
    cluster = validate_cluster_state(
        client, cluster,
        check_intermediate_state=check_intermediate_state,
        intermediate_state=intermediate_state,
        nodes_not_in_active_state=nodes_not_in_active_state)
    # Create Daemon set workload and have an Ingress with Workload
    # rule pointing to this daemonset
    create_kubeconfig(cluster)
    if k8s_version != "":
        check_cluster_version(cluster, k8s_version)
    if hasattr(cluster, 'rancherKubernetesEngineConfig'):
        check_cluster_state(len(get_role_nodes(cluster, "etcd")))
    project, ns = create_project_and_ns(ADMIN_TOKEN, cluster)
    p_client = get_project_client_for_token(project, ADMIN_TOKEN)
    con = [{"name": "test1",
            "image": TEST_IMAGE}]
    name = random_test_name("default")
    workload = p_client.create_workload(name=name,
                                        containers=con,
                                        namespaceId=ns.id,
                                        daemonSetConfig={})
    validate_workload(p_client, workload, "daemonSet", ns.name,
                      len(get_schedulable_nodes(cluster)))
    if not skipIngresscheck:
        host = "test" + str(random_int(10000, 99999)) + ".com"
        path = "/name.html"
        rule = {"host": host,
                "paths":
                    [{"workloadIds": [workload.id], "targetPort": "80"}]}
        ingress = p_client.create_ingress(name=name,
                                          namespaceId=ns.id,
                                          rules=[rule])
        wait_for_ingress_to_active(p_client, ingress)
        validate_ingress(p_client, cluster, [workload], host, path)
    return cluster


def check_cluster_version(cluster, version):
    cluster_k8s_version = \
        cluster.appliedSpec["rancherKubernetesEngineConfig"][
            "kubernetesVersion"]
    assert cluster_k8s_version == version, \
        "cluster_k8s_version: " + cluster_k8s_version + \
        " Expected: " + version
    expected_k8s_version = version[:version.find("-")]
    k8s_version = execute_kubectl_cmd("version")
    kubectl_k8s_version = k8s_version["serverVersion"]["gitVersion"]
    assert kubectl_k8s_version == expected_k8s_version, \
        "kubectl version: " + kubectl_k8s_version + \
        " Expected: " + expected_k8s_version


def check_cluster_state(etcd_count):
    css_resp = execute_kubectl_cmd("get cs")
    css = css_resp["items"]
    components = ["scheduler", "controller-manager"]
    for i in range(0, etcd_count):
        components.append("etcd-" + str(i))
    print("components to check - " + str(components))
    for cs in css:
        component_name = cs["metadata"]["name"]
        assert component_name in components
        components.remove(component_name)
        assert cs["conditions"][0]["status"] == "True"
        assert cs["conditions"][0]["type"] == "Healthy"
    assert len(components) == 0


def validate_dns_record(pod, record, expected):
    # requires pod with `dig` available - TEST_IMAGE
    host = '{0}.{1}.svc.cluster.local'.format(
        record["name"], record["namespaceId"])
    validate_dns_entry(pod, host, expected)


def validate_dns_entry(pod, host, expected):
    # requires pod with `dig` available - TEST_IMAGE
    cmd = 'ping -c 1 -W 1 {0}'.format(host)
    ping_output = kubectl_pod_exec(pod, cmd)

    ping_validation_pass = False
    for expected_value in expected:
        if expected_value in str(ping_output):
            ping_validation_pass = True
            break

    assert ping_validation_pass is True
    assert " 0% packet loss" in str(ping_output)

    dig_cmd = 'dig {0} +short'.format(host)
    dig_output = kubectl_pod_exec(pod, dig_cmd)

    for expected_value in expected:
        assert expected_value in str(dig_output)


def wait_for_nodes_to_become_active(client, cluster, exception_list=[],
                                    retry_count=0):
    nodes = client.list_node(clusterId=cluster.id).data
    node_auto_deleted = False
    for node in nodes:
        if node.requestedHostname not in exception_list:
            node = wait_for_node_status(client, node, "active")
            if node is None:
                print("Need to re-evalauate new node list")
                node_auto_deleted = True
                retry_count += 1
                print("Retry Count:" + str(retry_count))
    if node_auto_deleted and retry_count < 5:
        wait_for_nodes_to_become_active(client, cluster, exception_list,
                                        retry_count)


def wait_for_node_status(client, node, state):
    uuid = node.uuid
    start = time.time()
    nodes = client.list_node(uuid=uuid).data
    node_count = len(nodes)
    # Handle the case of nodes getting auto deleted when they are part of
    # nodepools
    if node_count == 1:
        node_status = nodes[0].state
    else:
        print("Node does not exist anymore -" + uuid)
        return None
    while node_status != state:
        if time.time() - start > MACHINE_TIMEOUT:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(5)
        nodes = client.list_node(uuid=uuid).data
        node_count = len(nodes)
        if node_count == 1:
            node_status = nodes[0].state
        else:
            print("Node does not exist anymore -" + uuid)
            return None
    return node


def wait_for_node_to_be_deleted(client, node, timeout=300):
    uuid = node.uuid
    start = time.time()
    nodes = client.list_node(uuid=uuid).data
    node_count = len(nodes)
    while node_count != 0:
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        nodes = client.list_node(uuid=uuid).data
        node_count = len(nodes)


def wait_for_cluster_node_count(client, cluster, expected_node_count,
                                timeout=300):
    start = time.time()
    nodes = client.list_node(clusterId=cluster.id).data
    node_count = len(nodes)
    while node_count != expected_node_count:
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        nodes = client.list_node(clusterId=cluster.id).data
        node_count = len(nodes)


def get_custom_host_registration_cmd(client, cluster, roles, node):
    allowed_roles = ["etcd", "worker", "controlplane"]
    cluster_tokens = client.list_cluster_registration_token(
        clusterId=cluster.id).data
    if len(cluster_tokens) > 0:
        cluster_token = cluster_tokens[0]
    else:
        cluster_token = create_custom_host_registration_token(client, cluster)
    cmd = cluster_token.nodeCommand
    for role in roles:
        assert role in allowed_roles
        cmd += " --" + role
    additional_options = " --address " + node.public_ip_address + \
                         " --internal-address " + node.private_ip_address
    cmd += additional_options
    return cmd


def create_custom_host_registration_token(client, cluster):
    cluster_token = client.create_cluster_registration_token(
        clusterId=cluster.id)
    cluster_token = client.wait_success(cluster_token)
    assert cluster_token.state == 'active'
    return cluster_token


def get_cluster_type(client, cluster):
    cluster_configs = [
        "amazonElasticContainerServiceConfig",
        "azureKubernetesServiceConfig",
        "googleKubernetesEngineConfig",
        "rancherKubernetesEngineConfig"
    ]
    if "rancherKubernetesEngineConfig" in cluster:
        nodes = client.list_node(clusterId=cluster.id).data
        if len(nodes) > 0:
            if nodes[0].nodeTemplateId is None:
                return "Custom"
    for cluster_config in cluster_configs:
        if cluster_config in cluster:
            return cluster_config
    return "Imported"


def delete_cluster(client, cluster):
    nodes = client.list_node(clusterId=cluster.id).data
    # Delete Cluster
    client.delete(cluster)
    # Delete nodes(in cluster) from AWS for Imported and Custom Cluster
    if (len(nodes) > 0):
        cluster_type = get_cluster_type(client, cluster)
        print(cluster_type)
        if get_cluster_type(client, cluster) in ["Imported", "Custom"]:
            nodes = client.list_node(clusterId=cluster.id).data
            filters = [
                {'Name': 'tag:Name',
                 'Values': ['testcustom*', 'teststess*']}]
            ip_filter = {}
            ip_list = []
            ip_filter['Name'] = \
                'network-interface.addresses.association.public-ip'
            ip_filter['Values'] = ip_list
            filters.append(ip_filter)
            for node in nodes:
                ip_list.append(node.externalIpAddress)
            assert len(ip_filter) > 0
            print(ip_filter)
            aws_nodes = AmazonWebServices().get_nodes(filters)
            for node in aws_nodes:
                print(node.public_ip_address)
                AmazonWebServices().delete_nodes(aws_nodes)


def check_connectivity_between_workloads(p_client1, workload1, p_client2,
                                         workload2, allow_connectivity=True):
    wl1_pods = p_client1.list_pod(workloadId=workload1.id).data
    wl2_pods = p_client2.list_pod(workloadId=workload2.id).data
    for pod in wl1_pods:
        for o_pod in wl2_pods:
            check_connectivity_between_pods(pod, o_pod, allow_connectivity)


def check_connectivity_between_workload_pods(p_client, workload):
    pods = p_client.list_pod(workloadId=workload.id).data
    for pod in pods:
        for o_pod in pods:
            check_connectivity_between_pods(pod, o_pod)


def check_connectivity_between_pods(pod1, pod2, allow_connectivity=True):
    pod_ip = pod2.status.podIp

    cmd = "ping -c 1 -W 1 " + pod_ip
    response = kubectl_pod_exec(pod1, cmd)
    print("Actual ping Response from " + pod1.name + ":" + str(response))
    if allow_connectivity:
        assert pod_ip in str(response) and " 0% packet loss" in str(response)
    else:
        assert pod_ip in str(response) and " 100% packet loss" in str(response)


def kubectl_pod_exec(pod, cmd):
    command = "exec " + pod.name + " -n " + pod.namespaceId + " -- " + cmd
    return execute_kubectl_cmd(command, json_out=False, stderr=True)


def exec_shell_command(ip, port, cmd, password):
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(ip, username="root", password=password, port=port)
    stdin, stdout, stderr = ssh.exec_command(cmd)
    response = stdout.readlines()
    return response


def wait_for_ns_to_become_active(client, ns, timeout=DEFAULT_TIMEOUT):
    start = time.time()
    time.sleep(2)
    nss = client.list_namespace(uuid=ns.uuid).data
    assert len(nss) == 1
    ns = nss[0]
    while ns.state != "active":
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        nss = client.list_namespace(uuid=ns.uuid).data
        assert len(nss) == 1
        ns = nss[0]
    return ns


def wait_for_pod_images(p_client, workload, ns_name, expectedimage, numofpods,
                        timeout=DEFAULT_TIMEOUT):
    start = time.time()

    for key, value in workload.workloadLabels.items():
        label = key + "=" + value
    get_pods = "get pods -l" + label + " -n " + ns_name
    pods = execute_kubectl_cmd(get_pods)

    for x in range(0, numofpods - 1):
        pod = pods["items"][x]
        podimage = pod["spec"]["containers"][0]["image"]
        while podimage != expectedimage:
            if time.time() - start > timeout:
                raise AssertionError(
                    "Timed out waiting for correct pod images")
            time.sleep(.5)
            pods = execute_kubectl_cmd(get_pods)
            pod = pods["items"][x]
            podimage = pod["spec"]["containers"][0]["image"]


def wait_for_pods_in_workload(p_client, workload, pod_count,
                              timeout=DEFAULT_TIMEOUT):
    start = time.time()
    pods = p_client.list_pod(workloadId=workload.id).data
    while len(pods) != pod_count:
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        pods = p_client.list_pod(workloadId=workload.id).data
    return pods


def get_admin_client_and_cluster():
    client = get_admin_client()
    if CLUSTER_NAME == "":
        clusters = client.list_cluster().data
    else:
        clusters = client.list_cluster(name=CLUSTER_NAME).data
    assert len(clusters) > 0
    cluster = clusters[0]
    return client, cluster


def validate_cluster_state(client, cluster,
                           check_intermediate_state=True,
                           intermediate_state="provisioning",
                           nodes_not_in_active_state=[]):
    if check_intermediate_state:
        cluster = wait_for_condition(
            client, cluster,
            lambda x: x.state == intermediate_state,
            lambda x: 'State is: ' + x.state,
            timeout=MACHINE_TIMEOUT)
        assert cluster.state == intermediate_state
    cluster = wait_for_condition(
        client, cluster,
        lambda x: x.state == "active",
        lambda x: 'State is: ' + x.state,
        timeout=MACHINE_TIMEOUT)
    assert cluster.state == "active"
    wait_for_nodes_to_become_active(client, cluster,
                                    exception_list=nodes_not_in_active_state)
    return cluster


def wait_until_available(client, obj, timeout=DEFAULT_TIMEOUT):
    start = time.time()
    sleep = 0.01
    while True:
        time.sleep(sleep)
        sleep *= 2
        if sleep > 2:
            sleep = 2
        try:
            obj = client.reload(obj)
        except ApiError as e:
            if e.error.status != 403:
                raise e
        else:
            return obj
        delta = time.time() - start
        if delta > timeout:
            msg = 'Timeout waiting for [{}:{}] for condition after {}' \
                  ' seconds'.format(obj.type, obj.id, delta)
            raise Exception(msg)


def delete_node(aws_nodes):
    for node in aws_nodes:
        AmazonWebServices().delete_node(node)


def cluster_cleanup(client, cluster, aws_nodes=None):
    if RANCHER_CLEANUP_CLUSTER:
        client.delete(cluster)
        if aws_nodes is not None:
            delete_node(aws_nodes)
    else:
        env_details = "env.CATTLE_TEST_URL='" + CATTLE_TEST_URL + "'\n"
        env_details += "env.ADMIN_TOKEN='" + ADMIN_TOKEN + "'\n"
        env_details += "env.CLUSTER_NAME='" + cluster.name + "'\n"
        create_config_file(env_details)


def create_config_file(env_details):
    file = open(env_file, "w")
    file.write(env_details)
    file.close()


def validate_hostPort(p_client, workload, source_port, cluster):
    pods = p_client.list_pod(workloadId=workload.id).data
    nodes = get_schedulable_nodes(cluster)
    for node in nodes:
        target_name_list = []
        for pod in pods:
            print(pod.nodeId + " check " + node.id)
            if pod.nodeId == node.id:
                target_name_list.append(pod.name)
                break
        host_ip = node.externalIpAddress
        curl_cmd = " http://" + host_ip + ":" + \
                   str(source_port) + "/name.html"
        validate_http_response(curl_cmd, target_name_list)


def validate_lb(p_client, workload):
    url = get_endpoint_url_for_workload(p_client, workload)
    target_name_list = get_target_names(p_client, [workload])
    wait_until_lb_is_active(url)
    validate_http_response(url + "/name.html", target_name_list)


def validate_nodePort(p_client, workload, cluster):
    source_port = workload.publicEndpoints[0]["port"]
    nodes = get_schedulable_nodes(cluster)
    pods = p_client.list_pod(workloadId=workload.id).data
    target_name_list = []
    for pod in pods:
        target_name_list.append(pod.name)
    print("target name list:" + str(target_name_list))
    for node in nodes:
        host_ip = node.externalIpAddress
        curl_cmd = " http://" + host_ip + ":" + \
                   str(source_port) + "/name.html"
        validate_http_response(curl_cmd, target_name_list)


def validate_clusterIp(p_client, workload, cluster_ip, test_pods):
    pods = p_client.list_pod(workloadId=workload.id).data
    target_name_list = []
    for pod in pods:
        target_name_list.append(pod["name"])
    curl_cmd = "http://" + cluster_ip + "/name.html"
    for pod in test_pods:
        validate_http_response(curl_cmd, target_name_list, pod)


def wait_for_pv_to_be_available(c_client, pv_object, timeout=DEFAULT_TIMEOUT):
    start = time.time()
    time.sleep(2)
    list = c_client.list_persistent_volume(uuid=pv_object.uuid).data
    assert len(list) == 1
    pv = list[0]
    while pv.state != "available":
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to available")
        time.sleep(.5)
        list = c_client.list_persistent_volume(uuid=pv_object.uuid).data
        assert len(list) == 1
        pv = list[0]
    return pv


def wait_for_pvc_to_be_bound(p_client, pvc_object, timeout=DEFAULT_TIMEOUT):
    start = time.time()
    time.sleep(2)
    list = p_client.list_persistent_volume_claim(uuid=pvc_object.uuid).data
    assert len(list) == 1
    pvc = list[0]
    while pvc.state != "bound":
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to bound")
        time.sleep(.5)
        list = p_client.list_persistent_volume_claim(uuid=pvc_object.uuid).data
        assert len(list) == 1
        pvc = list[0]
    return pvc


def create_wl_with_nfs(p_client, ns_id, pvc_name, wl_name,
                       mount_path, sub_path, is_daemonSet=False):
    volumes = [{"type": "volume",
                "name": "vol1",
                "persistentVolumeClaim": {
                    "readOnly": "false",
                    "type": "persistentVolumeClaimVolumeSource",
                    "persistentVolumeClaimId": pvc_name
                }}]
    volumeMounts = [{"readOnly": "False",
                     "type": "volumeMount",
                     "mountPath": mount_path,
                     "subPath": sub_path,
                     "name": "vol1"
                     }]
    con = [{"name": "test1",
            "image": TEST_IMAGE,
            "volumeMounts": volumeMounts
            }]
    if is_daemonSet:
        workload = p_client.create_workload(name=wl_name,
                                            containers=con,
                                            namespaceId=ns_id,
                                            volumes=volumes,
                                            daemonSetConfig={})
    else:
        workload = p_client.create_workload(name=wl_name,
                                            containers=con,
                                            namespaceId=ns_id,
                                            volumes=volumes)
    return workload


def write_content_to_file(pod, content, filename):
    cmd_write = "/bin/bash -c 'echo {1} > {0}'".format(filename, content)
    output = kubectl_pod_exec(pod, cmd_write)
    assert output.strip().decode('utf-8') == ""


def validate_file_content(pod, content, filename):
    cmd_get_content = "/bin/bash -c 'cat {0}' ".format(filename)
    output = kubectl_pod_exec(pod, cmd_get_content)
    assert output.strip().decode('utf-8') == content


def wait_for_mcapp_to_active(client, multiClusterApp, timeout=DEFAULT_MULTI_CLUSTER_APP_TIMEOUT):
    print("\nuuid:")
    print(multiClusterApp.uuid)
    time.sleep(5)
    mcapps = client.list_multiClusterApp(uuid=multiClusterApp.uuid, name=multiClusterApp.name).data
    start = time.time()
    assert len(mcapps) == 1
    mapp = mcapps[0]
    print(mapp.state)
    while mapp.state != "active":
        print(mapp.uuid)
        print(mapp.state)
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        multiclusterapps = client.list_multiClusterApp(uuid=multiClusterApp.uuid, name=multiClusterApp.name).data
        assert len(multiclusterapps) == 1
        mapp = multiclusterapps[0]
    return mapp


def validate_mcapp_cluster(app_id, p_client):
    mcapp = p_client.list_app(name=app_id).data
    assert len(mcapp) == 1
    app = mcapp[0]
    return app


def wait_for_mcapp_cluster_level_to_active(client, app_id, timeout=DEFAULT_MULTI_CLUSTER_APP_TIMEOUT):
    mcapps = client.list_app(name=app_id).data
    start = time.time()
    assert len(mcapps) == 1
    mapp = mcapps[0]
    while mapp.state != "active":
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for state to get to active")
        time.sleep(.5)
        apps = client.list_app(name=app_id).data
        assert len(apps) == 1
        mapp = apps[0]
    return mapp


def get_admin_client_and_cluster_mcapp():
    clusters = []
    client = get_admin_client()
    if CLUSTER_NAME == "" or CLUSTER_NAME_2 == "":
        clusters = client.list_cluster().data
    else:
        clusters.append(client.list_cluster(name=CLUSTER_NAME).data)
        clusters.append(client.list_cluster(name=CLUSTER_NAME_2).data)
    assert len(clusters) == 2
    return client, clusters


def validate_multi_cluster_app_cluster(app_id1, app_id2, p_client1, p_client2):
    validate_mcapp_cluster(app_id1, p_client1)
    if app_id2 != "":
        validate_mcapp_cluster(app_id2, p_client2)
    # verify app in cluster is active or not
    wait_for_mcapp_cluster_level_to_active(p_client1, app_id1)
    if app_id2 != "":
        wait_for_mcapp_cluster_level_to_active(p_client2, app_id2)
[(17, 'os.environ.get', 'os.environ.get', False, 'import os\n'), (18, 'os.environ.get', 'os.environ.get', False, 'import os\n'), (27, 'os.environ.get', 'os.environ.get', False, 'import os\n'), (24, 'os.environ.get', 'os.environ.get', False, 'import os\n'), (29, 'os.environ.get', 'os.environ.get', False, 'import os\n'), (40, 'random.randint', 'random.randint', False, 'import random\n'), (44, 'random.randint', 'random.randint', False, 'import random\n'), (52, 'rancher.Client', 'rancher.Client', False, 'import rancher\n'), (56, 'rancher.Client', 'rancher.Client', False, 'import rancher\n'), (61, 'rancher.Client', 'rancher.Client', False, 'import rancher\n'), (67, 'rancher.Client', 'rancher.Client', False, 'import rancher\n'), (73, 'rancher.Client', 'rancher.Client', False, 'import rancher\n'), (84, 'time.time', 'time.time', False, 'import time\n'), (100, 'time.time', 'time.time', False, 'import time\n'), (130, 'time.sleep', 'time.sleep', False, 'import time\n'), (319, 'subprocess.check_output', 'subprocess.check_output', False, 'import subprocess\n'), (335, 'time.time', 'time.time', False, 'import time\n'), (351, 'time.time', 'time.time', False, 'import time\n'), (368, 'time.time', 'time.time', False, 'import time\n'), (384, 'time.time', 'time.time', False, 'import time\n'), (434, 'time.sleep', 'time.sleep', False, 'import time\n'), (451, 'time.time', 'time.time', False, 'import time\n'), (471, 'time.sleep', 'time.sleep', False, 'import time\n'), (490, 'time.time', 'time.time', False, 'import time\n'), (509, 'time.time', 'time.time', False, 'import time\n'), (668, 'time.time', 'time.time', False, 'import time\n'), (695, 'time.time', 'time.time', False, 'import time\n'), (709, 'time.time', 'time.time', False, 'import time\n'), (828, 'paramiko.SSHClient', 'paramiko.SSHClient', False, 'import paramiko\n'), (837, 'time.time', 'time.time', False, 'import time\n'), (838, 'time.sleep', 'time.sleep', False, 'import time\n'), (855, 'time.time', 'time.time', False, 'import time\n'), (877, 'time.time', 'time.time', False, 'import time\n'), (922, 'time.time', 'time.time', False, 'import time\n'), (1015, 'time.time', 'time.time', False, 'import time\n'), (1016, 'time.sleep', 'time.sleep', False, 'import time\n'), (1032, 'time.time', 'time.time', False, 'import time\n'), (1033, 'time.sleep', 'time.sleep', False, 'import time\n'), (1096, 'time.sleep', 'time.sleep', False, 'import time\n'), (1098, 'time.time', 'time.time', False, 'import time\n'), (1124, 'time.time', 'time.time', False, 'import time\n'), (22, 'os.path.realpath', 'os.path.realpath', False, 'import os\n'), (31, 'os.path.realpath', 'os.path.realpath', False, 'import os\n'), (94, 'time.sleep', 'time.sleep', False, 'import time\n'), (103, 'time.sleep', 'time.sleep', False, 'import time\n'), (220, 'time.sleep', 'time.sleep', False, 'import time\n'), (313, 'json.loads', 'json.loads', False, 'import json\n'), (324, 'subprocess.check_output', 'subprocess.check_output', False, 'import subprocess\n'), (343, 'time.sleep', 'time.sleep', False, 'import time\n'), (359, 'time.sleep', 'time.sleep', False, 'import time\n'), (376, 'time.sleep', 'time.sleep', False, 'import time\n'), (392, 'time.sleep', 'time.sleep', False, 'import time\n'), (458, 'time.sleep', 'time.sleep', False, 'import time\n'), (495, 'time.sleep', 'time.sleep', False, 'import time\n'), (511, 'time.sleep', 'time.sleep', False, 'import time\n'), (520, 'requests.get', 'requests.get', False, 'import requests\n'), (682, 'time.sleep', 'time.sleep', False, 'import time\n'), (702, 'time.sleep', 'time.sleep', False, 'import time\n'), (716, 'time.sleep', 'time.sleep', False, 'import time\n'), (829, 'paramiko.AutoAddPolicy', 'paramiko.AutoAddPolicy', False, 'import paramiko\n'), (846, 'time.sleep', 'time.sleep', False, 'import time\n'), (883, 'time.sleep', 'time.sleep', False, 'import time\n'), (925, 'time.sleep', 'time.sleep', False, 'import time\n'), (1024, 'time.sleep', 'time.sleep', False, 'import time\n'), (1041, 'time.sleep', 'time.sleep', False, 'import time\n'), (1108, 'time.sleep', 'time.sleep', False, 'import time\n'), (1131, 'time.sleep', 'time.sleep', False, 'import time\n'), (36, 'time.time', 'time.time', False, 'import time\n'), (869, 'time.sleep', 'time.sleep', False, 'import time\n'), (936, 'time.time', 'time.time', False, 'import time\n'), (87, 'time.time', 'time.time', False, 'import time\n'), (90, 'inspect.getsource', 'inspect.getsource', False, 'import inspect\n'), (104, 'time.time', 'time.time', False, 'import time\n'), (340, 'time.time', 'time.time', False, 'import time\n'), (356, 'time.time', 'time.time', False, 'import time\n'), (373, 'time.time', 'time.time', False, 'import time\n'), (389, 'time.time', 'time.time', False, 'import time\n'), (455, 'time.time', 'time.time', False, 'import time\n'), (492, 'time.time', 'time.time', False, 'import time\n'), (513, 'time.time', 'time.time', False, 'import time\n'), (679, 'time.time', 'time.time', False, 'import time\n'), (699, 'time.time', 'time.time', False, 'import time\n'), (713, 'time.time', 'time.time', False, 'import time\n'), (843, 'time.time', 'time.time', False, 'import time\n'), (880, 'time.time', 'time.time', False, 'import time\n'), (945, 'lib.aws.AmazonWebServices', 'AmazonWebServices', False, 'from lib.aws import AmazonWebServices\n'), (1021, 'time.time', 'time.time', False, 'import time\n'), (1038, 'time.time', 'time.time', False, 'import time\n'), (1105, 'time.time', 'time.time', False, 'import time\n'), (1128, 'time.time', 'time.time', False, 'import time\n'), (788, 'lib.aws.AmazonWebServices', 'AmazonWebServices', False, 'from lib.aws import AmazonWebServices\n'), (866, 'time.time', 'time.time', False, 'import time\n'), (791, 'lib.aws.AmazonWebServices', 'AmazonWebServices', False, 'from lib.aws import AmazonWebServices\n')]