func (mpa *MasterProvisionerAliyun) CreateVirtualCluster(vc *tenancyv1alpha1.Virtualcluster) error {
	log.Info("setting up control plane for the Virtualcluster", "Virtualcluster", vc.Name)
	// 1. load aliyun accessKeyID/accessKeySecret from secret
	aliyunAKID, aliyunAKSrt, err := aliyunutil.GetAliyunAKPair(mpa, log)
	if err != nil {
		return err
	}

	askCfg, err := aliyunutil.GetASKConfigs(mpa, log)
	if err != nil {
		return err
	}

	// 2. send ASK creation request
	// NOTE http requests of a creation action will be sent by a same client
	cli, err := sdk.NewClientWithAccessKey(askCfg.RegionID, aliyunAKID, aliyunAKSrt)
	if err != nil {
		return err
	}

	var (
		clsID    string
		clsState string
		clsSlbId string
	)
	creationTimeout := time.After(600 * time.Second)
	clsID, err = aliyunutil.SendCreationRequest(cli, vc.Name, askCfg)
	if err != nil {
		if !aliyunutil.IsSDKErr(err) {
			return err
		}
		// check SDK error code
		if aliyunutil.GetSDKErrCode(err) == aliyunutil.ClusterNameAlreadyExist {
			// clusterName already exists, query Aliyun to get the clusterID
			// corresponding to the clusterName
			var getClsIDErr error
			clsID, getClsIDErr = aliyunutil.GetClusterIDByName(cli, vc.Name, askCfg.RegionID)
			if getClsIDErr != nil {
				return getClsIDErr
			}
			var getStErr error
			clsSlbId, clsState, getStErr = aliyunutil.GetASKStateAndSlbID(cli, clsID, askCfg.RegionID)
			if getStErr != nil {
				return getStErr
			}

			if clsState != "running" && clsState != "initial" {
				return fmt.Errorf("unknown ASK(%s) state: %s", vc.Name, clsState)
			}
		}
	}

	log.Info("creating the ASK", "ASK-ID", clsID)

	// 3. block until the newly created ASK is up and running
PollASK:
	for {
		select {
		case <-time.After(10 * time.Second):
			if clsState == "running" {
				// ASK is up and running, stop polling
				log.Info("ASK is up and running", "ASK-ID", clsID)
				break PollASK
			}
			var getStErr error
			clsSlbId, clsState, getStErr = aliyunutil.GetASKStateAndSlbID(cli, clsID, askCfg.RegionID)
			if getStErr != nil {
				return getStErr
			}

		case <-creationTimeout:
			return fmt.Errorf("creating cluster(%s) timeout", clsID)
		}
	}

	// 4. create the root namesapce of the Virtualcluster
	vcNs, err := kubeutil.CreateRootNS(mpa, vc)
	if err != nil {
		return err
	}
	log.Info("virtualcluster root ns is created", "ns", vcNs)

	// 5. get kubeconfig of the newly created ASK
	kbCfg, err := aliyunutil.GetASKKubeConfig(cli, clsID, askCfg.RegionID, askCfg.PrivateKbCfg)
	if err != nil {
		return err
	}
	log.Info("got kubeconfig of cluster", "cluster", clsID)

	// 6. serialize kubeconfig to secret and store it in the
	// corresponding namespace (i.e.)
	adminSrt := secret.KubeconfigToSecret(secret.AdminSecretName,
		vcNs, kbCfg)
	err = mpa.Create(context.TODO(), adminSrt)
	if err != nil && !apierrors.IsAlreadyExists(err) {
		return err
	}
	log.Info("admin kubeconfig is created for virtualcluster", "vc", vc.GetName())

	// 7. add annotations on vc cr, including,
	// tenancy.x-k8s.io/ask.clusterID,
	// tenancy.x-k8s.io/ask.slbID,
	// tenancy.x-k8s.io/cluster,
	// tenancy.x-k8s.io/admin-kubeconfig
	err = kubeutil.AnnotateVC(mpa, vc, aliyunutil.AnnotationSlbID, clsSlbId, log)
	if err != nil {
		return err
	}
	log.Info("slb id has been added to vc as an annotation", "vc", vc.GetName(), "id", clsSlbId)
	err = kubeutil.AnnotateVC(mpa, vc, aliyunutil.AnnotationClusterID, clsID, log)
	if err != nil {
		return err
	}
	log.Info("cluster ID has been added to vc as an annotation", "vc", vc.GetName(), "cluster-id", clsID)
	kbCfgB64 := base64.StdEncoding.EncodeToString([]byte(kbCfg))
	err = kubeutil.AnnotateVC(mpa, vc, aliyunutil.AnnotationKubeconfig, kbCfgB64, log)
	if err != nil {
		return err
	}
	log.Info("admin-kubeconfig has been added to vc as an annotation", "vc", vc.GetName())
	// the clusterkey is the vc root ns
	err = kubeutil.AnnotateVC(mpa, vc, constants.LabelCluster, vcNs, log)
	if err != nil {
		return err
	}
	log.Info("cluster key has been added to vc as an annotation", "vc", vc.GetName(), "cluster-key", vcNs)

	// 8. delete the node-controller service account to disable the
	// node periodic check
	tenantCliCfg, err := clientcmd.RESTConfigFromKubeConfig([]byte(kbCfg))
	if err != nil {
		return err
	}
	tenantCli, err := kubernetes.NewForConfig(tenantCliCfg)
	if err != nil {
		return err
	}
	// delete if exist
	if err = tenantCli.CoreV1().ServiceAccounts("kube-system").
		Delete("node-controller", &metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
		return err
	}
	log.Info("the node selector service account is deleted", "vc", vc.GetName())
	return nil
}
// CreateVirtualCluster creates a new ASK on aliyun for given Virtualcluster
CreateVirtualCluster creates a new ASK on aliyun for given Virtualcluster
// CreateVirtualCluster creates a new ASK on aliyun for given Virtualcluster
func (mpa *MasterProvisionerAliyun) CreateVirtualCluster(vc *tenancyv1alpha1.Virtualcluster) error {
	log.Info("setting up control plane for the Virtualcluster", "Virtualcluster", vc.Name)
	aliyunAKID, aliyunAKSrt, err := aliyunutil.GetAliyunAKPair(mpa, log)
	if err != nil {
		return err
	}
	askCfg, err := aliyunutil.GetASKConfigs(mpa, log)
	if err != nil {
		return err
	}
	cli, err := sdk.NewClientWithAccessKey(askCfg.RegionID, aliyunAKID, aliyunAKSrt)
	if err != nil {
		return err
	}
	var (
		clsID    string
		clsState string
		clsSlbId string
	)
	creationTimeout := time.After(600 * time.Second)
	clsID, err = aliyunutil.SendCreationRequest(cli, vc.Name, askCfg)
	if err != nil {
		if !aliyunutil.IsSDKErr(err) {
			return err
		}
		if aliyunutil.GetSDKErrCode(err) == aliyunutil.ClusterNameAlreadyExist {
			var getClsIDErr error
			clsID, getClsIDErr = aliyunutil.GetClusterIDByName(cli, vc.Name, askCfg.RegionID)
			if getClsIDErr != nil {
				return getClsIDErr
			}
			var getStErr error
			clsSlbId, clsState, getStErr = aliyunutil.GetASKStateAndSlbID(cli, clsID, askCfg.RegionID)
			if getStErr != nil {
				return getStErr
			}
			if clsState != "running" && clsState != "initial" {
				return fmt.Errorf("unknown ASK(%s) state: %s", vc.Name, clsState)
			}
		}
	}
	log.Info("creating the ASK", "ASK-ID", clsID)
PollASK:
	for {
		select {
		case <-time.After(10 * time.Second):
			if clsState == "running" {
				log.Info("ASK is up and running", "ASK-ID", clsID)
				break PollASK
			}
			var getStErr error
			clsSlbId, clsState, getStErr = aliyunutil.GetASKStateAndSlbID(cli, clsID, askCfg.RegionID)
			if getStErr != nil {
				return getStErr
			}
		case <-creationTimeout:
			return fmt.Errorf("creating cluster(%s) timeout", clsID)
		}
	}
	vcNs, err := kubeutil.CreateRootNS(mpa, vc)
	if err != nil {
		return err
	}
	log.Info("virtualcluster root ns is created", "ns", vcNs)
	kbCfg, err := aliyunutil.GetASKKubeConfig(cli, clsID, askCfg.RegionID, askCfg.PrivateKbCfg)
	if err != nil {
		return err
	}
	log.Info("got kubeconfig of cluster", "cluster", clsID)
	adminSrt := secret.KubeconfigToSecret(secret.AdminSecretName,
		vcNs, kbCfg)
	err = mpa.Create(context.TODO(), adminSrt)
	if err != nil && !apierrors.IsAlreadyExists(err) {
		return err
	}
	log.Info("admin kubeconfig is created for virtualcluster", "vc", vc.GetName())
	err = kubeutil.AnnotateVC(mpa, vc, aliyunutil.AnnotationSlbID, clsSlbId, log)
	if err != nil {
		return err
	}
	log.Info("slb id has been added to vc as an annotation", "vc", vc.GetName(), "id", clsSlbId)
	err = kubeutil.AnnotateVC(mpa, vc, aliyunutil.AnnotationClusterID, clsID, log)
	if err != nil {
		return err
	}
	log.Info("cluster ID has been added to vc as an annotation", "vc", vc.GetName(), "cluster-id", clsID)
	kbCfgB64 := base64.StdEncoding.EncodeToString([]byte(kbCfg))
	err = kubeutil.AnnotateVC(mpa, vc, aliyunutil.AnnotationKubeconfig, kbCfgB64, log)
	if err != nil {
		return err
	}
	log.Info("admin-kubeconfig has been added to vc as an annotation", "vc", vc.GetName())
	err = kubeutil.AnnotateVC(mpa, vc, constants.LabelCluster, vcNs, log)
	if err != nil {
		return err
	}
	log.Info("cluster key has been added to vc as an annotation", "vc", vc.GetName(), "cluster-key", vcNs)
	tenantCliCfg, err := clientcmd.RESTConfigFromKubeConfig([]byte(kbCfg))
	if err != nil {
		return err
	}
	tenantCli, err := kubernetes.NewForConfig(tenantCliCfg)
	if err != nil {
		return err
	}
	if err = tenantCli.CoreV1().ServiceAccounts("kube-system").
		Delete("node-controller", &metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
		return err
	}
	log.Info("the node selector service account is deleted", "vc", vc.GetName())
	return nil
}
func ( mpa * MasterProvisionerAliyun ) CreateVirtualCluster ( vc * tenancyv1alpha1 . Virtualcluster ) error { log . Info ( " " , " " , vc . Name ) aliyunAKID , aliyunAKSrt , err := aliyunutil . GetAliyunAKPair ( mpa , log ) if err != nil { return err } askCfg , err := aliyunutil . GetASKConfigs ( mpa , log ) if err != nil { return err } cli , err := sdk . NewClientWithAccessKey ( askCfg . RegionID , aliyunAKID , aliyunAKSrt ) if err != nil { return err } var ( clsID string clsState string clsSlbId string ) creationTimeout := time . After ( 600 * time . Second ) clsID , err = aliyunutil . SendCreationRequest ( cli , vc . Name , askCfg ) if err != nil { if ! aliyunutil . IsSDKErr ( err ) { return err } if aliyunutil . GetSDKErrCode ( err ) == aliyunutil . ClusterNameAlreadyExist { var getClsIDErr error clsID , getClsIDErr = aliyunutil . GetClusterIDByName ( cli , vc . Name , askCfg . RegionID ) if getClsIDErr != nil { return getClsIDErr } var getStErr error clsSlbId , clsState , getStErr = aliyunutil . GetASKStateAndSlbID ( cli , clsID , askCfg . RegionID ) if getStErr != nil { return getStErr } if clsState != " " && clsState != " " { return fmt . Errorf ( " " , vc . Name , clsState ) } } } log . Info ( " " , " " , clsID ) PollASK : for { select { case <- time . After ( 10 * time . Second ) : if clsState == " " { log . Info ( " " , " " , clsID ) break PollASK } var getStErr error clsSlbId , clsState , getStErr = aliyunutil . GetASKStateAndSlbID ( cli , clsID , askCfg . RegionID ) if getStErr != nil { return getStErr } case <- creationTimeout : return fmt . Errorf ( " " , clsID ) } } vcNs , err := kubeutil . CreateRootNS ( mpa , vc ) if err != nil { return err } log . Info ( " " , " " , vcNs ) kbCfg , err := aliyunutil . GetASKKubeConfig ( cli , clsID , askCfg . RegionID , askCfg . PrivateKbCfg ) if err != nil { return err } log . Info ( " " , " " , clsID ) adminSrt := secret . KubeconfigToSecret ( secret . AdminSecretName , vcNs , kbCfg ) err = mpa . Create ( context . TODO ( ) , adminSrt ) if err != nil && ! apierrors . IsAlreadyExists ( err ) { return err } log . Info ( " " , " " , vc . GetName ( ) ) err = kubeutil . AnnotateVC ( mpa , vc , aliyunutil . AnnotationSlbID , clsSlbId , log ) if err != nil { return err } log . Info ( " " , " " , vc . GetName ( ) , " " , clsSlbId ) err = kubeutil . AnnotateVC ( mpa , vc , aliyunutil . AnnotationClusterID , clsID , log ) if err != nil { return err } log . Info ( " " , " " , vc . GetName ( ) , " " , clsID ) kbCfgB64 := base64 . StdEncoding . EncodeToString ( [ ] byte ( kbCfg ) ) err = kubeutil . AnnotateVC ( mpa , vc , aliyunutil . AnnotationKubeconfig , kbCfgB64 , log ) if err != nil { return err } log . Info ( " " , " " , vc . GetName ( ) ) err = kubeutil . AnnotateVC ( mpa , vc , constants . LabelCluster , vcNs , log ) if err != nil { return err } log . Info ( " " , " " , vc . GetName ( ) , " " , vcNs ) tenantCliCfg , err := clientcmd . RESTConfigFromKubeConfig ( [ ] byte ( kbCfg ) ) if err != nil { return err } tenantCli , err := kubernetes . NewForConfig ( tenantCliCfg ) if err != nil { return err } if err = tenantCli . CoreV1 ( ) . ServiceAccounts ( " " ) . Delete ( " " , & metav1 . DeleteOptions { } ) ; err != nil && ! apierrors . IsNotFound ( err ) { return err } log . Info ( " " , " " , vc . GetName ( ) ) return nil }
CreateVirtualCluster creates a new ASK on aliyun for given Virtualcluster
CreateVirtualCluster creates a new ASK on aliyun for given Virtualcluster
// 1. load aliyun accessKeyID/accessKeySecret from secret // 2. send ASK creation request // NOTE http requests of a creation action will be sent by a same client // check SDK error code // clusterName already exists, query Aliyun to get the clusterID // corresponding to the clusterName // 3. block until the newly created ASK is up and running // ASK is up and running, stop polling // 4. create the root namesapce of the Virtualcluster // 5. get kubeconfig of the newly created ASK // 6. serialize kubeconfig to secret and store it in the // corresponding namespace (i.e.) // 7. add annotations on vc cr, including, // tenancy.x-k8s.io/ask.clusterID, // tenancy.x-k8s.io/ask.slbID, // tenancy.x-k8s.io/cluster, // tenancy.x-k8s.io/admin-kubeconfig // the clusterkey is the vc root ns // 8. delete the node-controller service account to disable the // node periodic check // delete if exist