Clusters
Clusters Red Hat Advanced Cluster Management for Kubernetes 2.12 Cluster management
create clusterrolebinding <role-binding-name> --clusterrole=open-cluster-management:cluster-manager-admin --user=<username> create clusterrolebinding (role-binding-name) --clusterrole=open-cluster-management:admin:<cluster-name> --user=<username> create rolebinding <role-binding-name> -n <cluster-name> --clusterrole=admin --user=<username> create clusterrolebinding <role-binding-name> --clusterrole=open-cluster-management:view:<cluster-name> --user=<username> create rolebinding <role-binding-name> -n <cluster-name> --clusterrole=view --user=<username> get managedclusters.clusterview.open-cluster-management.io get managedclustersets.clusterview.open-cluster-management.io adm policy add-cluster-role-to-group open-cluster-management:clusterset-admin:server-foundation-clusterset server-foundation-team-admin adm policy add-cluster-role-to-group open-cluster-management:clusterset-view:server-foundation-clusterset server-foundation-team-user adm new-project server-foundation-clusterpool adm policy add-role-to-group admin server-foundation-team-admin --namespace server-foundation-clusterpool E0809 18:45:29.450874 1 reflector.go:147] k8s.io/client-go@v0.29.4/tools/cache/reflector.go:229: Failed to watch *v1.CertificateSigningRequest: failed to list *v1.CertificateSigningRequest: Get "https://api.xxx.openshiftapps.com:443/apis/certificates.k8s.io/v1/certificatesigningrequests?limit=500&resourceVersion=0": tls: failed to verify certificate: x509: certificate signed by unknown authority apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerConfig: serverVerificationStrategy: UseSystemTruststore apply -f <filename> get secret local-cluster-import -n local-cluster -o jsonpath={.data.import\.yaml} | base64 --decode > import.yaml apply -f import.yaml E0203 07:10:38.266841 1 reflector.go:138] sigs.k8s.io/controller-runtime/pkg/cache/internal/informers_map.go:224: Failed to watch *v1.ClusterClaim: failed to list *v1.ClusterClaim: v1.ClusterClaimList.Items: []v1.ClusterClaim: v1.ClusterClaim.v1.ClusterClaim.Spec: v1.ClusterClaimSpec.Lifetime: unmarshalerDecoder: time: unknown unit "w" in duration "1w", error found in #10 byte of ...|time":"1w"}},{"apiVe|..., bigger context ...|clusterPoolName":"policy-aas-hubs","lifetime":"1w"}},{"apiVersion":"hive.openshift.io/v1","kind":"Cl| edit clusterdeployment/<mycluster> -n <namespace> delete ns <namespace> status: agentLabelSelector: matchLabels: infraenvs.agent-install.openshift.io: qe2 bootArtifacts: initrd: https://assisted-image-service-multicluster-engine.redhat.com/images/0000/pxe-initrd?api_key=0000000&arch=x86_64&version=4.11 ipxeScript: https://assisted-service-multicluster-engine.redhat.com/api/assisted-install/v2/infra-envs/00000/downloads/files?api_key=000000000&file_name=ipxe-script kernel: https://mirror.openshift.com/pub/openshift-v4/x86_64/dependencies/rhcos/4.12/latest/rhcos-live-kernel-x86_64 rootfs: https://mirror.openshift.com/pub/openshift-v4/x86_64/dependencies/rhcos/4.12/latest/rhcos-live-rootfs.x86_64.img for artifact in oc get infraenv qe2 -ojsonpath="{.status.bootArtifacts}" | jq ". | keys[]" | sed "s/\"//g" do curl -k oc get infraenv qe2 -ojsonpath="{.status.bootArtifacts.USD{artifact}}"` -o USDartifact patch clusterdeployment <clusterdeployment-name> -p '{"metadata":{"finalizers":null}}' --type=merge apiVersion: v1 kind: ConfigMap metadata: name: my-assisted-service-config namespace: multicluster-engine data: ALLOW_CONVERGED_FLOW: "false" 1 annotate --overwrite AgentServiceConfig agent unsupported.agent-install.openshift.io/assisted-service-configmap=my-assisted-service-config progressing... mca and work configs mismatch status: conditions: - lastTransitionTime: "2024-09-09T16:08:42Z" message: progressing... mca and work configs mismatch reason: Progressing status: "True" type: Progressing configReferences: - desiredConfig: name: deploy-config namespace: open-cluster-management-hub specHash: b81380f1f1a1920388d90859a5d51f5521cecd77752755ba05ece495f551ebd0 group: addon.open-cluster-management.io lastObservedGeneration: 1 name: deploy-config namespace: open-cluster-management-hub resource: addondeploymentconfigs - desiredConfig: name: cluster-proxy specHash: "" group: proxy.open-cluster-management.io lastObservedGeneration: 1 name: cluster-proxy resource: managedproxyconfigurations -n <cluster-name> delete managedclusteraddon <addon-name> get bmh -n <cluster_provisioning_namespace> describe bmh -n <cluster_provisioning_namespace> <bmh_name> Status: Error Count: 1 Error Message: Image provisioning failed: ... [Errno 36] File name too long apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv spec: osImageVersion: 4.15 -n openshift-console get route console console console-openshift-console.apps.new-coral.purple-chesterfield.com console https reencrypt/Redirect None apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: {} create namespace <namespace> project <namespace> apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: <default> namespace: <namespace> spec: targetNamespaces: - <namespace> apply -f <path-to-file>/<operator-group>.yaml apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: multicluster-engine spec: sourceNamespace: openshift-marketplace source: redhat-operators channel: stable-2.7 installPlanApproval: Automatic name: multicluster-engine apply -f <path-to-file>/<subscription>.yaml apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: {} apply -f <path-to-file>/<custom-resource>.yaml error: unable to recognize "./mce.yaml": no matches for kind "MultiClusterEngine" in version "operator.multicluster-engine.io/v1" get mce -o=jsonpath='{.items[0].status.phase}' metadata: labels: node-role.kubernetes.io/infra: "" spec: taints: - effect: NoSchedule key: node-role.kubernetes.io/infra spec: config: nodeSelector: node-role.kubernetes.io/infra: "" tolerations: - key: node-role.kubernetes.io/infra effect: NoSchedule operator: Exists nodeSelector: node-role.kubernetes.io/infra: "" tolerations: - key: node-role.kubernetes.io/infra effect: NoSchedule operator: Exists spec: nodeSelector: node-role.kubernetes.io/infra: "" -n openshift-console get route console console console-openshift-console.apps.new-name.purple-name.com console https reencrypt/Redirect None additionalTrustBundle: | -----BEGIN CERTIFICATE----- certificate_content -----END CERTIFICATE----- sshKey: >- - mirrors: - <your_registry>/rhacm2 source: registry.redhat.io/rhacm2 - mirrors: - <your_registry>/quay source: registry.redhat.io/quay - mirrors: - <your_registry>/compliance source: registry.redhat.io/compliance apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: name: mce-repo spec: repositoryDigestMirrors: - mirrors: - mirror.registry.com:5000/multicluster-engine source: registry.redhat.io/multicluster-engine apply -f mce-policy.yaml UPSTREAM_REGISTRY=quay.io PRODUCT_REPO=openshift-release-dev RELEASE_NAME=ocp-release OCP_RELEASE=4.15.2-x86_64 LOCAL_REGISTRY=USD(hostname):5000 LOCAL_SECRET_JSON=<pull-secret> adm -a USD{LOCAL_SECRET_JSON} release mirror --from=USD{UPSTREAM_REGISTRY}/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE} --to=USD{LOCAL_REGISTRY}/ocp4 --to-release-image=USD{LOCAL_REGISTRY}/ocp4/release:USD{OCP_RELEASE} git clone https://github.com/openshift/cincinnati-graph-data build -f <docker-path> -t <graph-path>:latest push <graph-path>:latest --authfile=<pull-secret>.json apiVersion: v1 kind: ConfigMap metadata: name: trusted-ca data: updateservice-registry: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- patch image.config.openshift.io cluster -p '{"spec":{"additionalTrustedCA":{"name":"<trusted_ca>"}}}' --type merge apiVersion: update-service.openshift.io/v1beta2 kind: update-service metadata: name: openshift-cincinnati-instance namespace: openshift-update-service spec: registry: <registry-host-name>:<port> 1 replicas: 1 repository: USD{LOCAL_REGISTRY}/ocp4/release graphDataImage: '<host-name>:<port>/cincinnati-graph-data-container' 2 apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-mirror namespace: default spec: disabled: false remediationAction: enforce policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-image-content-source-policy spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: name: <your-local-mirror-name> 1 spec: repositoryDigestMirrors: - mirrors: - <your-registry> 2 source: registry.redhat.io --- apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-mirror namespace: default placementRef: name: placement-policy-mirror kind: PlacementRule apiGroup: apps.open-cluster-management.io subjects: - name: policy-mirror kind: Policy apiGroup: policy.open-cluster-management.io --- apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: name: placement-policy-mirror namespace: default spec: clusterSelector: matchExpressions: [] 3 apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-catalog namespace: default spec: disabled: false remediationAction: enforce policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-catalog spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: config.openshift.io/v1 kind: OperatorHub metadata: name: cluster spec: disableAllDefaultSources: true - complianceType: musthave objectDefinition: apiVersion: operators.coreos.com/v1alpha1 kind: CatalogSource metadata: name: my-operator-catalog namespace: openshift-marketplace spec: sourceType: grpc image: '<registry_host_name>:<port>/olm/redhat-operators:v1' 1 displayName: My Operator Catalog publisher: grpc --- apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-catalog namespace: default placementRef: name: placement-policy-catalog kind: PlacementRule apiGroup: apps.open-cluster-management.io subjects: - name: policy-catalog kind: Policy apiGroup: policy.open-cluster-management.io --- apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: name: placement-policy-catalog namespace: default spec: clusterSelector: matchExpressions: [] 2 get clusterversion -o yaml get routes apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-cluster-version namespace: default annotations: policy.open-cluster-management.io/standards: null policy.open-cluster-management.io/categories: null policy.open-cluster-management.io/controls: null spec: disabled: false remediationAction: enforce policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-cluster-version spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: config.openshift.io/v1 kind: ClusterVersion metadata: name: version spec: channel: stable-4.4 upstream: >- https://example-cincinnati-policy-engine-uri/api/upgrades_info/v1/graph 1 --- apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-cluster-version namespace: default placementRef: name: placement-policy-cluster-version kind: PlacementRule apiGroup: apps.open-cluster-management.io subjects: - name: policy-cluster-version kind: Policy apiGroup: policy.open-cluster-management.io --- apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: name: placement-policy-cluster-version namespace: default spec: clusterSelector: matchExpressions: [] 2 get clusterversion -o yaml apiVersion: v1 items: - apiVersion: config.openshift.io/v1 kind: ClusterVersion [..] spec: channel: stable-4.4 upstream: https://<hub-cincinnati-uri>/api/upgrades_info/v1/graph apiVersion: operator.open-cluster-management.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: <name> 1 enabled: true patch MultiClusterEngine <multiclusterengine-name> --type=json -p='[{"op": "add", "path": "/spec/overrides/components/-","value":{"name":"<name>","enabled":true}}]' apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: local-cluster enabled: false create secret generic <secret> -n <namespace> --from-file=.dockerconfigjson=<path-to-pull-secret> --type=kubernetes.io/dockerconfigjson apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: imagePullSecret: <secret> apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: targetNamespace: <target> apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: availabilityConfig: "Basic" spec: nodeSelector: node-role.kubernetes.io/infra: "" spec: tolerations: - key: node-role.kubernetes.io/infra effect: NoSchedule operator: Exists apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: managedserviceaccount enabled: true patch MultiClusterEngine <multiclusterengine-name> --type=json -p='[{"op": "add", "path": "/spec/overrides/components/-","value":{"name":"managedserviceaccount","enabled":true}}]' Cannot delete MultiClusterEngine resource because ManagedCluster resource(s) exist project <namespace> delete multiclusterengine --all get multiclusterengine -o yaml ❯ oc get csv NAME DISPLAY VERSION REPLACES PHASE multicluster-engine.v2.0.0 multicluster engine for Kubernetes 2.0.0 Succeeded ❯ oc delete clusterserviceversion multicluster-engine.v2.0.0 ❯ oc delete sub multicluster-engine #!/bin/bash delete apiservice v1.admission.cluster.open-cluster-management.io v1.admission.work.open-cluster-management.io delete validatingwebhookconfiguration multiclusterengines.multicluster.openshift.io delete mce --all kind: Secret metadata: name: <managed-cluster-name>-aws-creds namespace: <managed-cluster-namespace> type: Opaque data: aws_access_key_id: USD(echo -n "USD{AWS_KEY}" | base64 -w0) aws_secret_access_key: USD(echo -n "USD{AWS_SECRET}" | base64 -w0) label secret hypershift-operator-oidc-provider-s3-credentials -n local-cluster "cluster.open-cluster-management.io/type=awss3" label secret hypershift-operator-oidc-provider-s3-credentials -n local-cluster "cluster.open-cluster-management.io/credentials=credentials=" az ad sp create-for-rbac --role Contributor --name <service_principal> --scopes <subscription_path> az ad sp create-for-rbac --role Contributor --name <service_principal> --scopes <subscription_path> az account show az account show kind: Secret metadata: name: <managed-cluster-name>-azure-creds namespace: <managed-cluster-namespace> type: Opaque data: baseDomainResourceGroupName: USD(echo -n "USD{azure_resource_group_name}" | base64 -w0) osServicePrincipal.json: USD(base64 -w0 "USD{AZURE_CRED_JSON}") kind: Secret metadata: name: <managed-cluster-name>-gcp-creds namespace: <managed-cluster-namespace> type: Opaque data: osServiceAccount.json: USD(base64 -w0 "USD{GCP_CRED_JSON}") - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-release-nightly - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-release - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-v4.0-art-dev - mirrors: - registry.example.com:5000/rhacm2 source: registry.redhat.io/rhacm2 kind: Secret metadata: name: <managed-cluster-name>-vsphere-creds namespace: <managed-cluster-namespace> type: Opaque data: username: USD(echo -n "USD{VMW_USERNAME}" | base64 -w0) password.json: USD(base64 -w0 "USD{VMW_PASSWORD}") - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-release-nightly - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-release - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-v4.0-art-dev - mirrors: - registry.example.com:5000/rhacm2 source: registry.redhat.io/rhacm2 kind: Secret metadata: name: <managed-cluster-name>-osp-creds namespace: <managed-cluster-namespace> type: Opaque data: clouds.yaml: USD(base64 -w0 "USD{OSP_CRED_YAML}") cloud: USD(echo -n "openstack" | base64 -w0) apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: fast visible: 'true' name: img4.x.1-x86-64-appsub spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.x.1-x86_64 get clusterimageset quay.io/openshift-release-dev/ocp-release:4.6.8-x86_64 apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: fast visible: 'true' name: img4.x.0-multi-appsub spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.x.0-multi pull quay.io/openshift-release-dev/ocp-release:4.x.1-x86_64 pull quay.io/openshift-release-dev/ocp-release:4.x.1-ppc64le pull quay.io/openshift-release-dev/ocp-release:4.x.1-s390x pull quay.io/openshift-release-dev/ocp-release:4.x.1-aarch64 login <private-repo> push quay.io/openshift-release-dev/ocp-release:4.x.1-x86_64 <private-repo>/ocp-release:4.x.1-x86_64 push quay.io/openshift-release-dev/ocp-release:4.x.1-ppc64le <private-repo>/ocp-release:4.x.1-ppc64le push quay.io/openshift-release-dev/ocp-release:4.x.1-s390x <private-repo>/ocp-release:4.x.1-s390x push quay.io/openshift-release-dev/ocp-release:4.x.1-aarch64 <private-repo>/ocp-release:4.x.1-aarch64 manifest create mymanifest manifest add mymanifest <private-repo>/ocp-release:4.x.1-x86_64 manifest add mymanifest <private-repo>/ocp-release:4.x.1-ppc64le manifest add mymanifest <private-repo>/ocp-release:4.x.1-s390x manifest add mymanifest <private-repo>/ocp-release:4.x.1-aarch64 manifest push mymanifest docker://<private-repo>/ocp-release:4.x.1 apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: fast visible: "true" name: img4.x.1-appsub spec: releaseImage: <private-repo>/ocp-release:4.x.1 apply -f <file-name>.yaml apiVersion: v1 kind: ConfigMap metadata: name: cluster-image-set-git-repo namespace: multicluster-engine data: gitRepoUrl: <forked acm-hive-openshift-releases repository URL> gitRepoBranch: backplane-<2.x> gitRepoPath: clusterImageSets channel: <fast or stable> get clusterImageSets delete clusterImageSet <clusterImageSet_NAME> apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: fast name: img<4.x.x>-x86-64-appsub spec: releaseImage: IMAGE_REGISTRY_IPADDRESS_or__DNSNAME/REPO_PATH/ocp-release@sha256:073a4e46289be25e2a05f5264c8f1d697410db66b960c9ceeddebd1c61e58717 adm release info <tagged_openshift_release_image> | grep "Pull From" Pull From: quay.io/openshift-release-dev/ocp-release@sha256:69d1292f64a2b67227c5592c1a7d499c7d00376e498634ff8e1946bc9ccdddfe create -f <clusterImageSet_FILE> create -f img4.11.9-x86_64.yaml delete -f subscribe/subscription-fast make subscribe-candidate make subscribe-fast make subscribe-stable delete -f subscribe/subscription-fast git clone https://github.com/stolostron/acm-hive-openshift-releases.git cd acm-hive-openshift-releases git checkout origin/backplane-<2.x> find clusterImageSets/fast -type d -exec oc apply -f {} \; 2> /dev/null apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: stable visible: 'true' name: img4.x.47-x86-64-appsub spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.x.47-x86_64 kind: ConfigMap apiVersion: v1 metadata: name: <my-baremetal-cluster-install-manifests> namespace: <mynamespace> data: 99_metal3-config.yaml: | kind: ConfigMap apiVersion: v1 metadata: name: metal3-config namespace: openshift-machine-api data: http_port: "6180" provisioning_interface: "enp1s0" provisioning_ip: "172.00.0.3/24" dhcp_range: "172.00.0.10,172.00.0.100" deploy_kernel_url: "http://172.00.0.3:6180/images/ironic-python-agent.kernel" deploy_ramdisk_url: "http://172.00.0.3:6180/images/ironic-python-agent.initramfs" ironic_endpoint: "http://172.00.0.3:6385/v1/" ironic_inspector_endpoint: "http://172.00.0.3:5150/v1/" cache_url: "http://192.168.111.1/images" rhcos_image_url: "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.3/43.81.201911192044.0/x86_64/rhcos-43.81.201911192044.0-openstack.x86_64.qcow2.gz" apply -f <filename>.yaml apiVersion: hive.openshift.io/v1 kind: ClusterDeployment metadata: name: <my-baremetal-cluster> namespace: <mynamespace> annotations: hive.openshift.io/try-install-once: "true" spec: baseDomain: test.example.com clusterName: <my-baremetal-cluster> controlPlaneConfig: servingCertificates: {} platform: baremetal: libvirtSSHPrivateKeySecretRef: name: provisioning-host-ssh-private-key provisioning: installConfigSecretRef: name: <my-baremetal-cluster-install-config> sshPrivateKeySecretRef: name: <my-baremetal-hosts-ssh-private-key> manifestsConfigMapRef: name: <my-baremetal-cluster-install-manifests> imageSetRef: name: <my-clusterimageset> sshKnownHosts: - "10.1.8.90 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXvVVVKUYVkuyvkuygkuyTCYTytfkufTYAAAAIbmlzdHAyNTYAAABBBKWjJRzeUVuZs4yxSy4eu45xiANFIIbwE3e1aPzGD58x/NX7Yf+S8eFKq4RrsfSaK2hVJyJjvVIhUsU9z2sBJP8=" pullSecretRef: name: <my-baremetal-cluster-pull-secret> apply -f <filename>.yaml vpc-1 (us-gov-east-1) : 10.0.0.0/20 subnet-11 (us-gov-east-1a): 10.0.0.0/23 subnet-12 (us-gov-east-1b): 10.0.2.0/23 subnet-13 (us-gov-east-1c): 10.0.4.0/23 subnet-12 (us-gov-east-1d): 10.0.8.0/23 subnet-12 (us-gov-east-1e): 10.0.10.0/23 subnet-12 (us-gov-east-1f): 10.0.12.0/2 vpc-2 (us-gov-east-1) : 10.0.16.0/20 subnet-21 (us-gov-east-1a): 10.0.16.0/23 subnet-22 (us-gov-east-1b): 10.0.18.0/23 subnet-23 (us-gov-east-1c): 10.0.20.0/23 subnet-24 (us-gov-east-1d): 10.0.22.0/23 subnet-25 (us-gov-east-1e): 10.0.24.0/23 subnet-26 (us-gov-east-1f): 10.0.28.0/23 ec2:CreateVpcEndpointServiceConfiguration ec2:DescribeVpcEndpointServiceConfigurations ec2:ModifyVpcEndpointServiceConfiguration ec2:DescribeVpcEndpointServicePermissions ec2:ModifyVpcEndpointServicePermissions ec2:DeleteVpcEndpointServiceConfigurations ec2:DescribeVpcEndpointServices ec2:DescribeVpcEndpoints ec2:CreateVpcEndpoint ec2:CreateTags ec2:DescribeNetworkInterfaces ec2:DescribeVPCs ec2:DeleteVpcEndpoints route53:CreateHostedZone route53:GetHostedZone route53:ListHostedZonesByVPC route53:AssociateVPCWithHostedZone route53:DisassociateVPCFromHostedZone route53:CreateVPCAssociationAuthorization route53:DeleteVPCAssociationAuthorization route53:ListResourceRecordSets route53:ChangeResourceRecordSets route53:DeleteHostedZone route53:AssociateVPCWithHostedZone route53:DisassociateVPCFromHostedZone ec2:DescribeVPCs spec: awsPrivateLink: ## The list of inventory of VPCs that can be used to create VPC ## endpoints by the controller. endpointVPCInventory: - region: us-east-1 vpcID: vpc-1 subnets: - availabilityZone: us-east-1a subnetID: subnet-11 - availabilityZone: us-east-1b subnetID: subnet-12 - availabilityZone: us-east-1c subnetID: subnet-13 - availabilityZone: us-east-1d subnetID: subnet-14 - availabilityZone: us-east-1e subnetID: subnet-15 - availabilityZone: us-east-1f subnetID: subnet-16 - region: us-east-1 vpcID: vpc-2 subnets: - availabilityZone: us-east-1a subnetID: subnet-21 - availabilityZone: us-east-1b subnetID: subnet-22 - availabilityZone: us-east-1c subnetID: subnet-23 - availabilityZone: us-east-1d subnetID: subnet-24 - availabilityZone: us-east-1e subnetID: subnet-25 - availabilityZone: us-east-1f subnetID: subnet-26 ## The credentialsSecretRef references a secret with permissions to create. ## The resources in the account where the inventory of VPCs exist. credentialsSecretRef: name: <hub-account-credentials-secret-name> ## A list of VPC where various mce clusters exists. associatedVPCs: - region: region-mce1 vpcID: vpc-mce1 credentialsSecretRef: name: <credentials-that-have-access-to-account-where-MCE1-VPC-exists> - region: region-mce2 vpcID: vpc-mce2 credentialsSecretRef: name: <credentials-that-have-access-to-account-where-MCE2-VPC-exists> api.<cluster_name>.<base_domain> *.apps.<cluster_name>.<base_domain> api.<cluster_name>.<base_domain> *.apps.<cluster_name>.<base_domain> apiVersion: v1 kind: Secret type: Opaque metadata: name: ocp3-openstack-trust namespace: ocp3 stringData: ca.crt: | -----BEGIN CERTIFICATE----- <Base64 certificate contents here> -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- <Base64 certificate contents here> -----END CERTIFICATE---- platform: openstack: certificatesSecretRef: name: ocp3-openstack-trust credentialsSecretRef: name: ocp3-openstack-creds cloud: openstack api.<cluster_name>.<base_domain> *.apps.<cluster_name>.<base_domain> apiVersion: v1 kind: Namespace metadata: name: sample-namespace apiVersion: v1 kind: Secret type: kubernetes.io/dockerconfigjson metadata: name: <pull-secret> namespace: sample-namespace stringData: .dockerconfigjson: 'your-pull-secret-json' 1 apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: name: openshift-v4.15.0 spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.15.0-rc.0-x86_64 apiVersion: hive.openshift.io/v1 kind: ClusterDeployment metadata: name: single-node namespace: demo-worker4 spec: baseDomain: hive.example.com clusterInstallRef: group: extensions.hive.openshift.io kind: AgentClusterInstall name: test-agent-cluster-install 1 version: v1beta1 clusterName: test-cluster controlPlaneConfig: servingCertificates: {} platform: agentBareMetal: agentSelector: matchLabels: location: internal pullSecretRef: name: <pull-secret> 2 apiVersion: extensions.hive.openshift.io/v1beta1 kind: AgentClusterInstall metadata: name: test-agent-cluster-install namespace: demo-worker4 spec: platformType: BareMetal 1 clusterDeploymentRef: name: single-node 2 imageSetRef: name: openshift-v4.15.0 3 networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 machineNetwork: - cidr: 192.168.111.0/24 serviceNetwork: - 172.30.0.0/16 provisionRequirements: controlPlaneAgents: 1 sshPublicKey: ssh-rsa <your-public-key-here> 4 apiVersion: agent-install.openshift.io/v1beta1 kind: NMStateConfig metadata: name: <mynmstateconfig> namespace: <demo-worker4> labels: demo-nmstate-label: <value> spec: config: interfaces: - name: eth0 type: ethernet state: up mac-address: 02:00:00:80:12:14 ipv4: enabled: true address: - ip: 192.168.111.30 prefix-length: 24 dhcp: false - name: eth1 type: ethernet state: up mac-address: 02:00:00:80:12:15 ipv4: enabled: true address: - ip: 192.168.140.30 prefix-length: 24 dhcp: false dns-resolver: config: server: - 192.168.126.1 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.111.1 next-hop-interface: eth1 table-id: 254 - destination: 0.0.0.0/0 next-hop-address: 192.168.140.1 next-hop-interface: eth1 table-id: 254 interfaces: - name: "eth0" macAddress: "02:00:00:80:12:14" - name: "eth1" macAddress: "02:00:00:80:12:15" apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: myinfraenv namespace: demo-worker4 spec: clusterRef: name: single-node 1 namespace: demo-worker4 2 pullSecretRef: name: pull-secret sshAuthorizedKey: <your_public_key_here> nmStateConfigLabelSelector: matchLabels: demo-nmstate-label: value proxy: httpProxy: http://USERNAME:PASSWORD@proxy.example.com:PORT httpsProxy: https://USERNAME:PASSWORD@proxy.example.com:PORT noProxy: .example.com,172.22.0.0/24,10.10.0.0/24 curl --insecure -o image.iso USD(kubectl -n sample-namespace get infraenvs.agent-install.openshift.io myinfraenv -o=jsonpath="{.status.isoDownloadURL}") -n sample-namespace patch agents.agent-install.openshift.io 07e80ea9-200c-4f82-aff4-4932acb773d4 -p '{"spec":{"approved":true}}' --type merge apiVersion: v1 kind: Proxy baseDomain: <domain> proxy: httpProxy: http://<username>:<password>@<proxy.example.com>:<port> httpsProxy: https://<username>:<password>@<proxy.example.com>:<port> noProxy: <wildcard-of-domain>,<provisioning-network/CIDR>,<BMC-address-range/CIDR> apiVersion: extensions.hive.openshift.io/v1beta1 kind: AgentClusterInstall spec: proxy: httpProxy: http://<username>:<password>@<proxy.example.com>:<port> 1 httpsProxy: https://<username>:<password>@<proxy.example.com>:<port> 2 noProxy: <wildcard-of-domain>,<provisioning-network/CIDR>,<BMC-address-range/CIDR> 3 create secret generic pull-secret -n <open-cluster-management> --from-file=.dockerconfigjson=<path-to-pull-secret> --type=kubernetes.io/dockerconfigjson edit managedcluster <cluster-name> spec: hubAcceptsClient: true managedClusterClientConfigs: - url: <https://api.new-managed.dev.redhat.com> 1 apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: open-cluster-management/nodeSelector: '{"dedicated":"acm"}' open-cluster-management/tolerations: '[{"key":"dedicated","operator":"Equal","value":"acm","effect":"NoSchedule"}]' apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: <klusterletconfigName> spec: nodePlacement: nodeSelector: dedicated: acm tolerations: - key: dedicated operator: Equal value: acm effect: NoSchedule delete po -n open-cluster-management `oc get pod -n open-cluster-management | grep multiclusterhub-operator| cut -d' ' -f1` login new-project <cluster_name> apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: <cluster_name> labels: cloud: auto-detect vendor: auto-detect spec: hubAcceptsClient: true cloud: Amazon vendor: OpenShift apply -f managed-cluster.yaml apiVersion: v1 kind: Secret metadata: name: auto-import-secret namespace: <cluster_name> stringData: autoImportRetry: "5" # If you are using the kubeconfig file, add the following value for the kubeconfig file # that has the current context set to the cluster to import: kubeconfig: |- <kubeconfig_file> # If you are using the token/server pair, add the following two values instead of # the kubeconfig file: token: <Token to access the cluster> server: <cluster_api_url> type: Opaque apply -f auto-import-secret.yaml -n <cluster_name> annotate secrets auto-import-secret managedcluster-import-controller.open-cluster-management.io/keeping-auto-import-secret="" get managedcluster <cluster_name> login get pod -n open-cluster-management-agent get secret <cluster_name>-import -n <cluster_name> -o jsonpath={.data.crds\\.yaml} | base64 --decode > klusterlet-crd.yaml get secret <cluster_name>-import -n <cluster_name> -o jsonpath={.data.import\\.yaml} | base64 --decode > import.yaml login apply -f klusterlet-crd.yaml apply -f import.yaml get managedcluster <cluster_name> apiVersion: agent.open-cluster-management.io/v1 kind: KlusterletAddonConfig metadata: name: <cluster_name> namespace: <cluster_name> spec: applicationManager: enabled: true certPolicyController: enabled: true policyController: enabled: true searchCollector: enabled: true apply -f klusterlet-addon-config.yaml get pod -n open-cluster-management-agent-addon delete managedcluster <cluster_name> export agent_registration_host=USD(oc get route -n multicluster-engine agent-registration -o=jsonpath="{.spec.host}") get configmap -n kube-system kube-root-ca.crt -o=jsonpath="{.data['ca\.crt']}" > ca.crt_ apiVersion: v1 kind: ServiceAccount metadata: name: managed-cluster-import-agent-registration-sa namespace: multicluster-engine --- apiVersion: v1 kind: Secret type: kubernetes.io/service-account-token metadata: name: managed-cluster-import-agent-registration-sa-token namespace: multicluster-engine annotations: kubernetes.io/service-account.name: "managed-cluster-import-agent-registration-sa" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: managedcluster-import-controller-agent-registration-client rules: - nonResourceURLs: ["/agent-registration/*"] verbs: ["get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: managed-cluster-import-agent-registration roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: managedcluster-import-controller-agent-registration-client subjects: - kind: ServiceAccount name: managed-cluster-import-agent-registration-sa namespace: multicluster-engine export token=USD(oc get secret -n multicluster-engine managed-cluster-import-agent-registration-sa-token -o=jsonpath='{.data.token}' | base64 -d) patch clustermanager cluster-manager --type=merge -p '{"spec":{"registrationConfiguration":{"featureGates":[ {"feature": "ManagedClusterAutoApproval", "mode": "Enable"}], "autoApproveUsers":["system:serviceaccount:multicluster-engine:agent-registration-bootstrap"]}}}' curl --cacert ca.crt -H "Authorization: Bearer USDtoken" https://USDagent_registration_host/agent-registration/crds/v1 | oc apply -f - curl --cacert ca.crt -H "Authorization: Bearer USDtoken" https://USDagent_registration_host/agent-registration/manifests/<clusterName>?klusterletconfig=<klusterletconfigName>&duration=<duration> | oc apply -f - apiVersion: v1 kind: Namespace metadata: name: managed-cluster apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: name: openshift-v4.15 spec: releaseImage: quay.io/openshift-release-dev/ocp-release@sha256:22e149142517dfccb47be828f012659b1ccf71d26620e6f62468c264a7ce7863 apiVersion: v1 kind: Secret type: kubernetes.io/dockerconfigjson metadata: name: pull-secret namespace: managed-cluster stringData: .dockerconfigjson: <pull-secret-json> 1 get secret -n openshift-kube-apiserver node-kubeconfigs -ojson | jq '.data["lb-ext.kubeconfig"]' --raw-output | base64 -d > /tmp/kubeconfig.some-other-cluster -n managed-cluster create secret generic some-other-cluster-admin-kubeconfig --from-file=kubeconfig=/tmp/kubeconfig.some-other-cluster apiVersion: extensions.hive.openshift.io/v1beta1 kind: AgentClusterInstall metadata: name: <your-cluster-name> 1 namespace: <managed-cluster> spec: networking: userManagedNetworking: true clusterDeploymentRef: name: <your-cluster> imageSetRef: name: openshift-v4.11.18 provisionRequirements: controlPlaneAgents: 2 sshPublicKey: <""> 3 apiVersion: hive.openshift.io/v1 kind: ClusterDeployment metadata: name: <your-cluster-name> 1 namespace: managed-cluster spec: baseDomain: <redhat.com> 2 installed: <true> 3 clusterMetadata: adminKubeconfigSecretRef: name: <your-cluster-name-admin-kubeconfig> 4 clusterID: <""> 5 infraID: <""> 6 clusterInstallRef: group: extensions.hive.openshift.io kind: AgentClusterInstall name: your-cluster-name-install version: v1beta1 clusterName: your-cluster-name platform: agentBareMetal: pullSecretRef: name: pull-secret apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: your-infraenv namespace: managed-cluster spec: clusterRef: name: your-cluster-name namespace: managed-cluster pullSecretRef: name: pull-secret sshAuthorizedKey: "" get infraenv -n managed-cluster some-other-infraenv -ojson | jq ".status.<url>" --raw-output | xargs curl -k -o /storage0/isos/some-other.iso -kubeconfig <source_hub_kubeconfig> -n <managed_cluster_name> get <resource_name> <cluster_provisioning_namespace> -oyaml > <resource_name>.yaml yq --in-place -y 'del(.metadata.ownerReferences)' AgentClusterInstall.yaml yq --in-place -y 'del(.metadata.ownerReferences)' AdminKubeconfigSecret.yaml -kubeconfig <target_hub_kubeconfig> delete ManagedCluster <cluster_name> -kubeconfig <target_hub_kubeconfig> apply -f <resource_name>.yaml apiVersion: imageregistry.open-cluster-management.io/v1alpha1 kind: ManagedClusterImageRegistry metadata: name: <imageRegistryName> namespace: <namespace> spec: placementRef: group: cluster.open-cluster-management.io resource: placements name: <placementName> 1 pullSecret: name: <pullSecretName> 2 registries: 3 - mirror: <mirrored-image-registry-address> source: <image-registry-address> - mirror: <mirrored-image-registry-address> source: <image-registry-address> registries: - mirror: localhost:5000/rhacm2/ source: registry.redhat.io/rhacm2 - mirror: localhost:5000/multicluster-engine source: registry.redhat.io/multicluster-engine registries: - mirror: localhost:5000/rhacm2-registration-rhel8-operator source: registry.redhat.io/rhacm2/registration-rhel8-operator apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: <klusterletconfigName> spec: pullSecret: namespace: <pullSecretNamespace> name: <pullSecretName> registries: - mirror: <mirrored-image-registry-address> source: <image-registry-address> - mirror: <mirrored-image-registry-address> source: <image-registry-address> kubectl create secret docker-registry myPullSecret --docker-server=<your-registry-server> --docker-username=<my-name> --docker-password=<my-password> apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: myPlacement namespace: myNamespace spec: clusterSets: - myClusterSet tolerations: - key: "cluster.open-cluster-management.io/unreachable" operator: Exists apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSet metadata: name: myClusterSet --- apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSetBinding metadata: name: myClusterSet namespace: myNamespace spec: clusterSet: myClusterSet apiVersion: imageregistry.open-cluster-management.io/v1alpha1 kind: ManagedClusterImageRegistry metadata: name: myImageRegistry namespace: myNamespace spec: placementRef: group: cluster.open-cluster-management.io resource: placements name: myPlacement pullSecret: name: myPullSecret registry: myRegistryAddress get machinepools -n <managed-cluster-namespace> edit machinepool <MachinePool-resource-name> -n <managed-cluster-namespace> get machinepools -n <managed-cluster-namespace> edit machinepool <name-of-MachinePool-resource> -n <namespace-of-managed-cluster> get machinepools.hive.openshift.io -n <managed-cluster-namespace> edit machinepool.hive.openshift.io <MachinePool-resource-name> -n <managed-cluster-namespace> export <kubeconfig_name>=USD(oc get cd USD<cluster_name> -o "jsonpath={.spec.clusterMetadata.adminKubeconfigSecretRef.name}") extract secret/USD<kubeconfig_name> --keys=kubeconfig --to=- > original-kubeconfig --kubeconfig=original-kubeconfig get node Unable to connect to the server: tls: failed to verify certificate: x509: certificate signed by unknown authority echo <base64 encoded blob> | base64 --decode > decoded-existing-certs.pem cp original-kubeconfig <new_kubeconfig_name> cat decoded-existing-certs.pem new-ca-certificate.pem | openssl base64 -A KUBECONFIG=<new_kubeconfig_name> oc get nodes patch secret USDoriginal-kubeconfig --type='json' -p="[{'op': 'replace', 'path': '/data/kubeconfig', 'value': 'USD(openssl base64 -A -in <new_kubeconfig_name>)'},{'op': 'replace', 'path': '/data/raw-kubeconfig', 'value': 'USD(openssl base64 -A -in <new_kubeconfig_name>)'}]" watch -n 5 "oc get agent -n managed-cluster" get agent -n managed-cluster -ojson | jq -r '.items[] | select(.spec.approved==false) |select(.spec.clusterDeploymentName==null) | .metadata.name'| xargs oc -n managed-cluster patch -p '{"spec":{"clusterDeploymentName":{"name":"some-other-cluster","namespace":"managed-cluster"}}}' --type merge agent get agent -n managed-cluster -ojson | jq -r '.items[] | select(.spec.approved==false) | .metadata.name'| xargs oc -n managed-cluster patch -p '{"spec":{"approved":true}}' --type merge agent patch agent <AGENT-NAME> -p '{"spec":{"role": "master"}}' --type=merge bmac.agent-install.openshift.io/role: master patch agent <AGENT-NAME> -p '{"spec":{"role": "master"}}' --type=merge edit clusterdeployment <name-of-cluster> -n <namespace-of-cluster> get clusterdeployment <name-of-cluster> -n <namespace-of-cluster> -o yaml edit clusterdeployment <name-of-cluster> -n <namespace-of-cluster> get clusterdeployment <name-of-cluster> -n <namespace-of-cluster> -o yaml UPSTREAM_REGISTRY=quay.io PRODUCT_REPO=openshift-release-dev RELEASE_NAME=ocp-release OCP_RELEASE=4.12.2-x86_64 LOCAL_REGISTRY=USD(hostname):5000 LOCAL_SECRET_JSON=/path/to/pull/secret 1 adm -a USD{LOCAL_SECRET_JSON} release mirror --from=USD{UPSTREAM_REGISTRY}/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE} --to=USD{LOCAL_REGISTRY}/ocp4 --to-release-image=USD{LOCAL_REGISTRY}/ocp4/release:USD{OCP_RELEASE} git clone https://github.com/openshift/cincinnati-graph-data FROM registry.access.redhat.com/ubi8/ubi:8.1 1 RUN curl -L -o cincinnati-graph-data.tar.gz https://github.com/openshift/cincinnati-graph-data/archive/master.tar.gz 2 RUN mkdir -p /var/lib/cincinnati/graph-data/ 3 CMD exec /bin/bash -c "tar xvzf cincinnati-graph-data.tar.gz -C /var/lib/ cincinnati/graph-data/ --strip-components=1" 4 build -f <path_to_Dockerfile> -t <USD{DISCONNECTED_REGISTRY}/cincinnati/cincinnati-graph-data-container>:latest 1 2 push <USD{DISCONNECTED_REGISTRY}/cincinnati/cincinnati-graph-data-container><2>:latest --authfile=</path/to/pull_secret>.json 3 apiVersion: v1 kind: ConfigMap metadata: name: trusted-ca data: updateservice-registry: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- patch image.config.openshift.io cluster -p '{"spec":{"additionalTrustedCA":{"name":"trusted-ca"}}}' --type merge apiVersion: cincinnati.openshift.io/v1beta2 kind: Cincinnati metadata: name: openshift-update-service-instance namespace: openshift-cincinnati spec: registry: <registry_host_name>:<port> 1 replicas: 1 repository: USD{LOCAL_REGISTRY}/ocp4/release graphDataImage: '<host_name>:<port>/cincinnati-graph-data-container' 2 apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: name: <your-local-mirror-name> 1 spec: repositoryDigestMirrors: - mirrors: - <your-registry> 2 source: registry.redhat.io apply -f mirror.yaml apiVersion: config.openshift.io/v1 kind: OperatorHub metadata: name: cluster spec: disableAllDefaultSources: true --- apiVersion: operators.coreos.com/v1alpha1 kind: CatalogSource metadata: name: my-operator-catalog namespace: openshift-marketplace spec: sourceType: grpc image: '<registry_host_name>:<port>/olm/redhat-operators:v1' 1 displayName: My Operator Catalog publisher: grpc apply -f source.yaml get clusterversion -o yaml apiVersion: v1 items: - apiVersion: config.openshift.io/v1 kind: ClusterVersion [..] spec: channel: stable-4.x upstream: https://api.openshift.com/api/upgrades_info/v1/graph get routes edit clusterversion version get routes -A get clusterversion -o yaml apiVersion: v1 items: - apiVersion: config.openshift.io/v1 kind: ClusterVersion [..] spec: channel: stable-4.x upstream: https://<hub-cincinnati-uri>/api/upgrades_info/v1/graph export KUBECONFIG=<managed-cluster-kubeconfig> create role -n default test-role --verb=list,get --resource=pods create rolebinding -n default test-rolebinding --serviceaccount=default:default --role=test-role get secret -n default | grep <default-token> export MANAGED_CLUSTER_TOKEN=USD(kubectl -n default get secret <default-token> -o jsonpath={.data.token} | base64 -d) config view --minify --raw=true > cluster-proxy.kubeconfig export TARGET_MANAGED_CLUSTER=<managed-cluster-name> export NEW_SERVER=https://USD(oc get route -n multicluster-engine cluster-proxy-addon-user -o=jsonpath='{.spec.host}')/USDTARGET_MANAGED_CLUSTER sed -i'' -e '/server:/c\ server: '"USDNEW_SERVER"'' cluster-proxy.kubeconfig export CADATA=USD(oc get configmap -n openshift-service-ca kube-root-ca.crt -o=go-template='{{index .data "ca.crt"}}' | base64) sed -i'' -e '/certificate-authority-data:/c\ certificate-authority-data: '"USDCADATA"'' cluster-proxy.kubeconfig sed -i'' -e '/client-certificate-data/d' cluster-proxy.kubeconfig sed -i'' -e '/client-key-data/d' cluster-proxy.kubeconfig sed -i'' -e '/token/d' cluster-proxy.kubeconfig sed -i'' -e 'USDa\ token: '"USDMANAGED_CLUSTER_TOKEN"'' cluster-proxy.kubeconfig get pods --kubeconfig=cluster-proxy.kubeconfig -n <default> export PROMETHEUS_TOKEN=USD(kubectl get secret -n openshift-monitoring USD(kubectl get serviceaccount -n openshift-monitoring prometheus-k8s -o=jsonpath='{.secrets[0].name}') -o=jsonpath='{.data.token}' | base64 -d) get configmap kube-root-ca.crt -o=jsonpath='{.data.ca\.crt}' > hub-ca.crt export SERVICE_NAMESPACE=openshift-monitoring export SERVICE_NAME=prometheus-k8s export SERVICE_PORT=9091 export SERVICE_PATH="api/v1/query?query=machine_cpu_sockets" curl --cacert hub-ca.crt USDNEW_SERVER/api/v1/namespaces/USDSERVICE_NAMESPACE/services/USDSERVICE_NAME:USDSERVICE_PORT/proxy-service/USDSERVICE_PATH -H "Authorization: Bearer USDPROMETHEUS_TOKEN" apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: <name> 1 namespace: <namespace> 2 spec: agentInstallNamespace: open-cluster-management-agent-addon proxyConfig: httpsProxy: "http://<username>:<password>@<ip>:<port>" 3 noProxy: ".cluster.local,.svc,172.30.0.1" 4 caBundle: <value> 5 apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ManagedClusterAddOn metadata: name: cluster-proxy namespace: <namespace> 1 spec: installNamespace: open-cluster-management-addon configs: group: addon.open-cluster-management.io resource: addondeploymentconfigs name: <name> 2 namespace: <namespace> 3 operation: retryPosthook: installPosthook operation: retryPosthook: upgradePosthook apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: test-inno namespace: test-inno spec: desiredCuration: upgrade destroy: {} install: {} scale: {} upgrade: channel: stable-4.x desiredUpdate: 4.x.1 monitorTimeout: 150 posthook: - extra_vars: {} clusterName: test-inno type: post_check name: ACM Upgrade Checks prehook: - extra_vars: {} clusterName: test-inno type: pre_check name: ACM Upgrade Checks towerAuthSecret: awx inventory: Demo Inventory apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: cluster1 {{{} namespace: cluster1 labels: test1: test1 test2: test2 {}}}spec: desiredCuration: install install: jobMonitorTimeout: 5 posthook: - extra_vars: {} name: Demo Job Template type: Job prehook: - extra_vars: {} name: Demo Job Template type: Job towerAuthSecret: toweraccess spec: desiredCuration: upgrade upgrade: intermediateUpdate: 4.14.x desiredUpdate: 4.15.x monitorTimeout: 120 posthook: - extra_vars: {} name: Unpause machinepool type: Job prehook: - extra_vars: {} name: Pause machinepool type: Job apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: annotations: cluster.open-cluster-management.io/upgrade-clusterversion-backoff-limit: "10" name: your-name namespace: your-namespace spec: desiredCuration: upgrade upgrade: intermediateUpdate: 4.14.x desiredUpdate: 4.15.x monitorTimeout: 120 posthook: - extra_vars: {} name: Unpause machinepool type: Job prehook: - extra_vars: {} name: Pause machinepool type: Job apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: name: my-cluster namespace: clusters spec: pausedUntil: 'true' apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: name: my-cluster-us-east-2 namespace: clusters spec: pausedUntil: 'true' apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: my-cluster namespace: clusters labels: open-cluster-management: curator spec: desiredCuration: install install: jobMonitorTimeout: 5 prehook: - name: Demo Job Template extra_vars: variable1: something-interesting variable2: 2 - name: Demo Job Template posthook: - name: Demo Job Template towerAuthSecret: toweraccess apiVersion: v1 kind: Secret metadata: name: toweraccess namespace: clusters stringData: host: https://my-tower-domain.io token: ANSIBLE_TOKEN_FOR_admin apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: my-cluster namespace: clusters labels: open-cluster-management: curator spec: desiredCuration: upgrade upgrade: desiredUpdate: 4.15.1 1 monitorTimeout: 120 prehook: - name: Demo Job Template extra_vars: variable1: something-interesting variable2: 2 - name: Demo Job Template posthook: - name: Demo Job Template towerAuthSecret: toweraccess apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: my-cluster namespace: clusters labels: open-cluster-management: curator spec: desiredCuration: destroy destroy: jobMonitorTimeout: 5 prehook: - name: Demo Job Template extra_vars: variable1: something-interesting variable2: 2 - name: Demo Job Template posthook: - name: Demo Job Template towerAuthSecret: toweraccess apiVersion: cluster.open-cluster-management.io/v1alpha1 kind: ClusterClaim metadata: name: id.openshift.io spec: value: 95f91f25-d7a2-4fc3-9237-2ef633d8451c apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: labels: cloud: Amazon clusterID: 95f91f25-d7a2-4fc3-9237-2ef633d8451c installer.name: multiclusterhub installer.namespace: open-cluster-management name: cluster1 vendor: OpenShift name: cluster1 spec: hubAcceptsClient: true leaseDurationSeconds: 60 status: allocatable: cpu: '15' memory: 65257Mi capacity: cpu: '18' memory: 72001Mi clusterClaims: - name: id.k8s.io value: cluster1 - name: kubeversion.open-cluster-management.io value: v1.18.3+6c42de8 - name: platform.open-cluster-management.io value: AWS - name: product.open-cluster-management.io value: OpenShift - name: id.openshift.io value: 95f91f25-d7a2-4fc3-9237-2ef633d8451c - name: consoleurl.openshift.io value: 'https://console-openshift-console.apps.xxxx.dev04.red-chesterfield.com' - name: version.openshift.io value: '4.x' conditions: - lastTransitionTime: '2020-10-26T07:08:49Z' message: Accepted by hub cluster admin reason: HubClusterAdminAccepted status: 'True' type: HubAcceptedManagedCluster - lastTransitionTime: '2020-10-26T07:09:18Z' message: Managed cluster joined reason: ManagedClusterJoined status: 'True' type: ManagedClusterJoined - lastTransitionTime: '2020-10-30T07:20:20Z' message: Managed cluster is available reason: ManagedClusterAvailable status: 'True' type: ManagedClusterConditionAvailable version: kubernetes: v1.18.3+6c42de8 apiVersion: cluster.open-cluster-management.io/v1alpha1 kind: ClusterClaim metadata: name: <custom_claim_name> spec: value: <custom_claim_value> get clusterclaims.cluster.open-cluster-management.io apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSetBinding metadata: name: global namespace: open-cluster-management-global-set spec: clusterSet: global apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSet metadata: name: <cluster_set> kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: clusterrole1 rules: - apiGroups: ["cluster.open-cluster-management.io"] resources: ["managedclustersets/join"] resourceNames: ["<cluster_set>"] verbs: ["create"] apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: <cluster_name> spec: hubAcceptsClient: true apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: <cluster_name> labels: cluster.open-cluster-management.io/clusterset: <cluster_set> spec: hubAcceptsClient: true apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSetBinding metadata: namespace: <namespace> name: <cluster_set> spec: clusterSet: <cluster_set> apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: <clusterrole> rules: - apiGroups: ["cluster.open-cluster-management.io"] resources: ["managedclustersets/bind"] resourceNames: ["<cluster_set>"] verbs: ["create"] patch managedcluster <managed_cluster_name> -p '{"spec":{"taints":[{"key": "key", "value": "value", "effect": "NoSelect"}]}}' --type=merge patch managedcluster <managed_cluster_name> --type='json' -p='[{"op": "add", "path": "/spec/taints/-", "value": {"key": "key", "value": "value", "effect": "NoSelect"}}]' apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: cluster.open-cluster-management.io/unavailable timeAdded: '2022-02-21T08:11:54Z' apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: cluster.open-cluster-management.io/unreachable timeAdded: '2022-02-21T08:11:06Z' apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: gpu value: "true" timeAdded: '2022-02-21T08:11:06Z' apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement1 namespace: default spec: tolerations: - key: gpu value: "true" operator: Equal apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: cluster.open-cluster-management.io/unreachable timeAdded: '2022-02-21T08:11:06Z' apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: demo4 namespace: demo1 spec: tolerations: - key: cluster.open-cluster-management.io/unreachable operator: Exists tolerationSeconds: 300 get managedclusters -l cluster.open-cluster-management.io/clusterset=<cluster_set> labels: cluster.open-cluster-management.io/clusterset: clusterset1 apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: predicates: - requiredClusterSelector: labelSelector: matchLabels: vendor: OpenShift apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: predicates: - requiredClusterSelector: claimSelector: matchExpressions: - key: region.open-cluster-management.io operator: In values: - us-west-1 apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: clusterSets: - clusterset1 - clusterset2 predicates: - requiredClusterSelector: claimSelector: matchExpressions: - key: region.open-cluster-management.io operator: In values: - us-west-1 apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: numberOfClusters: 3 1 predicates: - requiredClusterSelector: labelSelector: matchLabels: vendor: OpenShift claimSelector: matchExpressions: - key: region.open-cluster-management.io operator: In values: - us-west-1 apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: gpu value: "true" timeAdded: '2022-02-21T08:11:06Z' apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: tolerations: - key: gpu value: "true" operator: Equal apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: cluster.open-cluster-management.io/unreachable timeAdded: '2022-02-21T08:11:06Z' apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: tolerations: - key: cluster.open-cluster-management.io/unreachable operator: Exists tolerationSeconds: 300 1 feature.open-cluster-management.io/addon-application-manager: available apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement1 namespace: ns1 spec: predicates: - requiredClusterSelector: labelSelector: matchExpressions: - key: feature.open-cluster-management.io/addon-application-manager operator: Exists apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement2 namespace: ns1 spec: predicates: - requiredClusterSelector: labelSelector: matchLabels: "feature.open-cluster-management.io/addon-application-manager": "available" apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement3 namespace: ns1 spec: predicates: - requiredClusterSelector: labelSelector: matchExpressions: - key: feature.open-cluster-management.io/addon-application-manager operator: DoesNotExist apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: numberOfClusters: 1 prioritizerPolicy: configurations: - scoreCoordinate: builtIn: ResourceAllocatableMemory apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: numberOfClusters: 1 prioritizerPolicy: configurations: - scoreCoordinate: builtIn: ResourceAllocatableCPU weight: 2 - scoreCoordinate: builtIn: ResourceAllocatableMemory weight: 2 apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: numberOfClusters: 2 prioritizerPolicy: mode: Exact configurations: - scoreCoordinate: builtIn: Steady weight: 3 - scoreCoordinate: type: AddOn addOn: resourceName: default scoreName: cpuratio apiVersion: cluster.open-cluster-management.io/v1beta1 kind: PlacementDecision metadata: labels: cluster.open-cluster-management.io/placement: placement1 name: placement1-kbc7q namespace: ns1 ownerReferences: - apiVersion: cluster.open-cluster-management.io/v1beta1 blockOwnerDeletion: true controller: true kind: Placement name: placement1 uid: 05441cf6-2543-4ecc-8389-1079b42fe63e status: decisions: - clusterName: cluster1 reason: '' - clusterName: cluster2 reason: '' - clusterName: cluster3 reason: '' kind: ClusterClaim metadata: annotations: cluster.open-cluster-management.io/createmanagedcluster: "false" 1 apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ManagedClusterAddOn metadata: name: managed-serviceaccount namespace: <target_managed_cluster> spec: installNamespace: open-cluster-management-agent-addon apply -f - apiVersion: authentication.open-cluster-management.io/v1alpha1 kind: ManagedServiceAccount metadata: name: <managedserviceaccount_name> namespace: <target_managed_cluster> spec: rotation: {} get managedserviceaccount <managed_serviceaccount_name> -n <target_managed_cluster> -o yaml get secret <managed_serviceaccount_name> -n <target_managed_cluster> -o yaml apiVersion: config.openshift.io/v1 kind: APIServer metadata: name: cluster spec: audit: profile: Default servingCerts: namedCertificates: - names: - api.mycluster.example.com servingCertificate: name: old-cert-secret cp old.crt combined.crt cat new.crt >> combined.crt create secret tls combined-certs-secret --cert=combined.crt --key=old.key -n openshift-config apiVersion: config.openshift.io/v1 kind: APIServer metadata: name: cluster spec: audit: profile: Default servingCerts: namedCertificates: - names: - api.mycluster.example.com servingCertificate: name: combined-cert-secret create secret tls new-cert-secret --cert=new.crt --key=new.key -n openshift-config {code} apiVersion: config.openshift.io/v1 kind: APIServer metadata: name: cluster spec: audit: profile: Default servingCerts: namedCertificates: - names: - api.mycluster.example.com servingCertificate: name: new-cert-secret apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: http-proxy spec: hubKubeAPIServerConfig: proxyURL: "http://<username>:<password>@<ip>:<port>" apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: https-proxy spec: hubKubeAPIServerConfig: proxyURL: "https://<username>:<password>@<ip>:<port>" trustedCABundles: - name: "proxy-ca-bundle" caBundle: name: <configmap-name> namespace: <configmap-namespace> create -n <configmap-namespace> configmap <configmap-name> --from-file=ca.crt=/path/to/ca/file apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: agent.open-cluster-management.io/klusterlet-config: <klusterlet-config-name> name:<managed-cluster-name> spec: hubAcceptsClient: true leaseDurationSeconds: 60 apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: open-cluster-management/nodeSelector: '{"dedicated":"acm"}' open-cluster-management/tolerations: '[{"key":"dedicated","operator":"Equal","value":"acm","effect":"NoSchedule"}]' apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: <name> 1 spec: hubKubeAPIServerConfig: url: "https://api.example.com:6443" 2 serverVerificationStrategy: UseCustomCABundles trustedCABundles: - name: <custom-ca-bundle> 3 caBundle: name: <custom-ca-bundle-configmap> 4 namespace: <multicluster-engine> 5 apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: agent.open-cluster-management.io/klusterlet-config: 1 name: 2 spec: hubAcceptsClient: true leaseDurationSeconds: 60 apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: test spec: hubKubeAPIServerConfig: url: "https://api.example.test.com:6443" --- apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerConfig: url: "https://api.example.global.com:6443" apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: test spec: hubKubeAPIServerURL: "" - apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerURL: "example.global.com" apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: test - apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerURL: "example.global.com" apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: ca-strategy spec: hubKubeAPIServerConfig: serverVerificationStrategy: UseAutoDetectedCABundle trustedCABundles: - name: new-ca caBundle: name: new-ocp-ca namespace: default apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: ca-strategy spec: hubKubeAPIServerConfig: serverVerificationStrategy: UseSystemTruststore apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: ca-strategy spec: hubKubeAPIServerConfig: serverVerificationStrategy: UseCustomCABundles trustedCABundles: - name: ca caBundle: name: ocp-ca namespace: default apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerConfig: serverVerificationStrategy: UseSystemTruststore apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: test-ca spec: hubKubeAPIServerConfig: serverVerificationStrategy: UseCustomCABundles trustedCABundles: - name: ca caBundle: name: ocp-ca namespace: default -- apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: agent.open-cluster-management.io/klusterlet-config: test-ca name: cluster1 spec: hubAcceptsClient: true leaseDurationSeconds: 60 delete po -n open-cluster-management `oc get pod -n open-cluster-management | grep multiclusterhub-operator| cut -d' ' -f1` delete managedcluster USDCLUSTER_NAME delete clusterdeployment <CLUSTER_NAME> -n USDCLUSTER_NAME delete po -n open-cluster-management `oc get pod -n open-cluster-management | grep multiclusterhub-operator| cut -d' ' -f1` open-cluster-management-agent Active 10m open-cluster-management-agent-addon Active 10m get klusterlet | grep klusterlet | awk '{print USD1}' | xargs oc patch klusterlet --type=merge -p '{"metadata":{"finalizers": []}}' delete namespaces open-cluster-management-agent open-cluster-management-agent-addon --wait=false get crds | grep open-cluster-management.io | awk '{print USD1}' | xargs oc delete crds --wait=false get crds | grep open-cluster-management.io | awk '{print USD1}' | xargs oc patch crds --type=merge -p '{"metadata":{"finalizers": []}}' get crds | grep open-cluster-management.io | awk '{print USD1}' get ns | grep open-cluster-management-agent oc rsh -n openshift-etcd etcd-control-plane-0.example.com etcdctl endpoint status --cluster -w table sh-4.4#etcdctl compact USD(etcdctl endpoint status --write-out="json" | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*' -m1) compacted revision 158774421 apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveryConfig metadata: name: discovery namespace: <NAMESPACE_NAME> spec: credential: <SECRET_NAME> filters: lastActive: 7 openshiftVersions: - "4.15" apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: name: fd51aafa-95a8-41f7-a992-6fb95eed3c8e namespace: <NAMESPACE_NAME> spec: activity_timestamp: "2021-04-19T21:06:14Z" cloudProvider: vsphere console: https://console-openshift-console.apps.qe1-vmware-pkt.dev02.red-chesterfield.com creation_timestamp: "2021-04-19T16:29:53Z" credential: apiVersion: v1 kind: Secret name: <SECRET_NAME> namespace: <NAMESPACE_NAME> display_name: qe1-vmware-pkt.dev02.red-chesterfield.com name: fd51aafa-95a8-41f7-a992-6fb95eed3c8e openshiftVersion: 4.15 status: Stale apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: name: 28c17977-fc73-4050-b5cc-a5aa2d1d6892 namespace: discovery spec: openshiftVersion: <4.x.z> isManagedCluster: false cloudProvider: aws name: 28c17977-fc73-4050-b5cc-a5aa2d1d6892 displayName: rosa-dc status: Active importAsManagedCluster: true 1 type: <supported-type> 2 apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: annotations: discovery.open-cluster-management.io/previously-auto-imported: 'true' 2024-06-12T14:11:43.366Z INFO reconcile Skipped automatic import for DiscoveredCluster due to existing 'discovery.open-cluster-management.io/previously-auto-imported' annotation {"Name": "rosa-dc"} patch discoveredcluster <name> -n <namespace> --type='json' -p='[{"op": "replace", "path": "/spec/importAsManagedCluster", "value": true}]' get managedcluster <name> rosa describe cluster --cluster=<cluster-name> | grep -o '^ID:.* get crd baremetalhosts.metal3.io Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "baremetalhosts.metal3.io" not found apply -f get provisioning patch provisioning provisioning-configuration --type merge -p '{"spec":{"watchAllNamespaces": true }}' apiVersion: metal3.io/v1alpha1 kind: Provisioning metadata: name: provisioning-configuration spec: provisioningNetwork: "Disabled" watchAllNamespaces: true apply -f apiVersion: v1 kind: ConfigMap metadata: name: <mirror-config> namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | <certificate-content> registries.conf: | unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] prefix = "" location = "registry.redhat.io/multicluster-engine" mirror-by-digest-only = true [[registry.mirror]] location = "mirror.registry.com:5000/multicluster-engine" { "Authorization": "Basic xyz" } { "api_key": "myexampleapikey", } create secret generic -n multicluster-engine os-images-http-auth --from-file=./query_params --from-file=./headers -n multicluster-engine create configmap image-service-additional-ca --from-file=tls.crt apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: name: agent spec: databaseStorage: accessModes: - ReadWriteOnce resources: requests: storage: <db_volume_size> filesystemStorage: accessModes: - ReadWriteOnce resources: requests: storage: <fs_volume_size> mirrorRegistryRef: name: <mirror_config> 1 unauthenticatedRegistries: - <unauthenticated_registry> 2 imageStorage: accessModes: - ReadWriteOnce resources: requests: storage: <img_volume_size> 3 OSImageAdditionalParamsRef: name: os-images-http-auth OSImageCACertRef: name: image-service-additional-ca osImages: - openshiftVersion: "<ocp_version>" 4 version: "<ocp_release_version>" 5 url: "<iso_url>" 6 cpuArchitecture: "x86_64" apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: name: agent spec: databaseStorage: accessModes: - ReadWriteOnce resources: requests: storage: <db_volume_size> 1 filesystemStorage: accessModes: - ReadWriteOnce resources: requests: storage: <fs_volume_size> 2 imageStorage: accessModes: - ReadWriteOnce resources: requests: storage: <img_volume_size> 3 login apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: annotations: agent-install.openshift.io/service-image-base: el8 login get routes --all-namespaces | grep assisted-image-service apiVersion: operator.openshift.io/v1 kind: IngressController metadata: name: ingress-controller-with-nlb namespace: openshift-ingress-operator spec: domain: nlb-apps.<domain>.com routeSelector: matchLabels: router-type: nlb endpointPublishingStrategy: type: LoadBalancerService loadBalancer: scope: External providerParameters: type: AWS aws: type: NLB apply -f ingresscontroller.yaml get ingresscontroller -n openshift-ingress-operator edit ingresscontroller <name> -n openshift-ingress-operator edit route assisted-image-service -n <namespace> metadata: labels: router-type: nlb name: assisted-image-service assisted-image-service-multicluster-engine.apps.<yourdomain>.com get pods -n multicluster-engine | grep assist login apiVersion: v1 kind: Namespace metadata: name: <your_namespace> 1 apply -f namespace.yaml apiVersion: v1 kind: Secret type: kubernetes.io/dockerconfigjson metadata: name: pull-secret 1 namespace: <your_namespace> stringData: .dockerconfigjson: <your_pull_secret> 2 apply -f pull-secret.yaml apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: myinfraenv namespace: <your_namespace> spec: proxy: httpProxy: <http://user:password@ipaddr:port> httpsProxy: <http://user:password@ipaddr:port> noProxy: additionalNTPSources: sshAuthorizedKey: pullSecretRef: name: <name> agentLabels: <key>: <value> nmStateConfigLabelSelector: matchLabels: <key>: <value> clusterRef: name: <cluster_name> namespace: <project_name> ignitionConfigOverride: '{"ignition": {"version": "3.1.0"}, ...}' cpuArchitecture: x86_64 ipxeScriptType: DiscoveryImageAlways kernelArguments: - operation: append value: audit=0 additionalTrustBundle: <bundle> osImageVersion: <version> apply -f infra-env.yaml describe infraenv myinfraenv -n <your_namespace> apiVersion: agent-install.openshift.io/v1beta1 kind: NMStateConfig metadata: name: mynmstateconfig namespace: <your-infraenv-namespace> labels: some-key: <some-value> spec: config: interfaces: - name: eth0 type: ethernet state: up mac-address: 02:00:00:80:12:14 ipv4: enabled: true address: - ip: 192.168.111.30 prefix-length: 24 dhcp: false - name: eth1 type: ethernet state: up mac-address: 02:00:00:80:12:15 ipv4: enabled: true address: - ip: 192.168.140.30 prefix-length: 24 dhcp: false dns-resolver: config: server: - 192.168.126.1 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.111.1 next-hop-interface: eth1 table-id: 254 - destination: 0.0.0.0/0 next-hop-address: 192.168.140.1 next-hop-interface: eth1 table-id: 254 interfaces: - name: "eth0" macAddress: "02:00:00:80:12:14" - name: "eth1" macAddress: "02:00:00:80:12:15" apply -f nmstateconfig.yaml get infraenv -n <infra env namespace> <infra env name> -o jsonpath='{.status.isoDownloadURL}' https://assisted-image-service-assisted-installer.apps.example-acm-hub.com/byapikey/eyJhbGciOiJFUzI1NiIsInC93XVCJ9.eyJpbmZyYV9lbnZfaWQcTA0Y38sWVjYi02MTA0LTQ4NDMtODasdkOGIxYTZkZGM5ZTUifQ.3ydTpHaXJmTasd7uDp2NvGUFRKin3Z9Qct3lvDky1N-5zj3KsRePhAM48aUccBqmucGt3g/4.16/x86_64/minimal.iso get agent -n <infra env namespace> NAME CLUSTER APPROVED ROLE STAGE 24a92a6f-ea35-4d6f-9579-8f04c0d3591e false auto-assign patch agent -n <infra env namespace> <agent name> -p '{"spec":{"approved":true}}' --type merge get agent -n <infra env namespace> NAME CLUSTER APPROVED ROLE STAGE 173e3a84-88e2-4fe1-967f-1a9242503bec true auto-assign apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: name: agent spec: iPXEHTTPRoute: enabled apiVersion: v1 kind: Secret metadata: name: <bmc-secret-name> namespace: <your_infraenv_namespace> 1 type: Opaque data: username: <username> password: <password> apiVersion: metal3.io/v1alpha1 kind: BareMetalHost metadata: name: <bmh-name> namespace: <your-infraenv-namespace> 1 annotations: inspect.metal3.io: disabled bmac.agent-install.openshift.io/hostname: <hostname> 2 bmac.agent-install.openshift.io/role: <role> 3 labels: infraenvs.agent-install.openshift.io: <your-infraenv> 4 spec: online: true automatedCleaningMode: disabled 5 bootMACAddress: <your-mac-address> 6 bmc: address: <machine-address> 7 credentialsName: <bmc-secret-name> 8 rootDeviceHints: deviceName: /dev/sda 9 bmac.agent-install.openshift.io/remove-agent-and-node-on-delete: true delete bmh <bmh-name> ["--append-karg", "ip=192.0.2.2::192.0.2.254:255.255.255.0:core0.example.com:enp1s0:none", "--save-partindex", "4"] {"ignition": "version": "3.1.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}} GET /cluster.open-cluster-management.io/v1/managedclusters POST /cluster.open-cluster-management.io/v1/managedclusters { "apiVersion" : "cluster.open-cluster-management.io/v1", "kind" : "ManagedCluster", "metadata" : { "labels" : { "vendor" : "OpenShift" }, "name" : "cluster1" }, "spec": { "hubAcceptsClient": true, "managedClusterClientConfigs": [ { "caBundle": "test", "url": "https://test.com" } ] }, "status" : { } } GET /cluster.open-cluster-management.io/v1/managedclusters/{cluster_name} DELETE /cluster.open-cluster-management.io/v1/managedclusters/{cluster_name} "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?USD" GET /cluster.open-cluster-management.io/v1beta2/managedclustersets POST /cluster.open-cluster-management.io/v1beta2/managedclustersets { "apiVersion" : "cluster.open-cluster-management.io/v1beta2", "kind" : "ManagedClusterSet", "metadata" : { "name" : "clusterset1" }, "spec": { }, "status" : { } } GET /cluster.open-cluster-management.io/v1beta2/managedclustersets/{clusterset_name} DELETE /cluster.open-cluster-management.io/v1beta2/managedclustersets/{clusterset_name} GET /cluster.open-cluster-management.io/v1beta2/namespaces/{namespace}/managedclustersetbindings POST /cluster.open-cluster-management.io/v1beta2/namespaces/{namespace}/managedclustersetbindings { "apiVersion" : "cluster.open-cluster-management.io/v1", "kind" : "ManagedClusterSetBinding", "metadata" : { "name" : "clusterset1", "namespace" : "ns1" }, "spec": { "clusterSet": "clusterset1" }, "status" : { } } GET /cluster.open-cluster-management.io/v1beta2/namespaces/{namespace}/managedclustersetbindings/{clustersetbinding_name} DELETE /cluster.open-cluster-management.io/v1beta2/managedclustersetbindings/{clustersetbinding_name} GET /managedclusters.clusterview.open-cluster-management.io LIST /managedclusters.clusterview.open-cluster-management.io { "apiVersion" : "clusterview.open-cluster-management.io/v1alpha1", "kind" : "ClusterView", "metadata" : { "name" : "<user_ID>" }, "spec": { }, "status" : { } } WATCH /managedclusters.clusterview.open-cluster-management.io GET /managedclustersets.clusterview.open-cluster-management.io LIST /managedclustersets.clusterview.open-cluster-management.io WATCH /managedclustersets.clusterview.open-cluster-management.io POST /authentication.open-cluster-management.io/v1beta1/managedserviceaccounts apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.14.0 name: managedserviceaccounts.authentication.open-cluster-management.io spec: group: authentication.open-cluster-management.io names: kind: ManagedServiceAccount listKind: ManagedServiceAccountList plural: managedserviceaccounts singular: managedserviceaccount scope: Namespaced versions: - deprecated: true deprecationWarning: authentication.open-cluster-management.io/v1alpha1 ManagedServiceAccount is deprecated; use authentication.open-cluster-management.io/v1beta1 ManagedServiceAccount; version v1alpha1 will be removed in the next release name: v1alpha1 schema: openAPIV3Schema: description: ManagedServiceAccount is the Schema for the managedserviceaccounts API properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ManagedServiceAccountSpec defines the desired state of ManagedServiceAccount properties: rotation: description: Rotation is the policy for rotation the credentials. properties: enabled: default: true description: |- Enabled prescribes whether the ServiceAccount token will be rotated from the upstream type: boolean validity: default: 8640h0m0s description: Validity is the duration for which the signed ServiceAccount token is valid. type: string type: object ttlSecondsAfterCreation: description: |- ttlSecondsAfterCreation limits the lifetime of a ManagedServiceAccount. If the ttlSecondsAfterCreation field is set, the ManagedServiceAccount will be automatically deleted regardless of the ManagedServiceAccount's status. When the ManagedServiceAccount is deleted, its lifecycle guarantees (e.g. finalizers) will be honored. If this field is unset, the ManagedServiceAccount won't be automatically deleted. If this field is set to zero, the ManagedServiceAccount becomes eligible for deletion immediately after its creation. In order to use ttlSecondsAfterCreation, the EphemeralIdentity feature gate must be enabled. exclusiveMinimum: true format: int32 minimum: 0 type: integer required: - rotation type: object status: description: ManagedServiceAccountStatus defines the observed state of ManagedServiceAccount properties: conditions: description: Conditions is the condition list. items: description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t \ // other fields\n\t}" properties: lastTransitionTime: description: |- lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: description: |- message is a human readable message indicating details about the transition. This may be an empty string. maxLength: 32768 type: string observedGeneration: description: |- observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: description: |- reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?USD type: string status: description: status of the condition, one of True, False, Unknown. enum: - "True" - "False" - Unknown type: string type: description: |- type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])USD type: string required: - lastTransitionTime - message - reason - status - type type: object type: array expirationTimestamp: description: ExpirationTimestamp is the time when the token will expire. format: date-time type: string tokenSecretRef: description: |- TokenSecretRef is a reference to the corresponding ServiceAccount's Secret, which stores the CA certficate and token from the managed cluster. properties: lastRefreshTimestamp: description: |- LastRefreshTimestamp is the timestamp indicating when the token in the Secret is refreshed. format: date-time type: string name: description: Name is the name of the referenced secret. type: string required: - lastRefreshTimestamp - name type: object type: object type: object served: true storage: false subresources: status: {} - name: v1beta1 schema: openAPIV3Schema: description: ManagedServiceAccount is the Schema for the managedserviceaccounts API properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ManagedServiceAccountSpec defines the desired state of ManagedServiceAccount properties: rotation: description: Rotation is the policy for rotation the credentials. properties: enabled: default: true description: |- Enabled prescribes whether the ServiceAccount token will be rotated before it expires. Deprecated: All ServiceAccount tokens will be rotated before they expire regardless of this field. type: boolean validity: default: 8640h0m0s description: Validity is the duration of validity for requesting the signed ServiceAccount token. type: string type: object ttlSecondsAfterCreation: description: |- ttlSecondsAfterCreation limits the lifetime of a ManagedServiceAccount. If the ttlSecondsAfterCreation field is set, the ManagedServiceAccount will be automatically deleted regardless of the ManagedServiceAccount's status. When the ManagedServiceAccount is deleted, its lifecycle guarantees (e.g. finalizers) will be honored. If this field is unset, the ManagedServiceAccount won't be automatically deleted. If this field is set to zero, the ManagedServiceAccount becomes eligible for deletion immediately after its creation. In order to use ttlSecondsAfterCreation, the EphemeralIdentity feature gate must be enabled. exclusiveMinimum: true format: int32 minimum: 0 type: integer required: - rotation type: object status: description: ManagedServiceAccountStatus defines the observed state of ManagedServiceAccount properties: conditions: description: Conditions is the condition list. items: description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t \ // other fields\n\t}" properties: lastTransitionTime: description: |- lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: description: |- message is a human readable message indicating details about the transition. This may be an empty string. maxLength: 32768 type: string observedGeneration: description: |- observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: description: |- reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?USD type: string status: description: status of the condition, one of True, False, Unknown. enum: - "True" - "False" - Unknown type: string type: description: |- type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])USD type: string required: - lastTransitionTime - message - reason - status - type type: object type: array expirationTimestamp: description: ExpirationTimestamp is the time when the token will expire. format: date-time type: string tokenSecretRef: description: |- TokenSecretRef is a reference to the corresponding ServiceAccount's Secret, which stores the CA certficate and token from the managed cluster. properties: lastRefreshTimestamp: description: |- LastRefreshTimestamp is the timestamp indicating when the token in the Secret is refreshed. format: date-time type: string name: description: Name is the name of the referenced secret. type: string required: - lastRefreshTimestamp - name type: object type: object type: object served: true storage: true subresources: status: {} GET /authentication.open-cluster-management.io/v1beta1/namespaces/{namespace}/managedserviceaccounts/{managedserviceaccount_name} DELETE /authentication.open-cluster-management.io/v1beta1/namespaces/{namespace}/managedserviceaccounts/{managedserviceaccount_name} POST /apis/multicluster.openshift.io/v1alpha1/multiclusterengines { "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "controller-gen.kubebuilder.io/version": "v0.4.1" }, "creationTimestamp": null, "name": "multiclusterengines.multicluster.openshift.io" }, "spec": { "group": "multicluster.openshift.io", "names": { "kind": "MultiClusterEngine", "listKind": "MultiClusterEngineList", "plural": "multiclusterengines", "shortNames": [ "mce" ], "singular": "multiclusterengine" }, "scope": "Cluster", "versions": [ { "additionalPrinterColumns": [ { "description": "The overall state of the MultiClusterEngine", "jsonPath": ".status.phase", "name": "Status", "type": "string" }, { "jsonPath": ".metadata.creationTimestamp", "name": "Age", "type": "date" } ], "name": "v1alpha1", "schema": { "openAPIV3Schema": { "description": "MultiClusterEngine is the Schema for the multiclusterengines\nAPI", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation\nof an object. Servers should convert recognized schemas to the latest\ninternal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this\nobject represents. Servers may infer this from the endpoint the client\nsubmits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "MultiClusterEngineSpec defines the desired state of MultiClusterEngine", "properties": { "imagePullSecret": { "description": "Override pull secret for accessing MultiClusterEngine\noperand and endpoint images", "type": "string" }, "nodeSelector": { "additionalProperties": { "type": "string" }, "description": "Set the nodeselectors", "type": "object" }, "targetNamespace": { "description": "Location where MCE resources will be placed", "type": "string" }, "tolerations": { "description": "Tolerations causes all components to tolerate any taints.", "items": { "description": "The pod this Toleration is attached to tolerates any\ntaint that matches the triple <key,value,effect> using the matching\noperator <operator>.", "properties": { "effect": { "description": "Effect indicates the taint effect to match. Empty\nmeans match all taint effects. When specified, allowed values\nare NoSchedule, PreferNoSchedule and NoExecute.", "type": "string" }, "key": { "description": "Key is the taint key that the toleration applies\nto. Empty means match all taint keys. If the key is empty,\noperator must be Exists; this combination means to match all\nvalues and all keys.", "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the\nvalue. Valid operators are Exists and Equal. Defaults to Equal.\nExists is equivalent to wildcard for value, so that a pod\ncan tolerate all taints of a particular category.", "type": "string" }, "tolerationSeconds": { "description": "TolerationSeconds represents the period of time\nthe toleration (which must be of effect NoExecute, otherwise\nthis field is ignored) tolerates the taint. By default, it\nis not set, which means tolerate the taint forever (do not\nevict). Zero and negative values will be treated as 0 (evict\nimmediately) by the system.", "format": "int64", "type": "integer" }, "value": { "description": "Value is the taint value the toleration matches\nto. If the operator is Exists, the value should be empty,\notherwise just a regular string.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" }, "status": { "description": "MultiClusterEngineStatus defines the observed state of MultiClusterEngine", "properties": { "components": { "items": { "description": "ComponentCondition contains condition information for\ntracked components", "properties": { "kind": { "description": "The resource kind this condition represents", "type": "string" }, "lastTransitionTime": { "description": "LastTransitionTime is the last time the condition\nchanged from one status to another.", "format": "date-time", "type": "string" }, "message": { "description": "Message is a human-readable message indicating\ndetails about the last status change.", "type": "string" }, "name": { "description": "The component name", "type": "string" }, "reason": { "description": "Reason is a (brief) reason for the condition's\nlast status change.", "type": "string" }, "status": { "description": "Status is the status of the condition. One of True,\nFalse, Unknown.", "type": "string" }, "type": { "description": "Type is the type of the cluster condition.", "type": "string" } }, "type": "object" }, "type": "array" }, "conditions": { "items": { "properties": { "lastTransitionTime": { "description": "LastTransitionTime is the last time the condition\nchanged from one status to another.", "format": "date-time", "type": "string" }, "lastUpdateTime": { "description": "The last time this condition was updated.", "format": "date-time", "type": "string" }, "message": { "description": "Message is a human-readable message indicating\ndetails about the last status change.", "type": "string" }, "reason": { "description": "Reason is a (brief) reason for the condition's\nlast status change.", "type": "string" }, "status": { "description": "Status is the status of the condition. One of True,\nFalse, Unknown.", "type": "string" }, "type": { "description": "Type is the type of the cluster condition.", "type": "string" } }, "type": "object" }, "type": "array" }, "phase": { "description": "Latest observed overall state", "type": "string" } }, "type": "object" } }, "type": "object" } }, "served": true, "storage": true, "subresources": { "status": {} } } ] }, "status": { "acceptedNames": { "kind": "", "plural": "" }, "conditions": [], "storedVersions": [] } } GET /apis/multicluster.openshift.io/v1alpha1/multiclusterengines DELETE /apis/multicluster.openshift.io/v1alpha1/multiclusterengines/{name} GET /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placements POST /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placements { "apiVersion" : "cluster.open-cluster-management.io/v1beta1", "kind" : "Placement", "metadata" : { "name" : "placement1", "namespace": "ns1" }, "spec": { "predicates": [ { "requiredClusterSelector": { "labelSelector": { "matchLabels": { "vendor": "OpenShift" } } } } ] }, "status" : { } } GET /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placements/{placement_name} DELETE /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placements/{placement_name} GET /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placementdecisions POST /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placementdecisions { "apiVersion" : "cluster.open-cluster-management.io/v1beta1", "kind" : "PlacementDecision", "metadata" : { "labels" : { "cluster.open-cluster-management.io/placement" : "placement1" }, "name" : "placement1-decision1", "namespace": "ns1" }, "status" : { } } GET /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placementdecisions/{placementdecision_name} DELETE /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placementdecisions/{placementdecision_name} GET /config.open-cluster-management.io/v1alpha1/klusterletconfigs POST /config.open-cluster-management.io/v1alpha1/klusterletconfigs { "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "controller-gen.kubebuilder.io/version": "v0.7.0" }, "creationTimestamp": null, "name": "klusterletconfigs.config.open-cluster-management.io" }, "spec": { "group": "config.open-cluster-management.io", "names": { "kind": "KlusterletConfig", "listKind": "KlusterletConfigList", "plural": "klusterletconfigs", "singular": "klusterletconfig" }, "preserveUnknownFields": false, "scope": "Cluster", "versions": [ { "name": "v1alpha1", "schema": { "openAPIV3Schema": { "description": "KlusterletConfig contains the configuration of a klusterlet including the upgrade strategy, config overrides, proxy configurations etc.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "Spec defines the desired state of KlusterletConfig", "properties": { "appliedManifestWorkEvictionGracePeriod": { "description": "AppliedManifestWorkEvictionGracePeriod is the eviction grace period the work agent will wait before evicting the AppliedManifestWorks, whose corresponding ManifestWorks are missing on the hub cluster, from the managed cluster. If not present, the default value of the work agent will be used. If its value is set to \"INFINITE\", it means the AppliedManifestWorks will never been evicted from the managed cluster.", "pattern": "^([0-9]+(s|m|h))+USD|^INFINITEUSD", "type": "string" }, "bootstrapKubeConfigs": { "description": "BootstrapKubeConfigSecrets is the list of secrets that reflects the Klusterlet.Spec.RegistrationConfiguration.BootstrapKubeConfigs.", "properties": { "localSecretsConfig": { "description": "LocalSecretsConfig include a list of secrets that contains the kubeconfigs for ordered bootstrap kubeconifigs. The secrets must be in the same namespace where the agent controller runs.", "properties": { "hubConnectionTimeoutSeconds": { "default": 600, "description": "HubConnectionTimeoutSeconds is used to set the timeout of connecting to the hub cluster. When agent loses the connection to the hub over the timeout seconds, the agent do a rebootstrap. By default is 10 mins.", "format": "int32", "minimum": 180, "type": "integer" }, "kubeConfigSecrets": { "description": "KubeConfigSecrets is a list of secret names. The secrets are in the same namespace where the agent controller runs.", "items": { "properties": { "name": { "description": "Name is the name of the secret.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" }, "type": { "default": "None", "description": "Type specifies the type of priority bootstrap kubeconfigs. By default, it is set to None, representing no priority bootstrap kubeconfigs are set.", "enum": [ "None", "LocalSecrets" ], "type": "string" } }, "type": "object" }, "hubKubeAPIServerCABundle": { "description": "HubKubeAPIServerCABundle is the CA bundle to verify the server certificate of the hub kube API against. If not present, CA bundle will be determined with the logic below: 1). Use the certificate of the named certificate configured in APIServer/cluster if FQDN matches; 2). Otherwise use the CA certificates from kube-root-ca.crt ConfigMap in the cluster namespace; \n Deprecated and maintained for backward compatibility, use HubKubeAPIServerConfig.ServerVarificationStrategy and HubKubeAPIServerConfig.TrustedCABundles instead", "format": "byte", "type": "string" }, "hubKubeAPIServerConfig": { "description": "HubKubeAPIServerConfig specifies the settings required for connecting to the hub Kube API server. If this field is present, the below deprecated fields will be ignored: - HubKubeAPIServerProxyConfig - HubKubeAPIServerURL - HubKubeAPIServerCABundle", "properties": { "proxyURL": { "description": "ProxyURL is the URL to the proxy to be used for all requests made by client If an HTTPS proxy server is configured, you may also need to add the necessary CA certificates to TrustedCABundles.", "type": "string" }, "serverVerificationStrategy": { "description": "ServerVerificationStrategy is the strategy used for verifying the server certification; The value could be \"UseSystemTruststore\", \"UseAutoDetectedCABundle\", \"UseCustomCABundles\", empty. \n When this strategy is not set or value is empty; if there is only one klusterletConfig configured for a cluster, the strategy is eaual to \"UseAutoDetectedCABundle\", if there are more than one klusterletConfigs, the empty strategy will be overrided by other non-empty strategies.", "enum": [ "UseSystemTruststore", "UseAutoDetectedCABundle", "UseCustomCABundles" ], "type": "string" }, "trustedCABundles": { "description": "TrustedCABundles refers to a collection of user-provided CA bundles used for verifying the server certificate of the hub Kubernetes API If the ServerVerificationStrategy is set to \"UseSystemTruststore\", this field will be ignored. Otherwise, the CA certificates from the configured bundles will be appended to the klusterlet CA bundle.", "items": { "description": "CABundle is a user-provided CA bundle", "properties": { "caBundle": { "description": "CABundle refers to a ConfigMap with label \"import.open-cluster-management.io/ca-bundle\" containing the user-provided CA bundle The key of the CA data could be \"ca-bundle.crt\", \"ca.crt\", or \"tls.crt\".", "properties": { "name": { "description": "name is the metadata.name of the referenced config map", "type": "string" }, "namespace": { "description": "name is the metadata.namespace of the referenced config map", "type": "string" } }, "required": [ "name", "namespace" ], "type": "object" }, "name": { "description": "Name is the identifier used to reference the CA bundle; Do not use \"auto-detected\" as the name since it is the reserved name for the auto-detected CA bundle.", "type": "string" } }, "required": [ "caBundle", "name" ], "type": "object" }, "type": "array", "x-kubernetes-list-map-keys": [ "name" ], "x-kubernetes-list-type": "map" }, "url": { "description": "URL is the endpoint of the hub Kube API server. If not present, the .status.apiServerURL of Infrastructure/cluster will be used as the default value. e.g. `oc get infrastructure cluster -o jsonpath='{.status.apiServerURL}'`", "type": "string" } }, "type": "object" }, "hubKubeAPIServerProxyConfig": { "description": "HubKubeAPIServerProxyConfig holds proxy settings for connections between klusterlet/add-on agents on the managed cluster and the kube-apiserver on the hub cluster. Empty means no proxy settings is available. \n Deprecated and maintained for backward compatibility, use HubKubeAPIServerConfig.ProxyURL instead", "properties": { "caBundle": { "description": "CABundle is a CA certificate bundle to verify the proxy server. It will be ignored if only HTTPProxy is set; And it is required when HTTPSProxy is set and self signed CA certificate is used by the proxy server.", "format": "byte", "type": "string" }, "httpProxy": { "description": "HTTPProxy is the URL of the proxy for HTTP requests", "type": "string" }, "httpsProxy": { "description": "HTTPSProxy is the URL of the proxy for HTTPS requests HTTPSProxy will be chosen if both HTTPProxy and HTTPSProxy are set.", "type": "string" } }, "type": "object" }, "hubKubeAPIServerURL": { "description": "HubKubeAPIServerURL is the URL of the hub Kube API server. If not present, the .status.apiServerURL of Infrastructure/cluster will be used as the default value. e.g. `oc get infrastructure cluster -o jsonpath='{.status.apiServerURL}'` \n Deprecated and maintained for backward compatibility, use HubKubeAPIServerConfig.URL instead", "type": "string" }, "installMode": { "description": "InstallMode is the mode to install the klusterlet", "properties": { "noOperator": { "description": "NoOperator is the setting of klusterlet installation when install type is noOperator.", "properties": { "postfix": { "description": "Postfix is the postfix of the klusterlet name. The name of the klusterlet is \"klusterlet\" if it is not set, and \"klusterlet-{Postfix}\". The install namespace is \"open-cluster-management-agent\" if it is not set, and \"open-cluster-management-{Postfix}\".", "maxLength": 33, "pattern": "^[-a-z0-9]*[a-z0-9]USD", "type": "string" } }, "type": "object" }, "type": { "default": "default", "description": "InstallModeType is the type of install mode.", "enum": [ "default", "noOperator" ], "type": "string" } }, "type": "object" }, "nodePlacement": { "description": "NodePlacement enables explicit control over the scheduling of the agent components. If the placement is nil, the placement is not specified, it will be omitted. If the placement is an empty object, the placement will match all nodes and tolerate nothing.", "properties": { "nodeSelector": { "additionalProperties": { "type": "string" }, "description": "NodeSelector defines which Nodes the Pods are scheduled on. The default is an empty list.", "type": "object" }, "tolerations": { "description": "Tolerations are attached by pods to tolerate any taint that matches the triple <key,value,effect> using the matching operator <operator>. The default is an empty list.", "items": { "description": "The pod this Toleration is attached to tolerates any taint that matches the triple <key,value,effect> using the matching operator <operator>.", "properties": { "effect": { "description": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", "type": "string" }, "key": { "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", "type": "string" }, "tolerationSeconds": { "description": "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.", "format": "int64", "type": "integer" }, "value": { "description": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" }, "pullSecret": { "description": "PullSecret is the name of image pull secret.", "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "fieldPath": { "description": "If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: \"spec.containers{name}\" (where \"name\" refers to the name of the container that triggered the event) or if no container name is specified \"spec.containers[2]\" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.", "type": "string" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "namespace": { "description": "Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/", "type": "string" }, "resourceVersion": { "description": "Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "uid": { "description": "UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids", "type": "string" } }, "type": "object", "x-kubernetes-map-type": "atomic" }, "registries": { "description": "Registries includes the mirror and source registries. The source registry will be replaced by the Mirror.", "items": { "properties": { "mirror": { "description": "Mirror is the mirrored registry of the Source. Will be ignored if Mirror is empty.", "type": "string" }, "source": { "description": "Source is the source registry. All image registries will be replaced by Mirror if Source is empty.", "type": "string" } }, "required": [ "mirror" ], "type": "object" }, "type": "array" } }, "type": "object" }, "status": { "description": "Status defines the observed state of KlusterletConfig", "type": "object" } }, "type": "object" } }, "served": true, "storage": true, "subresources": { "status": {} } } ] }, "status": { "acceptedNames": { "kind": "", "plural": "" }, "conditions": [], "storedVersions": [] } } GET /config.open-cluster-management.io/v1alpha1/klusterletconfigs/{klusterletconfig_name} DELETE /config.open-cluster-management.io/v1alpha1/klusterletconfigs/{klusterletconfig_name} <your-directory>/cluster-scoped-resources/gather-managed.log> adm must-gather --image=registry.redhat.io/multicluster-engine/must-gather-rhel9:v2.7 --dest-dir=<directory> REGISTRY=registry.example.com:5000 IMAGE=USDREGISTRY/multicluster-engine/must-gather-rhel9@sha256:ff9f37eb400dc1f7d07a9b6f2da9064992934b69847d17f59e385783c071b9d8> adm must-gather --image=USDIMAGE --dest-dir=./data This host is pending user action. Host timed out when pulling ignition. Check the host console... Rebooting info: networking config is defined in the real root info: will not attempt to propagate initramfs networking "bmac.agent-install.openshift.io/installer-args": "[\"--append-karg\", \"coreos.force_persist_ip\"]" 2024-02-22T09:56:19-05:00 ERROR HostedCluster deletion failed {"namespace": "clusters", "name": "hosted-0", "error": "context deadline exceeded"} 2024-02-22T09:56:19-05:00 ERROR Failed to destroy cluster {"error": "context deadline exceeded"} get machine -n <hosted_cluster_namespace> NAMESPACE NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION clusters-hosted-0 hosted-0-9gg8b hosted-0-nhdbp Deleting 10h 4.14.0-rc.8 edit machines -n <hosted_cluster_namespace> get agentmachine -n <hosted_cluster_namespace> hcp destroy cluster agent --name <cluster_name> reason: Unschedulable message: '0/6 nodes are available: 3 Insufficient cpu, 3 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate.' #!/bin/bash MCE_NAMESPACE=<namespace> delete multiclusterengine --all delete apiservice v1.admission.cluster.open-cluster-management.io v1.admission.work.open-cluster-management.io delete crd discoveredclusters.discovery.open-cluster-management.io discoveryconfigs.discovery.open-cluster-management.io delete mutatingwebhookconfiguration ocm-mutating-webhook managedclustermutators.admission.cluster.open-cluster-management.io delete validatingwebhookconfiguration ocm-validating-webhook delete ns USDMCE_NAMESPACE -n multicluster-engine get pods -l app=managedcluster-import-controller-v2 -n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1 -n <managed_cluster_name> get secrets <managed_cluster_name>-import -n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1 | grep importconfig-controller get managedcluster <managed_cluster_name> -o=jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}' | grep ManagedClusterImportSucceeded -n multicluster-engine logs -l app=managedcluster-import-controller-v2 -f cluster_name=<your-managed-cluster-name> kubeconfig_secret_name=USD(oc -n USD{cluster_name} get clusterdeployments USD{cluster_name} -ojsonpath='{.spec.clusterMetadata.adminKubeconfigSecretRef.name}') -n USD{cluster_name} get secret USD{kubeconfig_secret_name} -ojsonpath={.data.kubeconfig} | base64 -d > kubeconfig.old export KUBECONFIG=kubeconfig.old get ns cluster_name=<managed_cluster_name> kubeconfig_file=<path_to_kubeconfig> kubeconfig=USD(cat USD{kubeconfig_file} | base64 -w0) kubeconfig=USD(cat USD{kubeconfig_file} | base64) kubeconfig_patch="[\{\"op\":\"replace\", \"path\":\"/data/kubeconfig\", \"value\":\"USD{kubeconfig}\"}, \{\"op\":\"replace\", \"path\":\"/data/raw-kubeconfig\", \"value\":\"USD{kubeconfig}\"}]" kubeconfig_secret_name=USD(oc -n USD{cluster_name} get clusterdeployments USD{cluster_name} -ojsonpath='{.spec.clusterMetadata.adminKubeconfigSecretRef.name}') -n USD{cluster_name} patch secrets USD{kubeconfig_secret_name} --type='json' -p="USD{kubeconfig_patch}" get pod -n open-cluster-management-agent | grep klusterlet-registration-agent logs <registration_agent_pod> -n open-cluster-management-agent get infrastructure cluster -o yaml | grep apiServerURL E0917 03:04:05.874759 1 manifestwork_controller.go:179] Reconcile work test-1-klusterlet-addon-workmgr fails with err: Failed to update work status with err Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks/test-1-klusterlet-addon-workmgr": x509: certificate signed by unknown authority E0917 03:04:05.874887 1 base_controller.go:231] "ManifestWorkAgent" controller failed to sync "test-1-klusterlet-addon-workmgr", err: Failed to update work status with err Get "api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks/test-1-klusterlet-addon-workmgr": x509: certificate signed by unknown authority E0917 03:04:37.245859 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManifestWork: failed to list *v1.ManifestWork: Get "api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks?resourceVersion=607424": x509: certificate signed by unknown authority I0917 02:27:41.525026 1 event.go:282] Event(v1.ObjectReference{Kind:"Namespace", Namespace:"open-cluster-management-agent", Name:"open-cluster-management-agent", UID:"", APIVersion:"v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'ManagedClusterAvailableConditionUpdated' update managed cluster "test-1" available condition to "True", due to "Managed cluster is available" E0917 02:58:26.315984 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1beta1.CertificateSigningRequest: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true"": x509: certificate signed by unknown authority E0917 02:58:26.598343 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManagedCluster: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true": x509: certificate signed by unknown authority E0917 02:58:27.613963 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManagedCluster: failed to list *v1.ManagedCluster: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true"": x509: certificate signed by unknown authority delete secret -n <cluster_name> <cluster_name>-import delete secret -n <cluster_name> <cluster_name>-import get secret -n <cluster_name> <cluster_name>-import -ojsonpath='{.data.import\.yaml}' | base64 --decode > import.yaml apply -f import.yaml edit managedcluster <cluster-name> time="2020-08-07T15:27:55Z" level=error msg="Error: error setting up new vSphere SOAP client: Post https://147.1.1.1/sdk: x509: cannot validate certificate for xx.xx.xx.xx because it doesn't contain any IP SANs" time="2020-08-07T15:27:55Z" level=error Error: error setting up new vSphere SOAP client: Post https://vspherehost.com/sdk: x509: certificate signed by unknown authority" x509: certificate has expired or is not yet valid time="2020-08-07T19:41:58Z" level=debug msg="vsphere_tag_category.category: Creating..." time="2020-08-07T19:41:58Z" level=error time="2020-08-07T19:41:58Z" level=error msg="Error: could not create category: POST https://vspherehost.com/rest/com/vmware/cis/tagging/category: 403 Forbidden" time="2020-08-07T19:41:58Z" level=error time="2020-08-07T19:41:58Z" level=error msg=" on ../tmp/openshift-install-436877649/main.tf line 54, in resource \"vsphere_tag_category\" \"category\":" time="2020-08-07T19:41:58Z" level=error msg=" 54: resource \"vsphere_tag_category\" \"category\" {" failed to fetch Master Machines: failed to load asset \\\"Install Config\\\": invalid \\\"install-config.yaml\\\" file: platform.vsphere.dnsVIP: Invalid value: \\\"\\\": \\\"\\\" is not a valid IP time="2020-08-11T14:31:38-04:00" level=debug msg="vsphereprivate_import_ova.import: Creating..." time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=error msg="Error: rpc error: code = Unavailable desc = transport is closing" time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=fatal msg="failed to fetch Cluster: failed to generate asset \"Cluster\": failed to create cluster: failed to apply Terraform: failed to complete the change" ERROR ERROR Error: error reconfiguring virtual machine: error processing disk changes post-clone: disk.0: ServerFaultCode: NoPermission: RESOURCE (vm-71:2000), ACTION (queryAssociatedProfile): RESOURCE (vm-71), ACTION (PolicyIDByVirtualDisk) get pod -n <new_cluster_name> logs <new_cluster_name_provision_pod_name> -n <new_cluster_name> -c hive describe clusterdeployments -n <new_cluster_name> No subnets provided for zones get klusterlets klusterlet -oyaml api-resources --verbs=list --namespaced -o name | grep -E '^secrets|^serviceaccounts|^managedclusteraddons|^roles|^rolebindings|^manifestworks|^leases|^managedclusterinfo|^appliedmanifestworks'|^clusteroauths' | xargs -n 1 oc get --show-kind --ignore-not-found -n <cluster_name> edit <resource_kind> <resource_name> -n <namespace> delete ns <cluster-name> delete secret auto-import-secret -n <cluster-namespace> describe placement <placement-name> Name: demo-placement Namespace: default Labels: <none> Annotations: <none> API Version: cluster.open-cluster-management.io/v1beta1 Kind: Placement Status: Conditions: Last Transition Time: 2022-09-30T07:39:45Z Message: Placement configurations check pass Reason: Succeedconfigured Status: False Type: PlacementMisconfigured Last Transition Time: 2022-09-30T07:39:45Z Message: No valid ManagedClusterSetBindings found in placement namespace Reason: NoManagedClusterSetBindings Status: False Type: PlacementSatisfied Number Of Selected Clusters: 0 Name: demo-placement Namespace: default Labels: <none> Annotations: <none> API Version: cluster.open-cluster-management.io/v1beta1 Kind: Placement Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DecisionCreate 2m10s placementController Decision demo-placement-decision-1 is created with placement demo-placement in namespace default Normal DecisionUpdate 2m10s placementController Decision demo-placement-decision-1 is updated with placement demo-placement in namespace default Normal ScoreUpdate 2m10s placementController cluster1:0 cluster2:100 cluster3:200 Normal DecisionUpdate 3s placementController Decision demo-placement-decision-1 is updated with placement demo-placement in namespace default Normal ScoreUpdate 3s placementController cluster1:200 cluster2:145 cluster3:189 cluster4:200 ProvisioningError 51s metal3-baremetal-controller Image provisioning failed: Deploy step deploy.deploy failed with BadRequestError: HTTP POST https://<bmc_address>/redfish/v1/Managers/iDRAC.Embedded.1/VirtualMedia/CD/Actions/VirtualMedia.InsertMedia returned code 400. Base.1.8.GeneralError: A general error has occurred. See ExtendedInfo for more information Extended information: [ {"Message": "Unable to mount remote share https://<ironic_address>/redfish/boot-<uuid>.iso.", 'MessageArgs': ["https://<ironic_address>/redfish/boot-<uuid>.iso"], "MessageArgs@odata.count": 1, "MessageId": "IDRAC.2.5.RAC0720", "RelatedProperties": ["#/Image"], "RelatedProperties@odata.count": 1, "Resolution": "Retry the operation.", "Severity": "Informational"} ] get imagecontentsourcepolicy -o json | jq -r '.items[].spec.repositoryDigestMirrors[0].mirrors[0]' get clusterversion version -ojsonpath='{.status.desired.image}' image extract --file /release-manifests/0000_50_installer_coreos-bootimages.yaml <payload_image> --confirm cat 0000_50_installer_coreos-bootimages.yaml | yq -r .data.stream | jq -r '.architectures.x86_64.images.kubevirt."digest-ref"' image mirror <rhcos_image> <internal_registry> apiVersion: config.openshift.io/v1 kind: ImageDigestMirrorSet metadata: name: rhcos-boot-kubevirt spec: repositoryDigestMirrors: - mirrors: - <rhcos_image_no_digest> 1 source: virthost.ostest.test.metalkube.org:5000/localimages/ocp-v4.0-art-dev 2 apply -f rhcos-boot-kubevirt.yaml E0809 18:45:29.450874 1 reflector.go:147] k8s.io/client-go@v0.29.4/tools/cache/reflector.go:229: Failed to watch *v1.CertificateSigningRequest: failed to list *v1.CertificateSigningRequest: Get "https://api.xxx.openshiftapps.com:443/apis/certificates.k8s.io/v1/certificatesigningrequests?limit=500&resourceVersion=0": tls: failed to verify certificate: x509: certificate signed by unknown authority apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerConfig: serverVerificationStrategy: UseSystemTruststore apply -f <filename> get secret <cluster_name>-import -n <cluster_name> -o jsonpath={.data.import\.yaml} | base64 --decode > <cluster_name>-import.yaml apply -f <cluster_name>-import.yaml get clusterversion version -o jsonpath='{.status.availableUpdates[*].version}' -n <cluster_name> get managedclusterinfo <cluster_name> -o jsonpath='{.status.distributionInfo.ocp.availableUpdates[*]}' -n <cluster_name> get ClusterCurator <cluster_name> -o yaml -n <cluster_name> delete ClusterCurator <cluster_name> -n open-cluster-management-agent-addon logs klusterlet-addon-workmgr-<your_pod_name>