Troubleshooting
Troubleshooting Red Hat Advanced Cluster Management for Kubernetes 2.11 Troubleshooting
adm must-gather --image=registry.redhat.io/rhacm2/acm-must-gather-rhel9:v2.11 --dest-dir=<directory> <your-directory>/cluster-scoped-resources/gather-managed.log> REGISTRY=<internal.repo.address:port> IMAGE1=USDREGISTRY/rhacm2/acm-must-gather-rhel9:v<2.x> adm must-gather --image=USDIMAGE1 --dest-dir=<directory> adm must-gather --image=quay.io/stolostron/backplane-must-gather:SNAPSHOTNAME /usr/bin/gather hosted-cluster-namespace=HOSTEDCLUSTERNAMESPACE hosted-cluster-name=HOSTEDCLUSTERNAME adm must-gather --image=quay.io/stolostron/backplane-must-gather:SNAPSHOTNAME /usr/bin/gather hosted-cluster-namespace=HOSTEDCLUSTERNAMESPACE hosted-cluster-name=HOSTEDCLUSTERNAME --dest-dir=NAME ; tar -cvzf NAME.tgz NAME REGISTRY=registry.example.com:5000 IMAGE=USDREGISTRY/multicluster-engine/must-gather-rhel8@sha256:ff9f37eb400dc1f7d07a9b6f2da9064992934b69847d17f59e385783c071b9d8 adm must-gather --image=USDIMAGE /usr/bin/gather hosted-cluster-namespace=HOSTEDCLUSTERNAMESPACE hosted-cluster-name=HOSTEDCLUSTERNAME --dest-dir=./data reason: Unschedulable message: '0/6 nodes are available: 3 Insufficient cpu, 3 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate.' #!/bin/bash ACM_NAMESPACE=<namespace> delete mch --all -n USDACM_NAMESPACE delete apiservice v1.admission.cluster.open-cluster-management.io v1.admission.work.open-cluster-management.io delete clusterimageset --all delete clusterrole multiclusterengines.multicluster.openshift.io-v1-admin multiclusterengines.multicluster.openshift.io-v1-crdview multiclusterengines.multicluster.openshift.io-v1-edit multiclusterengines.multicluster.openshift.io-v1-view open-cluster-management:addons:application-manager open-cluster-management:admin-aggregate open-cluster-management:cert-policy-controller-hub open-cluster-management:cluster-manager-admin-aggregate open-cluster-management:config-policy-controller-hub open-cluster-management:edit-aggregate open-cluster-management:iam-policy-controller-hub open-cluster-management:policy-framework-hub open-cluster-management:view-aggregate delete crd klusterletaddonconfigs.agent.open-cluster-management.io placementbindings.policy.open-cluster-management.io policies.policy.open-cluster-management.io userpreferences.console.open-cluster-management.io discoveredclusters.discovery.open-cluster-management.io discoveryconfigs.discovery.open-cluster-management.io delete mutatingwebhookconfiguration ocm-mutating-webhook managedclustermutators.admission.cluster.open-cluster-management.io multicluster-observability-operator delete validatingwebhookconfiguration channels.apps.open.cluster.management.webhook.validator application-webhook-validator multiclusterhub-operator-validating-webhook ocm-validating-webhook multicluster-observability-operator multiclusterengines.multicluster.openshift.io Error from server: request to convert CR from an invalid group/version: cluster.open-cluster-management.io/v1beta1 annotate mce multiclusterengine pause=true patch deployment cluster-manager -n multicluster-engine -p \ '{"spec":{"template":{"spec":{"containers":[{"name":"registration-operator","image":"registry.redhat.io/multicluster-engine/registration-operator-rhel8@sha256:35999c3a1022d908b6fe30aa9b85878e666392dbbd685e9f3edcb83e3336d19f"}]}}}}' export ORIGIN_REGISTRATION_IMAGE=USD(oc get clustermanager cluster-manager -o jsonpath='{.spec.registrationImagePullSpec}') patch clustermanager cluster-manager --type='json' -p='[{"op": "replace", "path": "/spec/registrationImagePullSpec", "value": "registry.redhat.io/multicluster-engine/registration-rhel8@sha256:a3c22aa4326859d75986bf24322068f0aff2103cccc06e1001faaf79b9390515"}]' annotate crds managedclustersets.cluster.open-cluster-management.io operator.open-cluster-management.io/version- annotate crds managedclustersetbindings.cluster.open-cluster-management.io operator.open-cluster-management.io/version- -n multicluster-engine delete pods -l app=cluster-manager wait crds managedclustersets.cluster.open-cluster-management.io --for=jsonpath="{.metadata.annotations['operator\.open-cluster-management\.io/version']}"="2.3.3" --timeout=120s wait crds managedclustersetbindings.cluster.open-cluster-management.io --for=jsonpath="{.metadata.annotations['operator\.open-cluster-management\.io/version']}"="2.3.3" --timeout=120s patch StorageVersionMigration managedclustersets.cluster.open-cluster-management.io --type='json' -p='[{"op":"replace", "path":"/spec/resource/version", "value":"v1beta1"}]' patch StorageVersionMigration managedclustersets.cluster.open-cluster-management.io --type='json' --subresource status -p='[{"op":"remove", "path":"/status/conditions"}]' patch StorageVersionMigration managedclustersetbindings.cluster.open-cluster-management.io --type='json' -p='[{"op":"replace", "path":"/spec/resource/version", "value":"v1beta1"}]' patch StorageVersionMigration managedclustersetbindings.cluster.open-cluster-management.io --type='json' --subresource status -p='[{"op":"remove", "path":"/status/conditions"}]' wait storageversionmigration managedclustersets.cluster.open-cluster-management.io --for=condition=Succeeded --timeout=120s wait storageversionmigration managedclustersetbindings.cluster.open-cluster-management.io --for=condition=Succeeded --timeout=120s annotate mce multiclusterengine pause- patch clustermanager cluster-manager --type='json' -p='[{"op": "replace", "path": "/spec/registrationImagePullSpec", "value": "'USDORIGIN_REGISTRATION_IMAGE'"}]' get managedclusterset get managedclustersetbinding -A -n multicluster-engine get pods -l app=managedcluster-import-controller-v2 -n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1 -n <managed_cluster_name> get secrets <managed_cluster_name>-import -n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1 | grep importconfig-controller get managedcluster <managed_cluster_name> -o=jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}' | grep ManagedClusterImportSucceeded get pod -n open-cluster-management-agent | grep klusterlet-registration-agent logs <registration_agent_pod> -n open-cluster-management-agent get infrastructure cluster -o yaml | grep apiServerURL error log: Warning: apiextensions.k8s.io/v1beta1 CustomResourceDefinition is deprecated in v1.16+, unavailable in v1.22+; use apiextensions.k8s.io/v1 CustomResourceDefinition Error from server (AlreadyExists): error when creating "STDIN": customresourcedefinitions.apiextensions.k8s.io "klusterlets.operator.open-cluster-management.io" already exists The cluster cannot be imported because its Klusterlet CRD already exists. Either the cluster was already imported, or it was not detached completely during a previous detach process. Detach the existing cluster before trying the import again." get all -n open-cluster-management-agent get all -n open-cluster-management-agent-addon get klusterlet | grep klusterlet | awk '{print USD1}' | xargs oc patch klusterlet --type=merge -p '{"metadata":{"finalizers": []}}' delete namespaces open-cluster-management-agent open-cluster-management-agent-addon --wait=false get crds | grep open-cluster-management.io | awk '{print USD1}' | xargs oc delete crds --wait=false get crds | grep open-cluster-management.io | awk '{print USD1}' | xargs oc patch crds --type=merge -p '{"metadata":{"finalizers": []}}' time="2020-08-07T15:27:55Z" level=error msg="Error: error setting up new vSphere SOAP client: Post https://147.1.1.1/sdk: x509: cannot validate certificate for xx.xx.xx.xx because it doesn't contain any IP SANs" time="2020-08-07T15:27:55Z" level=error Error: error setting up new vSphere SOAP client: Post https://vspherehost.com/sdk: x509: certificate signed by unknown authority" x509: certificate has expired or is not yet valid time="2020-08-07T19:41:58Z" level=debug msg="vsphere_tag_category.category: Creating..." time="2020-08-07T19:41:58Z" level=error time="2020-08-07T19:41:58Z" level=error msg="Error: could not create category: POST https://vspherehost.com/rest/com/vmware/cis/tagging/category: 403 Forbidden" time="2020-08-07T19:41:58Z" level=error time="2020-08-07T19:41:58Z" level=error msg=" on ../tmp/openshift-install-436877649/main.tf line 54, in resource \"vsphere_tag_category\" \"category\":" time="2020-08-07T19:41:58Z" level=error msg=" 54: resource \"vsphere_tag_category\" \"category\" {" failed to fetch Master Machines: failed to load asset \\\"Install Config\\\": invalid \\\"install-config.yaml\\\" file: platform.vsphere.dnsVIP: Invalid value: \\\"\\\": \\\"\\\" is not a valid IP time="2020-08-11T14:31:38-04:00" level=debug msg="vsphereprivate_import_ova.import: Creating..." time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=error msg="Error: rpc error: code = Unavailable desc = transport is closing" time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=fatal msg="failed to fetch Cluster: failed to generate asset \"Cluster\": failed to create cluster: failed to apply Terraform: failed to complete the change" ERROR ERROR Error: error reconfiguring virtual machine: error processing disk changes post-clone: disk.0: ServerFaultCode: NoPermission: RESOURCE (vm-71:2000), ACTION (queryAssociatedProfile): RESOURCE (vm-71), ACTION (PolicyIDByVirtualDisk) clouds: openstack: cacert: "/etc/pki/ca-trust/source/anchors/ca.crt" spec: baseDomain: dev09.red-chesterfield.com clusterName: txue-osspoke platform: openstack: cloud: openstack credentialsSecretRef: name: txue-osspoke-openstack-creds certificatesSecretRef: name: txue-osspoke-openstack-certificatebundle create secret generic txue-osspoke-openstack-certificatebundle --from-file=ca.crt=ca.crt.pem -n USDCLUSTERNAME customresourcedefinition.apiextensions.k8s.io/klusterlets.operator.open-cluster-management.io configured clusterrole.rbac.authorization.k8s.io/klusterlet configured clusterrole.rbac.authorization.k8s.io/open-cluster-management:klusterlet-admin-aggregate-clusterrole configured clusterrolebinding.rbac.authorization.k8s.io/klusterlet configured namespace/open-cluster-management-agent configured secret/open-cluster-management-image-pull-credentials unchanged serviceaccount/klusterlet configured deployment.apps/klusterlet unchanged klusterlet.operator.open-cluster-management.io/klusterlet configured Error from server (BadRequest): error when creating "STDIN": Secret in version "v1" cannot be handled as a Secret: v1.Secret.ObjectMeta: v1.ObjectMeta.TypeMeta: Kind: Data: decode base64: illegal base64 data at input byte 1313, error found in #10 byte of ...|dhruy45="},"kind":"|..., bigger context ...|tye56u56u568yuo7i67i67i67o556574i"},"kind":"Secret","metadata":{"annotations":{"kube| version apply -f import.yaml E0917 03:04:05.874759 1 manifestwork_controller.go:179] Reconcile work test-1-klusterlet-addon-workmgr fails with err: Failed to update work status with err Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks/test-1-klusterlet-addon-workmgr": x509: certificate signed by unknown authority E0917 03:04:05.874887 1 base_controller.go:231] "ManifestWorkAgent" controller failed to sync "test-1-klusterlet-addon-workmgr", err: Failed to update work status with err Get "api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks/test-1-klusterlet-addon-workmgr": x509: certificate signed by unknown authority E0917 03:04:37.245859 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManifestWork: failed to list *v1.ManifestWork: Get "api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks?resourceVersion=607424": x509: certificate signed by unknown authority I0917 02:27:41.525026 1 event.go:282] Event(v1.ObjectReference{Kind:"Namespace", Namespace:"open-cluster-management-agent", Name:"open-cluster-management-agent", UID:"", APIVersion:"v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'ManagedClusterAvailableConditionUpdated' update managed cluster "test-1" available condition to "True", due to "Managed cluster is available" E0917 02:58:26.315984 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1beta1.CertificateSigningRequest: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true"": x509: certificate signed by unknown authority E0917 02:58:26.598343 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManagedCluster: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true": x509: certificate signed by unknown authority E0917 02:58:27.613963 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManagedCluster: failed to list *v1.ManagedCluster: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true"": x509: certificate signed by unknown authority delete secret -n <cluster_name> <cluster_name>-import delete secret -n <cluster_name> <cluster_name>-import get secret -n <cluster_name> <cluster_name>-import -ojsonpath='{.data.import\.yaml}' | base64 --decode > import.yaml apply -f import.yaml api-resources --verbs=list --namespaced -o name | grep -E '^secrets|^serviceaccounts|^managedclusteraddons|^roles|^rolebindings|^manifestworks|^leases|^managedclusterinfo|^appliedmanifestworks'|^clusteroauths' | xargs -n 1 oc get --show-kind --ignore-not-found -n <cluster_name> edit <resource_kind> <resource_name> -n <namespace> delete ns <cluster-name> delete secret auto-import-secret -n <cluster-namespace> apiVersion: snapshot.storage.k8s.io/v1 deletionPolicy: Delete driver: cinder.csi.openstack.org kind: VolumeSnapshotClass metadata: annotations: snapshot.storage.kubernetes.io/is-default-class: 'true' name: standard-csi parameters: force-create: 'true' adm must-gather --image=quay.io/stolostron/must-gather:SNAPSHOTNAME adm must-gather --image=quay.io/stolostron/must-gather:SNAPSHOTNAME --dest-dir=<SOMENAME> ; tar -cvzf <SOMENAME>.tgz <SOMENAME> There are two ways to access the provisioned PostgreSQL database. exec -it multicluster-global-hub-postgres-0 -c multicluster-global-hub-postgres -n multicluster-global-hub -- psql -U postgres -d hoh Or access the database installed by crunchy operator exec -it USD(kubectl get pods -n multicluster-global-hub -l postgres-operator.crunchydata.com/role=master -o jsonpath='{.items..metadata.name}') -c database -n multicluster-global-hub -- psql -U postgres -d hoh -c "SELECT 1" cat <<EOF | oc apply -f - apiVersion: v1 kind: Service metadata: name: multicluster-global-hub-postgres-lb namespace: multicluster-global-hub spec: ports: - name: postgres port: 5432 protocol: TCP targetPort: 5432 selector: name: multicluster-global-hub-postgres type: LoadBalancer EOF Host get svc postgres-ha -ojsonpath='{.status.loadBalancer.ingress[0].hostname}' Password get secrets -n multicluster-global-hub postgres-pguser-postgres -o go-template='{{index (.data) "password" | base64decode}}' patch postgrescluster postgres -n multicluster-global-hub -p '{"spec":{"service":{"type":"LoadBalancer"}}}' --type merge Host get svc -n multicluster-global-hub postgres-ha -ojsonpath='{.status.loadBalancer.ingress[0].hostname}' Username get secrets -n multicluster-global-hub postgres-pguser-postgres -o go-template='{{index (.data) "user" | base64decode}}' Password get secrets -n multicluster-global-hub postgres-pguser-postgres -o go-template='{{index (.data) "password" | base64decode}}' Database get secrets -n multicluster-global-hub postgres-pguser-postgres -o go-template='{{index (.data) "dbname" | base64decode}}' pg_dump hoh > hoh.sql pg_dump -h my.host.com -p 5432 -U postgres -F t hoh -f hoh-USD(date +%d-%m-%y_%H-%M).tar psql -h another.host.com -p 5432 -U postgres -d hoh < hoh.sql pg_restore -h another.host.com -p 5432 -U postgres -d hoh hoh-USD(date +%d-%m-%y_%H-%M).tar edit managedcluster <cluster-name> apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: <cluster-name> spec: hubAcceptsClient: true leaseDurationSeconds: 60 get pod -n <new_cluster_name> logs <new_cluster_name_provision_pod_name> -n <new_cluster_name> -c hive describe clusterdeployments -n <new_cluster_name> No subnets provided for zones get secret grafana-config -n open-cluster-management-observability -o jsonpath="{.data.grafana\.ini}" | base64 -d | grep dataproxy -A 4 [dataproxy] timeout = 300 dial_timeout = 30 keep_alive_seconds = 300 get secret/grafana-datasources -n open-cluster-management-observability -o jsonpath="{.data.datasources\.yaml}" | base64 -d | grep queryTimeout queryTimeout: 300s annotate route grafana -n open-cluster-management-observability --overwrite haproxy.router.openshift.io/timeout=300s % oc get managedclusters NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE local-cluster true True True 56d cluster1 true True True 16h apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: name: all-ready-clusters namespace: default spec: clusterSelector: {} status: decisions: - clusterName: cluster1 clusterNamespace: cluster1 apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: managedcluster-admin-user-zisis namespace: local-cluster rules: - apiGroups: - cluster.open-cluster-management.io resources: - managedclusters verbs: - get apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: managedcluster-admin-user-zisis namespace: local-cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: managedcluster-admin-user-zisis namespace: local-cluster subjects: - kind: User name: zisis apiGroup: rbac.authorization.k8s.io failed to install release: unable to build kubernetes objects from release manifest: unable to recognize "": no matches for kind "Deployment" in version "extensions/v1beta1" error: unable to recognize "old.yaml": no matches for kind "Deployment" in version "deployment/v1beta1" apiVersion: apps/v1 kind: Deployment explain <resource> get klusterlets klusterlet -oyaml apiVersion: apps.open-cluster-management.io/v1 kind: Channel metadata: name: deva namespace: ch-obj labels: name: obj-sub spec: type: ObjectBucket pathname: http://ec2-100-26-232-156.compute-1.amazonaws.com:9000/deva sourceNamespaces: - default secretRef: name: dev --- apiVersion: v1 kind: Secret metadata: name: dev namespace: ch-obj labels: name: obj-sub data: AccessKeyID: YWRtaW4= SecretAccessKey: cGFzc3dvcmRhZG1pbg== annotate appsub -n <subscription-namespace> <subscription-name> test=true get pods -n open-cluster-management|grep observability get crd|grep observ multiclusterobservabilities.observability.open-cluster-management.io observabilityaddons.observability.open-cluster-management.io observatoria.core.observatorium.io storageclass.kubernetes.io/is-default-class: "true" error: response status code is 500 Internal Server Error, response body is x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "observability-client-ca-certificate") delete secret observability-controller-open-cluster-management.io-observability-signer-client-cert -n open-cluster-management-addon-observability project open-cluster-management patch search -n open-cluster-management search-v2-operator --type json -p '[{"op": "add", "path": "/spec/deployments/database/resources", "value": {"limits": {"memory": "16Gi"}, "requests": {"memory": "32Mi", "cpu": "25m"}}}]' annotate search search-v2-operator search-pause=true edit cm search-postgres -n open-cluster-management postgresql.conf: |- work_mem = '128MB' # Higher values allocate more memory max_parallel_workers_per_gather = '0' # Disables parallel queries shared_buffers = '1GB' # Higher values allocate more memory delete pod search-postgres-xyz search-api-xzy get cm search-postgres -n open-cluster-management -o yaml ts=2024-01-24T15:34:51.948653839Z caller=compact.go:491 level=error msg="critical error detected; halting" err="compaction: group 0@5827190780573537664: compact blocks [ /var/thanos/compact/compact/0@15699422364132557315/01HKZGQGJCKQWF3XMA8EXAMPLE]: 2 errors: populate block: add series: write series data: write /var/thanos/compact/compact/0@15699422364132557315/01HKZGQGJCKQWF3XMA8EXAMPLE.tmp-for-creation/index: no space left on device; write /var/thanos/compact/compact/0@15699422364132557315/01HKZGQGJCKQWF3XMA8EXAMPLE.tmp-for-creation/index: no space left on device" delete pod observability-thanos-compact-0 -n open-cluster-management-observability ts=2024-01-24T15:34:51.948653839Z caller=compact.go:491 level=error msg="critical error detected; halting" err="compaction: group 0@15699422364132557315: compact blocks [/var/thanos/compact/compact/0@15699422364132557315/01HKZGQGJCKQWF3XMA8EXAMPLE /var/thanos/compact/compact/0@15699422364132557315/01HKZQK7TD06J2XWGR5EXAMPLE /var/thanos/compact/compact/0@15699422364132557315/01HKZYEZ2DVDQXF1STVEXAMPLE /var/thanos/compact/compact/0@15699422364132557315/01HM05APAHXBQSNC0N5EXAMPLE]: populate block: chunk iter: cannot populate chunk 8 from block 01HKZYEZ2DVDQXF1STVEXAMPLE: segment index 0 out of range" rsh observability-thanos-compact-0 [..] thanos tools bucket verify -r --objstore.config="USDOBJSTORE_CONFIG" --objstore-backup.config="USDOBJSTORE_CONFIG" --id=01HKZYEZ2DVDQXF1STVEXAMPLE thanos tools bucket mark --id "01HKZYEZ2DVDQXF1STVEXAMPLE" --objstore.config="USDOBJSTORE_CONFIG" --marker=deletion-mark.json --details=DELETE thanos tools bucket cleanup --objstore.config="USDOBJSTORE_CONFIG" subctl diagnose all --kubeconfig <path-to-kubeconfig-file> subctl diagnose firewall inter-cluster <path-to-local-kubeconfig> <path-to-remote-cluster-kubeconfig> subctl verify --kubecontexts <cluster1>,<cluster2> [flags] message: >- [spec.tls.caCertificate: Invalid value: "redacted ca certificate data": failed to parse CA certificate: data does not contain any valid RSA or ECDSA certificates, spec.tls.certificate: Invalid value: "redacted certificate data": data does not contain any valid RSA or ECDSA certificates, spec.tls.key: Invalid value: "": no key specified] tls: certificate: | {{ print "{{hub fromSecret "open-cluster-management" "minio-cert" "tls.crt" hub}}" | base64dec | autoindent }}