GitOps
GitOps Red Hat Advanced Cluster Management for Kubernetes 2.12 GitOps Red Hat Advanced Cluster Management for Kubernetes Team
apiVersion: apps.open-cluster-management.io/v1beta1 kind: GitOpsCluster metadata: name: gitops-cluster-sample namespace: dev spec: argoServer: cluster: local-cluster argoNamespace: openshift-gitops placementRef: kind: Placement apiVersion: cluster.open-cluster-management.io/v1beta1 name: all-openshift-clusters 1 get managedclusters eks-1 NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE eks-1 true https://5E336C922AB16684A332C10535B8D407.gr7.us-east-2.eks.amazonaws.com True True 37m edit managedclusters eks-1 spec: managedClusterClientConfigs: - caBundle: ZW1wdHlDQWJ1bmRsZQo= url: https://5E336C922AB16684A332C10535B8D407.gr7.us-east-2.eks.amazonaws.com get managedclusters eks-1 NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE eks-1 true https://5E336C922AB16684A332C10535B8D407.gr7.us-east-2.eks.amazonaws.com True True 37m apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: tolerations: - key: cluster.open-cluster-management.io/unreachable operator: Exists - key: cluster.open-cluster-management.io/unavailable operator: Exists cannot create resource "services" in API group "" in the namespace "mortgage",deployments.apps is forbidden: User "system:serviceaccount:openshift-gitops:openshift-gitops-Argo CD-application-controller" kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: argo-admin subjects: - kind: ServiceAccount name: openshift-gitops-argocd-application-controller namespace: openshift-gitops roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin apiVersion: v1 kind: Namespace metadata: name: mortgage2 labels: argocd.argoproj.io/managed-by: openshift-gitops apiVersion: argoproj.io/v1alpha1 kind: `ApplicationSet` metadata: name: guestbook-allclusters-app-set namespace: openshift-gitops spec: generators: - clusterDecisionResource: configMapRef: ocm-placement-generator labelSelector: matchLabels: cluster.open-cluster-management.io/placement: aws-app-placement requeueAfterSeconds: 30 template: metadata: annotations: apps.open-cluster-management.io/ocm-managed-cluster: '{{name}}' 1 apps.open-cluster-management.io/ocm-managed-cluster-app-namespace: openshift-gitops argocd.argoproj.io/skip-reconcile: "true" 2 labels: apps.open-cluster-management.io/pull-to-ocm-managed-cluster: "true" 3 name: '{{name}}-guestbook-app' spec: destination: namespace: guestbook server: https://kubernetes.default.svc project: default sources: [ { repoURL: https://github.com/argoproj/argocd-example-apps.git targetRevision: main path: guestbook } ] syncPolicy: automated: {} syncOptions: - CreateNamespace=true NAMESPACE NAME READY STATUS open-cluster-management multicluster-integrations-7c46498d9-fqbq4 3/3 Running apiVersion: apps.open-cluster-management.io/v1alpha1 kind: MulticlusterApplicationSetReport metadata: labels: apps.open-cluster-management.io/hosting-applicationset: openshift-gitops.guestbook-allclusters-app-set name: guestbook-allclusters-app-set namespace: openshift-gitops statuses: clusterConditions: - cluster: cluster1 conditions: - message: 'Failed sync attempt: one or more objects failed to apply, reason: services is forbidden: User "system:serviceaccount:openshift-gitops:openshift-gitops-Argo CD-application-controller" cannot create resource "services" in API group "" in the namespace "guestbook",deployments.apps is forbidden: User <name> cannot create resource "deployments" in API group "apps" in the namespace "guestboo...' type: SyncError healthStatus: Missing syncStatus: OutOfSync - cluster: pcluster1 healthStatus: Progressing syncStatus: Synced - cluster: pcluster2 healthStatus: Progressing syncStatus: Synced summary: clusters: "3" healthy: "0" inProgress: "2" notHealthy: "3" notSynced: "1" synced: "2" kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: openshift-gitops-policy-admin rules: - verbs: - get - list - watch - create - update - patch - delete apiGroups: - policy.open-cluster-management.io resources: - policies - configurationpolicies - certificatepolicies - operatorpolicies - policysets - placementbindings - verbs: - get - list - watch - create - update - patch - delete apiGroups: - apps.open-cluster-management.io resources: - placementrules - verbs: - get - list - watch - create - update - patch - delete apiGroups: - cluster.open-cluster-management.io resources: - placements - placements/status - placementdecisions - placementdecisions/status kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: openshift-gitops-policy-admin subjects: - kind: ServiceAccount name: openshift-gitops-argocd-application-controller namespace: openshift-gitops roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: openshift-gitops-policy-admin -n openshift-gitops edit argocd openshift-gitops apiVersion: argoproj.io/v1beta1 kind: ArgoCD metadata: name: openshift-gitops namespace: openshift-gitops spec: kustomizeBuildOptions: --enable-alpha-plugins repo: env: - name: KUSTOMIZE_PLUGIN_HOME value: /etc/kustomize/plugin initContainers: - args: - -c - cp /policy-generator/PolicyGenerator-not-fips-compliant /policy-generator-tmp/PolicyGenerator command: - /bin/bash image: registry.redhat.io/rhacm2/multicluster-operators-subscription-rhel9:v<version> name: policy-generator-install volumeMounts: - mountPath: /policy-generator-tmp name: policy-generator volumeMounts: - mountPath: /etc/kustomize/plugin/policy.open-cluster-management.io/v1/policygenerator name: policy-generator volumes: - emptyDir: {} name: policy-generator image: '{{ (index (lookup "apps/v1" "Deployment" "open-cluster-management" "multicluster-operators-hub-subscription").spec.template.spec.containers 0).image }}' env: - name: POLICY_GEN_ENABLE_HELM value: "true" kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: openshift-gitops-policy-admin subjects: - kind: ServiceAccount name: openshift-gitops-argocd-application-controller namespace: openshift-gitops roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: openshift-gitops-policy-admin wget https://raw.githubusercontent.com/open-cluster-management-io/policy-collection/main/stable/CM-Configuration-Management/argocd-policy-healthchecks.yaml apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: openshift-gitops-operator namespace: openshift-operators spec: channel: stable name: openshift-gitops-operator source: redhat-operators sourceNamespace: openshift-marketplace apiVersion: policy.open-cluster-management.io/v1 kind: PolicyGenerator metadata: name: install-openshift-gitops policyDefaults: namespace: policies placement: clusterSelectors: vendor: "OpenShift" remediationAction: enforce policies: - name: install-openshift-gitops manifests: - path: openshift-gitops-subscription.yaml generators: - policy-generator-config.yaml apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: name: placement-install-openshift-gitops namespace: policies spec: clusterSelector: matchExpressions: - key: vendor operator: In values: - OpenShift --- apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-install-openshift-gitops namespace: policies placementRef: apiGroup: apps.open-cluster-management.io kind: PlacementRule name: placement-install-openshift-gitops subjects: - apiGroup: policy.open-cluster-management.io kind: Policy name: install-openshift-gitops --- apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: annotations: policy.open-cluster-management.io/categories: CM Configuration Management policy.open-cluster-management.io/controls: CM-2 Baseline Configuration policy.open-cluster-management.io/standards: NIST SP 800-53 policy.open-cluster-management.io/description: name: install-openshift-gitops namespace: policies spec: disabled: false policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: install-openshift-gitops spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: openshift-gitops-operator namespace: openshift-operators spec: channel: stable name: openshift-gitops-operator source: redhat-operators sourceNamespace: openshift-marketplace remediationAction: enforce severity: low apiVersion: rbac.open-cluster-management.io/v1alpha1 kind: ClusterPermission metadata: name: <clusterpermission-msa-subject-sample> namespace: <managed cluster> spec: roles: - namespace: default rules: - apiGroups: ["apps"] resources: ["deployments"] verbs: ["get", "list", "create", "update", "delete", "patch"] - apiGroups: [""] resources: ["configmaps", "secrets", "pods", "podtemplates", "persistentvolumeclaims", "persistentvolumes"] verbs: ["get", "update", "list", "create", "delete", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["*"] verbs: ["list"] - namespace: mortgage rules: - apiGroups: ["apps"] resources: ["deployments"] verbs: ["get", "list", "create", "update", "delete", "patch"] - apiGroups: [""] resources: ["configmaps", "secrets", "pods", "services", "namespace"] verbs: ["get", "update", "list", "create", "delete", "patch"] clusterRole: rules: - apiGroups: ["*"] resources: ["*"] verbs: ["get", "list"] roleBindings: - namespace: default roleRef: kind: Role subject: apiGroup: authentication.open-cluster-management.io kind: ManagedServiceAccount name: <managed-sa-sample> - namespace: mortgage roleRef: kind: Role subject: apiGroup: authentication.open-cluster-management.io kind: ManagedServiceAccount name: <managed-sa-sample> clusterRoleBinding: subject: apiGroup: authentication.open-cluster-management.io kind: ManagedServiceAccount name: <managed-sa-sample> apiVersion: apps.open-cluster-management.io/v1beta1 kind: GitOpsCluster metadata: name: argo-acm-importer namespace: openshift-gitops spec: managedServiceAccountRef: <managed-sa-sample> argoServer: cluster: notused argoNamespace: openshift-gitops placementRef: kind: Placement apiVersion: cluster.open-cluster-management.io/v1beta1 name: all-openshift-clusters namespace: openshift-gitops get secrets -n openshift-gitops <managed cluster-managed-sa-sample-cluster-secret> NAME TYPE DATA AGE <managed cluster-managed-sa-sample-cluster-secret> Opaque 3 4m2s When the GitOpsCluster resource is updated with the `managedServiceAccountRef`, each managed cluster in the placement of this GitOpsCluster needs to have the service account. If you have several managed clusters, it becomes tedious for you to create the managed service account and cluster permission for each managed cluster. You can simply this process by using a policy to create the managed service account and cluster permission for all your managed clusters apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-gitops namespace: openshift-gitops annotations: policy.open-cluster-management.io/standards: NIST-CSF policy.open-cluster-management.io/categories: PR.PT Protective Technology policy.open-cluster-management.io/controls: PR.PT-3 Least Functionality spec: remediationAction: enforce disabled: false policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-gitops-sub spec: pruneObjectBehavior: None remediationAction: enforce severity: low object-templates-raw: | {{ range USDplacedec := (lookup "cluster.open-cluster-management.io/v1beta1" "PlacementDecision" "openshift-gitops" "" "cluster.open-cluster-management.io/placement=aws-app-placement").items }} {{ range USDclustdec := USDplacedec.status.decisions }} - complianceType: musthave objectDefinition: apiVersion: authentication.open-cluster-management.io/v1alpha1 kind: ManagedServiceAccount metadata: name: <managed-sa-sample> namespace: {{ USDclustdec.clusterName }} spec: rotation: {} - complianceType: musthave objectDefinition: apiVersion: rbac.open-cluster-management.io/v1alpha1 kind: ClusterPermission metadata: name: <clusterpermission-msa-subject-sample> namespace: {{ USDclustdec.clusterName }} spec: roles: - namespace: default rules: - apiGroups: ["apps"] resources: ["deployments"] verbs: ["get", "list", "create", "update", "delete"] - apiGroups: [""] resources: ["configmaps", "secrets", "pods", "podtemplates", "persistentvolumeclaims", "persistentvolumes"] verbs: ["get", "update", "list", "create", "delete"] - apiGroups: ["storage.k8s.io"] resources: ["*"] verbs: ["list"] - namespace: mortgage rules: - apiGroups: ["apps"] resources: ["deployments"] verbs: ["get", "list", "create", "update", "delete"] - apiGroups: [""] resources: ["configmaps", "secrets", "pods", "services", "namespace"] verbs: ["get", "update", "list", "create", "delete"] clusterRole: rules: - apiGroups: ["*"] resources: ["*"] verbs: ["get", "list"] roleBindings: - namespace: default roleRef: kind: Role subject: apiGroup: authentication.open-cluster-management.io kind: ManagedServiceAccount name: <managed-sa-sample> - namespace: mortgage roleRef: kind: Role subject: apiGroup: authentication.open-cluster-management.io kind: ManagedServiceAccount name: <managed-sa-sample> clusterRoleBinding: subject: apiGroup: authentication.open-cluster-management.io kind: ManagedServiceAccount name: <managed-sa-sample> {{ end }} {{ end }} --- apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-gitops namespace: openshift-gitops placementRef: name: lc-app-placement kind: Placement apiGroup: cluster.open-cluster-management.io subjects: - name: policy-gitops kind: Policy apiGroup: policy.open-cluster-management.io --- apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: lc-app-placement namespace: openshift-gitops spec: numberOfClusters: 1 predicates: - requiredClusterSelector: labelSelector: matchLabels: name: local-cluster