Machine configuration
Machine configuration OpenShift Container Platform 4.17 Managing and applying configuration and updates of the base operating system and container runtimes in OpenShift Container Platform Red Hat OpenShift Documentation Team
oc get mcp worker NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE worker rendered-worker-404caf3180818d8ac1f50c32f14b57c3 False True True 2 1 1 1 5h51m oc describe mcp worker Last Transition Time: 2021-12-20T18:54:00Z Message: Node ci-ln-j4h8nkb-72292-pxqxz-worker-a-fjks4 is reporting: "content mismatch for file \"/etc/mco-test-file\"" 1 Reason: 1 nodes are reporting degraded status on sync Status: True Type: NodeDegraded 2 oc describe node/ci-ln-j4h8nkb-72292-pxqxz-worker-a-fjks4 Annotations: cloud.network.openshift.io/egress-ipconfig: [{"interface":"nic0","ifaddr":{"ipv4":"10.0.128.0/17"},"capacity":{"ip":10}}] csi.volume.kubernetes.io/nodeid: {"pd.csi.storage.gke.io":"projects/openshift-gce-devel-ci/zones/us-central1-a/instances/ci-ln-j4h8nkb-72292-pxqxz-worker-a-fjks4"} machine.openshift.io/machine: openshift-machine-api/ci-ln-j4h8nkb-72292-pxqxz-worker-a-fjks4 machineconfiguration.openshift.io/controlPlaneTopology: HighlyAvailable machineconfiguration.openshift.io/currentConfig: rendered-worker-67bd55d0b02b0f659aef33680693a9f9 machineconfiguration.openshift.io/desiredConfig: rendered-worker-67bd55d0b02b0f659aef33680693a9f9 machineconfiguration.openshift.io/reason: content mismatch for file "/etc/mco-test-file" 1 machineconfiguration.openshift.io/state: Degraded 2 oc get machineconfigpool NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-06c9c4... True False False 3 3 3 0 4h42m worker rendered-worker-f4b64... False True False 3 2 2 0 4h42m NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-06c9c4... True False False 3 3 3 0 4h42m worker rendered-worker-c1b41a... False True False 3 2 3 0 4h42m oc describe mcp worker Degraded Machine Count: 0 Machine Count: 3 Observed Generation: 2 Ready Machine Count: 3 Unavailable Machine Count: 0 Updated Machine Count: 3 Events: <none> Degraded Machine Count: 0 Machine Count: 3 Observed Generation: 2 Ready Machine Count: 2 Unavailable Machine Count: 1 Updated Machine Count: 3 oc get machineconfigs NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m 00-worker 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m 01-master-container-runtime 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m 01-master-kubelet 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m rendered-master-dde... 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m rendered-worker-fde... 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m oc describe machineconfigs 01-master-kubelet Name: 01-master-kubelet Spec: Config: Ignition: Version: 3.2.0 Storage: Files: Contents: Source: data:, Mode: 420 Overwrite: true Path: /etc/kubernetes/cloud.conf Contents: Source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fkubelet-ca.crt%0A%20%20anonymous Mode: 420 Overwrite: true Path: /etc/kubernetes/kubelet.conf Systemd: Units: Contents: [Unit] Description=Kubernetes Kubelet Wants=rpc-statd.service network-online.target crio.service After=network-online.target crio.service ExecStart=/usr/bin/hyperkube kubelet --config=/etc/kubernetes/kubelet.conf \ oc delete -f ./myconfig.yaml oc get machineconfignodes NAME UPDATED UPDATEPREPARED UPDATEEXECUTED UPDATEPOSTACTIONCOMPLETED UPDATECOMPLETED RESUMED ip-10-0-12-194.ec2.internal True False False False False False ip-10-0-17-102.ec2.internal False True False False False False ip-10-0-2-232.ec2.internal False False True False False False ip-10-0-59-251.ec2.internal False False False True False False ip-10-0-59-56.ec2.internal False False False False True True ip-10-0-6-214.ec2.internal False False Unknown False False False oc get machineconfignodes USD(oc get machineconfignodes -o json | jq -r '.items[]|select(.spec.pool.name=="<pool_name>")|.metadata.name') 1 NAME UPDATED UPDATEPREPARED UPDATEEXECUTED UPDATEPOSTACTIONCOMPLETE UPDATECOMPLETE RESUMED ip-10-0-48-226.ec2.internal True False False False False False ip-10-0-5-241.ec2.internal True False False False False False ip-10-0-74-108.ec2.internal True False False False False False oc describe machineconfignode/<node_name> 1 Name: <node_name> Namespace: Labels: <none> Annotations: <none> API Version: machineconfiguration.openshift.io/v1alpha1 Kind: MachineConfigNode Metadata: Creation Timestamp: 2023-10-17T13:08:58Z Generation: 1 Resource Version: 49443 UID: 4bd758ab-2187-413c-ac42-882e61761b1d Spec: Node Ref: Name: <node_name> Pool: Name: master ConfigVersion: Desired: rendered-worker-823ff8dc2b33bf444709ed7cd2b9855b 1 Status: Conditions: Last Transition Time: 2023-10-17T13:09:02Z Message: Node has completed update to config rendered-master-cf99e619747ab19165f11e3546c71f1e Reason: NodeUpgradeComplete Status: True Type: Updated Last Transition Time: 2023-10-17T13:09:02Z Message: This node has not yet entered the UpdatePreparing phase Reason: NotYetOccured Status: False Config Version: Current: rendered-worker-823ff8dc2b33bf444709ed7cd2b9855b Desired: rendered-worker-823ff8dc2b33bf444709ed7cd2b9855b 2 Health: Healthy Most Recent Error: Observed Generation: 3 oc get controllerconfig/machine-config-controller -o yaml | yq -y '.status.controllerCertificates' - bundleFile: KubeAPIServerServingCAData notAfter: '2034-10-23T13:13:02Z' notBefore: '2024-10-25T13:13:02Z' signer: CN=admin-kubeconfig-signer,OU=openshift subject: CN=admin-kubeconfig-signer,OU=openshift - bundleFile: KubeAPIServerServingCAData notAfter: '2024-10-26T13:13:05Z' notBefore: '2024-10-25T13:27:14Z' signer: CN=kubelet-signer,OU=openshift subject: CN=kube-csr-signer_@1729862835 - bundleFile: KubeAPIServerServingCAData notAfter: '2024-10-26T13:13:05Z' notBefore: '2024-10-25T13:13:05Z' signer: CN=kubelet-signer,OU=openshift subject: CN=kubelet-signer,OU=openshift oc get mcp master -o yaml | yq -y '.status.certExpirys' - bundle: KubeAPIServerServingCAData expiry: '2034-10-23T13:13:02Z' subject: CN=admin-kubeconfig-signer,OU=openshift - bundle: KubeAPIServerServingCAData expiry: '2024-10-26T13:13:05Z' subject: CN=kube-csr-signer_@1729862835 - bundle: KubeAPIServerServingCAData expiry: '2024-10-26T13:13:05Z' subject: CN=kubelet-signer,OU=openshift - bundle: KubeAPIServerServingCAData expiry: '2025-10-25T13:13:05Z' subject: CN=kube-apiserver-to-kubelet-signer,OU=openshift oc debug node/<node_name> sh-5.1# chroot /host sh-5.1# ls /etc/docker/certs.d image-registry.openshift-image-registry.svc.cluster.local:5000 image-registry.openshift-image-registry.svc:5000 variant: openshift version: 4.17.0 metadata: name: 99-worker-chrony 1 labels: machineconfiguration.openshift.io/role: worker 2 storage: files: - path: /etc/chrony.conf mode: 0644 3 overwrite: true contents: inline: | pool 0.rhel.pool.ntp.org iburst 4 driftfile /var/lib/chrony/drift makestep 1.0 3 rtcsync logdir /var/log/chrony butane 99-worker-chrony.bu -o 99-worker-chrony.yaml oc apply -f ./99-worker-chrony.yaml apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: <node_role> 1 name: disable-chronyd spec: config: ignition: version: 3.2.0 systemd: units: - contents: | [Unit] Description=NTP client/server Documentation=man:chronyd(8) man:chrony.conf(5) After=ntpdate.service sntp.service ntpd.service Conflicts=ntpd.service systemd-timesyncd.service ConditionCapability=CAP_SYS_TIME [Service] Type=forking PIDFile=/run/chrony/chronyd.pid EnvironmentFile=-/etc/sysconfig/chronyd ExecStart=/usr/sbin/chronyd USDOPTIONS ExecStartPost=/usr/libexec/chrony-helper update-daemon PrivateTmp=yes ProtectHome=yes ProtectSystem=full [Install] WantedBy=multi-user.target enabled: false name: "chronyd.service" - name: "kubelet-dependencies.target" contents: | [Unit] Description=Dependencies necessary to run kubelet Documentation=https://github.com/openshift/machine-config-operator/ Requires=basic.target network-online.target Wants=NetworkManager-wait-online.service crio-wipe.service Wants=rpc-statd.service oc create -f disable-chronyd.yaml oc get MachineConfig NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 00-worker 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-ssh 3.2.0 40m 99-worker-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-worker-ssh 3.2.0 40m rendered-master-23e785de7587df95a4b517e0647e5ab7 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m rendered-worker-5d596d9293ca3ea80c896a1191735bb1 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker 1 name: 05-worker-kernelarg-selinuxpermissive 2 spec: kernelArguments: - enforcing=0 3 oc create -f 05-worker-kernelarg-selinuxpermissive.yaml oc get MachineConfig NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 00-worker 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 05-worker-kernelarg-selinuxpermissive 3.2.0 105s 99-master-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-ssh 3.2.0 40m 99-worker-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-worker-ssh 3.2.0 40m rendered-master-23e785de7587df95a4b517e0647e5ab7 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m rendered-worker-5d596d9293ca3ea80c896a1191735bb1 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m oc get nodes NAME STATUS ROLES AGE VERSION ip-10-0-136-161.ec2.internal Ready worker 28m v1.30.3 ip-10-0-136-243.ec2.internal Ready master 34m v1.30.3 ip-10-0-141-105.ec2.internal Ready,SchedulingDisabled worker 28m v1.30.3 ip-10-0-142-249.ec2.internal Ready master 34m v1.30.3 ip-10-0-153-11.ec2.internal Ready worker 28m v1.30.3 ip-10-0-153-150.ec2.internal Ready master 34m v1.30.3 oc debug node/ip-10-0-141-105.ec2.internal Starting pod/ip-10-0-141-105ec2internal-debug To use host binaries, run `chroot /host` sh-4.2# cat /host/proc/cmdline BOOT_IMAGE=/ostree/rhcos-... console=tty0 console=ttyS0,115200n8 rootflags=defaults,prjquota rw root=UUID=fd0... ostree=/ostree/boot.0/rhcos/16 coreos.oem.id=qemu coreos.oem.id=ec2 ignition.platform.id=ec2 enforcing=0 sh-4.2# exit apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: "master" name: 99-master-kargs-mpath spec: kernelArguments: - 'rd.multipath=default' - 'root=/dev/disk/by-label/dm-mpath-root' apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: "worker" name: 99-worker-kargs-mpath spec: kernelArguments: - 'rd.multipath=default' - 'root=/dev/disk/by-label/dm-mpath-root' oc create -f ./99-worker-kargs-mpath.yaml oc get MachineConfig NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 00-worker 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-ssh 3.2.0 40m 99-worker-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-worker-kargs-mpath 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 105s 99-worker-ssh 3.2.0 40m rendered-master-23e785de7587df95a4b517e0647e5ab7 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m rendered-worker-5d596d9293ca3ea80c896a1191735bb1 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m oc get nodes NAME STATUS ROLES AGE VERSION ip-10-0-136-161.ec2.internal Ready worker 28m v1.30.3 ip-10-0-136-243.ec2.internal Ready master 34m v1.30.3 ip-10-0-141-105.ec2.internal Ready,SchedulingDisabled worker 28m v1.30.3 ip-10-0-142-249.ec2.internal Ready master 34m v1.30.3 ip-10-0-153-11.ec2.internal Ready worker 28m v1.30.3 ip-10-0-153-150.ec2.internal Ready master 34m v1.30.3 oc debug node/ip-10-0-141-105.ec2.internal Starting pod/ip-10-0-141-105ec2internal-debug To use host binaries, run `chroot /host` sh-4.2# cat /host/proc/cmdline rd.multipath=default root=/dev/disk/by-label/dm-mpath-root sh-4.2# exit cat << EOF > 99-worker-realtime.yaml apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: "worker" name: 99-worker-realtime spec: kernelType: realtime EOF oc create -f 99-worker-realtime.yaml oc get nodes NAME STATUS ROLES AGE VERSION ip-10-0-143-147.us-east-2.compute.internal Ready worker 103m v1.30.3 ip-10-0-146-92.us-east-2.compute.internal Ready worker 101m v1.30.3 ip-10-0-169-2.us-east-2.compute.internal Ready worker 102m v1.30.3 oc debug node/ip-10-0-143-147.us-east-2.compute.internal Starting pod/ip-10-0-143-147us-east-2computeinternal-debug To use host binaries, run `chroot /host` sh-4.4# uname -a Linux <worker_node> 4.18.0-147.3.1.rt24.96.el8_1.x86_64 #1 SMP PREEMPT RT Wed Nov 27 18:29:55 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux oc delete -f 99-worker-realtime.yaml variant: openshift version: 4.17.0 metadata: name: 40-worker-custom-journald labels: machineconfiguration.openshift.io/role: worker storage: files: - path: /etc/systemd/journald.conf mode: 0644 overwrite: true contents: inline: | # Disable rate limiting RateLimitInterval=1s RateLimitBurst=10000 Storage=volatile Compress=no MaxRetentionSec=30s butane 40-worker-custom-journald.bu -o 40-worker-custom-journald.yaml oc apply -f 40-worker-custom-journald.yaml oc get machineconfigpool NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-35 True False False 3 3 3 0 34m worker rendered-worker-d8 False True False 3 1 1 0 34m oc get node | grep worker ip-10-0-0-1.us-east-2.compute.internal Ready worker 39m v0.0.0-master+USDFormat:%hUSD oc debug node/ip-10-0-0-1.us-east-2.compute.internal Starting pod/ip-10-0-141-142us-east-2computeinternal-debug sh-4.2# chroot /host sh-4.4# cat /etc/systemd/journald.conf Disable rate limiting RateLimitInterval=1s RateLimitBurst=10000 Storage=volatile Compress=no MaxRetentionSec=30s sh-4.4# exit cat << EOF > 80-extensions.yaml apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker name: 80-worker-extensions spec: config: ignition: version: 3.2.0 extensions: - usbguard EOF oc create -f 80-extensions.yaml oc get machineconfig 80-worker-extensions NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 80-worker-extensions 3.2.0 57s oc get machineconfigpool NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-35 True False False 3 3 3 0 34m worker rendered-worker-d8 False True False 3 1 1 0 34m oc get node | grep worker NAME STATUS ROLES AGE VERSION ip-10-0-169-2.us-east-2.compute.internal Ready worker 102m v1.30.3 oc debug node/ip-10-0-169-2.us-east-2.compute.internal To use host binaries, run `chroot /host` sh-4.4# chroot /host sh-4.4# rpm -q usbguard usbguard-0.7.4-4.el8.x86_64.rpm variant: openshift version: 4.17.0 metadata: labels: machineconfiguration.openshift.io/role: worker name: 98-worker-firmware-blob storage: files: - path: /var/lib/firmware/<package_name> 1 contents: local: <package_name> 2 mode: 0644 3 openshift: kernel_arguments: - 'firmware_class.path=/var/lib/firmware' 4 butane 98-worker-firmware-blob.bu -o 98-worker-firmware-blob.yaml --files-dir <directory_including_package_name> oc apply -f 98-worker-firmware-blob.yaml mkpasswd -m SHA-512 testpass USD6USDCBZwA6s6AVFOtiZeUSDaUKDWpthhJEyR3nnhM02NM1sKCpHn9XN.NPrJNQ3HYewioaorpwL3mKGLxvW0AOb4pJxqoqP4nFX77y0p00.8. apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker name: set-core-user-password spec: config: ignition: version: 3.2.0 passwd: users: - name: core 1 passwordHash: <password> 2 oc create -f <file-name>.yaml NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-d686a3ffc8fdec47280afec446fce8dd True False False 3 3 3 0 64m worker rendered-worker-4605605a5b1f9de1d061e9d350f251e5 False True False 3 0 0 0 64m oc debug node/<node_name> sh-4.4# chroot /host core:USD6USD2sE/010goDuRSxxvUSDo18K52wor.wIwZp:19418:0:99999:7::: apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: logLevel: Normal managementState: Managed operatorLogLevel: Normal status: nodeDisruptionPolicyStatus: clusterPolicies: files: - actions: - type: None path: /etc/mco/internal-registry-pull-secret.json - actions: - type: None path: /var/lib/kubelet/config.json - actions: - reload: serviceName: crio.service type: Reload path: /etc/machine-config-daemon/no-reboot/containers-gpg.pub - actions: - reload: serviceName: crio.service type: Reload path: /etc/containers/policy.json - actions: - type: Special path: /etc/containers/registries.conf - actions: - reload: serviceName: crio.service type: Reload path: /etc/containers/registries.d - actions: - type: None path: /etc/nmstate/openshift - actions: - restart: serviceName: coreos-update-ca-trust.service type: Restart - restart: serviceName: crio.service type: Restart path: /etc/pki/ca-trust/source/anchors/openshift-config-user-ca-bundle.crt sshkey: actions: - type: None observedGeneration: 9 apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: sshkey: actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: files: - actions: - restart: serviceName: chronyd.service type: Restart path: /etc/chrony.conf - actions: - type: None path: /var/run apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: units: - name: auditd.service actions: - type: Drain - type: Reload reload: serviceName: crio.service - type: DaemonReload - type: Restart restart: serviceName: crio.service apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: files: - actions: - type: None path: /etc/containers/registries.conf oc edit MachineConfiguration cluster -n openshift-machine-config-operator apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: 1 files: 2 - actions: 3 - restart: 4 serviceName: chronyd.service 5 type: Restart path: /etc/chrony.conf 6 sshkey: 7 actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart units: 8 - actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart name: test.service oc get MachineConfiguration/cluster -o yaml apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: labels: machineconfiguration.openshift.io/role: worker name: cluster status: nodeDisruptionPolicyStatus: 1 clusterPolicies: files: - actions: - restart: serviceName: chronyd.service type: Restart path: /etc/chrony.conf sshkey: actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart units: - actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart name: test.se apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: name: infra spec: machineConfigSelector: matchExpressions: - {key: machineconfiguration.openshift.io/role, operator: In, values: [worker,infra]} oc get kubeletconfig NAME AGE set-kubelet-config 15m oc get mc | grep kubelet 99-worker-generated-kubelet-1 b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 26m oc describe machineconfigpool <name> oc describe machineconfigpool worker apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: creationTimestamp: 2019-02-08T14:52:39Z generation: 1 labels: custom-kubelet: set-kubelet-config 1 oc label machineconfigpool worker custom-kubelet=set-kubelet-config oc get machineconfig oc describe node <node_name> oc describe node ci-ln-5grqprb-f76d1-ncnqq-worker-a-mdv94 Allocatable: attachable-volumes-aws-ebs: 25 cpu: 3500m hugepages-1Gi: 0 hugepages-2Mi: 0 memory: 15341844Ki pods: 250 apiVersion: machineconfiguration.openshift.io/v1 kind: KubeletConfig metadata: name: set-kubelet-config spec: machineConfigPoolSelector: matchLabels: custom-kubelet: set-kubelet-config 1 kubeletConfig: 2 podPidsLimit: 8192 containerLogMaxSize: 50Mi maxPods: 500 apiVersion: machineconfiguration.openshift.io/v1 kind: KubeletConfig metadata: name: set-kubelet-config spec: machineConfigPoolSelector: matchLabels: custom-kubelet: set-kubelet-config kubeletConfig: maxPods: <pod_count> kubeAPIBurst: <burst_rate> kubeAPIQPS: <QPS> oc label machineconfigpool worker custom-kubelet=set-kubelet-config oc create -f change-maxPods-cr.yaml oc get kubeletconfig NAME AGE set-kubelet-config 15m oc describe node <node_name> Allocatable: attachable-volumes-gce-pd: 127 cpu: 3500m ephemeral-storage: 123201474766 hugepages-1Gi: 0 hugepages-2Mi: 0 memory: 14225400Ki pods: 500 1 oc get kubeletconfigs set-kubelet-config -o yaml spec: kubeletConfig: containerLogMaxSize: 50Mi maxPods: 500 podPidsLimit: 8192 machineConfigPoolSelector: matchLabels: custom-kubelet: set-kubelet-config status: conditions: - lastTransitionTime: "2021-06-30T17:04:07Z" message: Success status: "True" type: Success oc get ctrcfg NAME AGE ctr-overlay 15m ctr-level 5m45s oc get mc | grep container 01-master-container-runtime b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 57m 01-worker-container-runtime b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 57m 99-worker-generated-containerruntime b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 26m 99-worker-generated-containerruntime-1 b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 17m 99-worker-generated-containerruntime-2 b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 7m26s apiVersion: machineconfiguration.openshift.io/v1 kind: ContainerRuntimeConfig metadata: name: overlay-size spec: machineConfigPoolSelector: matchLabels: pools.operator.machineconfiguration.openshift.io/worker: '' 1 containerRuntimeConfig: logLevel: debug 2 overlaySize: 8G 3 defaultRuntime: "crun" 4 apiVersion: machineconfiguration.openshift.io/v1 kind: ContainerRuntimeConfig metadata: name: overlay-size spec: machineConfigPoolSelector: matchLabels: pools.operator.machineconfiguration.openshift.io/worker: '' 1 containerRuntimeConfig: 2 logLevel: debug overlaySize: 8G oc create -f <file_name>.yaml oc get ContainerRuntimeConfig NAME AGE overlay-size 3m19s oc get machineconfigs | grep containerrun 99-worker-generated-containerruntime 2c9371fbb673b97a6fe8b1c52691999ed3a1bfc2 3.2.0 31s oc get mcp worker NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE worker rendered-worker-169 False True False 3 1 1 0 9h oc debug node/<node_name> sh-4.4# chroot /host sh-4.4# crio config | grep 'log_level' log_level = "debug" sh-4.4# head -n 7 /etc/containers/storage.conf [storage] driver = "overlay" runroot = "/var/run/containers/storage" graphroot = "/var/lib/containers/storage" [storage.options] additionalimagestores = [] size = "8G" apiVersion: machineconfiguration.openshift.io/v1 kind: ContainerRuntimeConfig metadata: name: overlay-size spec: machineConfigPoolSelector: matchLabels: custom-crio: overlay-size containerRuntimeConfig: logLevel: debug overlaySize: 8G oc apply -f overlaysize.yml oc edit machineconfigpool worker apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: creationTimestamp: "2020-07-09T15:46:34Z" generation: 3 labels: custom-crio: overlay-size machineconfiguration.openshift.io/mco-built-in: "" oc get machineconfigs 99-worker-generated-containerruntime 4173030d89fbf4a7a0976d1665491a4d9a6e54f1 3.2.0 7m42s rendered-worker-xyz 4173030d89fbf4a7a0976d1665491a4d9a6e54f1 3.2.0 7m36s oc get mcp worker NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE worker rendered-worker-xyz False True False 3 2 2 0 20h NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE worker rendered-worker-xyz True False False 3 3 3 0 20h head -n 7 /etc/containers/storage.conf [storage] driver = "overlay" runroot = "/var/run/containers/storage" graphroot = "/var/lib/containers/storage" [storage.options] additionalimagestores = [] size = "8G" ~ USD df -h Filesystem Size Used Available Use% Mounted on overlay 8.0G 8.0K 8.0G 0% / oc get ctrcfg NAME AGE ctr-overlay 15m ctr-level 5m45s cat /proc/1/status | grep Cap capsh --decode=<decode_CapBnd_value> 1 apiVersion: machine.openshift.io/v1beta1 kind: MachineSet metadata: name: ci-ln-hmy310k-72292-5f87z-worker-a namespace: openshift-machine-api spec: template: spec: providerSpec: value: disks: - autoDelete: true boot: true image: projects/rhcos-cloud/global/images/rhcos-412-85-202203181601-0-gcp-x86-64 1 oc edit MachineConfiguration cluster apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster namespace: openshift-machine-config-operator spec: managedBootImages: 1 machineManagers: - resource: machinesets apiGroup: machine.openshift.io selection: mode: All 2 apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster namespace: openshift-machine-config-operator spec: managedBootImages: 1 machineManagers: - resource: machinesets apiGroup: machine.openshift.io selection: mode: Partial partial: machineResourceSelector: matchLabels: update-boot-image: "true" 2 oc label machineset.machine ci-ln-hmy310k-72292-5f87z-worker-a update-boot-image=true -n openshift-machine-api oc get machineconfiguration cluster -n openshift-machine-api -o yaml kind: MachineConfiguration metadata: name: cluster status: conditions: - lastTransitionTime: "2024-09-09T13:51:37Z" 1 message: Reconciled 1 of 2 MAPI MachineSets | Reconciled 0 of 0 CAPI MachineSets | Reconciled 0 of 0 CAPI MachineDeployments reason: BootImageUpdateConfigurationAdded status: "True" type: BootImageUpdateProgressing - lastTransitionTime: "2024-09-09T13:51:37Z" 2 message: 0 Degraded MAPI MachineSets | 0 Degraded CAPI MachineSets | 0 CAPI MachineDeployments reason: BootImageUpdateConfigurationAdded status: "False" type: BootImageUpdateDegraded oc get machinesets <machineset_name> -n openshift-machine-api -o yaml apiVersion: machine.openshift.io/v1beta1 kind: MachineSet metadata: labels: machine.openshift.io/cluster-api-cluster: ci-ln-77hmkpt-72292-d4pxp update-boot-image: "true" name: ci-ln-77hmkpt-72292-d4pxp-worker-a namespace: openshift-machine-api spec: template: spec: providerSpec: value: disks: - autoDelete: true boot: true image: projects/rhcos-cloud/global/images/rhcos-416-92-202402201450-0-gcp-x86-64 1 oc edit MachineConfiguration cluster apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster namespace: openshift-machine-config-operator spec: managedBootImages: 1 machineManagers: - resource: machinesets apiGroup: machine.openshift.io selection: mode: All oc adm prune renderedmachineconfigs list --in-use=false --pool-name=worker worker rendered-worker-f38bf61ced3c920cf5a29a200ed43243 -- 2025-01-21 13:45:01 +0000 UTC (Currently in use: false) rendered-worker-fc94397dc7c43808c7014683c208956e-- 2025-01-30 17:20:53 +0000 UTC (Currently in use: false) rendered-worker-708c652868f7597eaa1e2622edc366ef -- 2025-01-31 18:01:16 +0000 UTC (Currently in use: true) oc adm prune renderedmachineconfigs --pool-name=worker Dry run enabled - no modifications will be made. Add --confirm to remove rendered machine configs. dry-run deleting rendered MachineConfig rendered-worker-f38bf61ced3c920cf5a29a200ed43243 dry-run deleting MachineConfig rendered-worker-fc94397dc7c43808c7014683c208956e Skip dry-run deleting rendered MachineConfig rendered-worker-708c652868f7597eaa1e2622edc366ef as it's currently in use oc adm prune renderedmachineconfigs --pool-name=worker Dry run enabled - no modifications will be made. Add --confirm to remove rendered machine configs. dry-run deleting rendered MachineConfig rendered-worker-f38bf61ced3c920cf5a29a200ed43243 dry-run deleting MachineConfig rendered-worker-fc94397dc7c43808c7014683c208956e Skip dry-run deleting rendered MachineConfig rendered-worker-708c652868f7597eaa1e2622edc366ef as it's currently in use oc adm prune renderedmachineconfigs --pool-name=worker --count=2 --confirm deleting rendered MachineConfig rendered-worker-f38bf61ced3c920cf5a29a200ed43243 deleting rendered MachineConfig rendered-worker-fc94397dc7c43808c7014683c208956e Skip deleting rendered MachineConfig rendered-worker-708c652868f7597eaa1e2622edc366ef as it's currently in use Using a 4.17.0 image containerfileArch: noarch content: |- FROM configs AS final #Install hotfix rpm RUN dnf install -y https://example.com/myrepo/haproxy-1.0.16-5.el8.src.rpm && dnf clean all && ostree container commit Using a 4.17.0 image FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256 #Install hotfix rpm RUN dnf install -y https://example.com/myrepo/haproxy-1.0.16-5.el8.src.rpm && dnf clean all && ostree container commit Get RHCOS base image of target cluster `oc adm release info --image-for rhel-coreos` hadolint ignore=DL3006 FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256 Install our config file COPY my-host-to-host.conf /etc/ipsec.d/ RHEL entitled host is needed here to access RHEL packages Install libreswan as extra RHEL package RUN dnf install -y libreswan && dnf clean all && systemctl enable ipsec && ostree container commit FROM configs AS final #Enable EPEL (more info at https://docs.fedoraproject.org/en-US/epel/ ) and install htop RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && dnf install -y htop && dnf clean all && ostree container commit Get RHCOS base image of target cluster `oc adm release info --image-for rhel-coreos` hadolint ignore=DL3006 FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256 #Enable EPEL (more info at https://docs.fedoraproject.org/en-US/epel/ ) and install htop RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && dnf install -y htop && dnf clean all && ostree container commit FROM configs AS final RHEL entitled host is needed here to access RHEL packages Install fish as third party package from EPEL RUN dnf install -y https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/Packages/f/fish-3.3.1-3.el9.x86_64.rpm && dnf clean all && ostree container commit Get RHCOS base image of target cluster `oc adm release info --image-for rhel-coreos` hadolint ignore=DL3006 FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256 RHEL entitled host is needed here to access RHEL packages Install fish as third party package from EPEL RUN dnf install -y https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/Packages/f/fish-3.3.1-3.el9.x86_64.rpm && dnf clean all && ostree container commit apiVersion: machineconfiguration.openshift.io/v1alpha1 kind: MachineOSConfig metadata: name: layered spec: machineConfigPool: name: <mcp_name> 1 buildInputs: containerFile: 2 - containerfileArch: noarch 3 content: |- FROM configs AS final 4 RUN rpm-ostree install tree && ostree container commit imageBuilder: 5 imageBuilderType: PodImageBuilder baseImagePullSecret: 6 name: global-pull-secret-copy renderedImagePushspec: image-registry.openshift-image-registry.svc:5000/openshift/os-image:latest 7 renderedImagePushSecret: 8 name: builder-dockercfg-7lzwl buildOutputs: 9 currentImagePullSecret: name: builder-dockercfg-7lzwl oc create -f <file_name>.yaml oc get machineosbuild NAME PREPARED BUILDING SUCCEEDED INTERRUPTED FAILED layered-rendered-layered-ad5a3cad36303c363cf458ab0524e7c0-builder False False True False False oc label node <node_name> 'node-role.kubernetes.io/<mcp_name>=' oc get pods -n openshift-machine-config-operator NAME READY STATUS RESTARTS AGE build-rendered-layered-ad5a3cad36303c363cf458ab0524e7c0 2/2 Running 0 2m40s 1 machine-os-builder-6fb66cfb99-zcpvq 1/1 Running 0 2m42s 2 oc get machineosbuilds NAME PREPARED BUILDING SUCCEEDED INTERRUPTED FAILED layered-rendered-layered-ef6460613affe503b530047a11b28710-builder False True False False False oc describe machineosbuild <object_name> apiVersion: machineconfiguration.openshift.io/v1alpha1 kind: MachineOSBuild metadata: name: layered-rendered-layered-ad5a3cad36303c363cf458ab0524e7c0-builder spec: desiredConfig: name: rendered-layered-ad5a3cad36303c363cf458ab0524e7c0 machineOSConfig: name: layered renderedImagePushspec: image-registry.openshift-image-registry.svc:5000/openshift-machine-config-operator/os-image:latest status: conditions: - lastTransitionTime: "2024-05-21T20:25:06Z" message: Build Ready reason: Ready status: "True" type: Succeeded finalImagePullspec: image-registry.openshift-image-registry.svc:5000/openshift-machine-config-operator/os-image@sha256:f636fa5b504e92e6faa22ecd71a60b089dab72200f3d130c68dfec07148d11cd 1 oc debug node/<node_name> sh-4.4# chroot /host sh-5.1# rpm-ostree status Deployments: * ostree-unverified-registry:quay.io/openshift-release-dev/os-image@sha256:f636fa5b504e92e6faa22ecd71a60b089dab72200f3d130c68dfec07148d11cd 1 Digest: sha256:bcea2546295b2a55e0a9bf6dd4789433a9867e378661093b6fdee0031ed1e8a4 Version: 416.94.202405141654-0 (2024-05-14T16:58:43Z) Using a 4.17.0 image FROM quay.io/openshift-release/ocp-release@sha256... 1 #Install hotfix rpm RUN rpm-ostree override replace http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/kernel-{,core-,modules-,modules-core-,modules-extra-}5.14.0-295.el9.x86_64.rpm && \ 2 rpm-ostree cleanup -m && ostree container commit apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker 1 name: os-layer-custom spec: osImageURL: quay.io/my-registry/custom-image@sha256... 2 oc create -f <file_name>.yaml oc get mc NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 00-worker 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 01-master-container-runtime 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 01-master-kubelet 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 01-worker-container-runtime 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 01-worker-kubelet 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 99-master-generated-registries 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 99-master-ssh 3.2.0 98m 99-worker-generated-registries 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 99-worker-ssh 3.2.0 98m os-layer-custom 10s 1 rendered-master-15961f1da260f7be141006404d17d39b 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m rendered-worker-5aff604cb1381a4fe07feaf1595a797e 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m rendered-worker-5de4837625b1cbc237de6b22bc0bc873 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 4s 2 oc describe mc rendered-worker-5de4837625b1cbc237de6b22bc0bc873 Name: rendered-worker-5de4837625b1cbc237de6b22bc0bc873 Namespace: Labels: <none> Annotations: machineconfiguration.openshift.io/generated-by-controller-version: 5bdb57489b720096ef912f738b46330a8f577803 machineconfiguration.openshift.io/release-image-version: 4.17.0-ec.3 API Version: machineconfiguration.openshift.io/v1 Kind: MachineConfig Os Image URL: quay.io/my-registry/custom-image@sha256 oc get mcp NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-15961f1da260f7be141006404d17d39b True False False 3 3 3 0 39m worker rendered-worker-5de4837625b1cbc237de6b22bc0bc873 True False False 3 0 0 0 39m 1 oc get nodes NAME STATUS ROLES AGE VERSION ip-10-0-148-79.us-west-1.compute.internal Ready worker 32m v1.30.3 ip-10-0-155-125.us-west-1.compute.internal Ready,SchedulingDisabled worker 35m v1.30.3 ip-10-0-170-47.us-west-1.compute.internal Ready control-plane,master 42m v1.30.3 ip-10-0-174-77.us-west-1.compute.internal Ready control-plane,master 42m v1.30.3 ip-10-0-211-49.us-west-1.compute.internal Ready control-plane,master 42m v1.30.3 ip-10-0-218-151.us-west-1.compute.internal Ready worker 31m v1.30.3 oc debug node/ip-10-0-155-125.us-west-1.compute.internal sh-4.4# chroot /host sh-4.4# sudo rpm-ostree status State: idle Deployments: * ostree-unverified-registry:quay.io/my-registry/ Digest: sha256: oc delete mc os-layer-custom oc get mcp NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-6faecdfa1b25c114a58cf178fbaa45e2 True False False 3 3 3 0 39m worker rendered-worker-6b000dbc31aaee63c6a2d56d04cd4c1b False True False 3 0 0 0 39m 1 oc get nodes NAME STATUS ROLES AGE VERSION ip-10-0-148-79.us-west-1.compute.internal Ready worker 32m v1.30.3 ip-10-0-155-125.us-west-1.compute.internal Ready,SchedulingDisabled worker 35m v1.30.3 ip-10-0-170-47.us-west-1.compute.internal Ready control-plane,master 42m v1.30.3 ip-10-0-174-77.us-west-1.compute.internal Ready control-plane,master 42m v1.30.3 ip-10-0-211-49.us-west-1.compute.internal Ready control-plane,master 42m v1.30.3 ip-10-0-218-151.us-west-1.compute.internal Ready worker 31m v1.30.3 oc debug node/ip-10-0-155-125.us-west-1.compute.internal sh-4.4# chroot /host sh-4.4# sudo rpm-ostree status State: idle Deployments: * ostree-unverified-registry:podman pull quay.io/openshift-release-dev/ocp-release@sha256:e2044c3cfebe0ff3a99fc207ac5efe6e07878ad59fd4ad5e41f88cb016dacd73 Digest: sha256:e2044c3cfebe0ff3a99fc207ac5efe6e07878ad59fd4ad5e41f88cb016dacd73