Managing storage devices
Managing storage devices Red Hat Enterprise Linux 9 Configuring and managing local and remote storage devices Red Hat Customer Content Services
parted block-device (parted) print (parted) mklabel table-type (parted) mklabel gpt (parted) print (parted) quit parted /dev/sda (parted) print Model: ATA SAMSUNG MZNLN256 (scsi) Disk /dev/sda: 256GB Sector size (logical/physical): 512B/512B Partition Table: msdos Disk Flags: Number Start End Size Type File system Flags 1 1049kB 269MB 268MB primary xfs boot 2 269MB 34.6GB 34.4GB primary 3 34.6GB 45.4GB 10.7GB primary 4 45.4GB 256GB 211GB extended 5 45.4GB 256GB 211GB logical (parted) select block-device parted block-device (parted) print (parted) mkpart part-type name fs-type start end (parted) mkpart primary 1024MiB 2048MiB (parted) print (parted) quit udevadm settle cat /proc/partitions fdisk block-device Command (m for help): print Command (m for help): type Partition number ( 1,2,3 default 3 ): 2 Hex code (type L to list all codes): L Hex code (type L to list all codes): 8e Command (m for help): write The partition table has been altered. Syncing disks. fdisk --list block-device parted block-device (parted) print (parted) resizepart 1 2GiB (parted) print (parted) quit cat /proc/partitions parted block-device (parted) print (parted) rm minor-number (parted) print (parted) quit cat /proc/partitions systemctl daemon-reload grubby --info=ALL grubby --update-kernel=ALL --remove-args=" option " dracut --force --verbose dnf install targetcli systemctl start target systemctl enable target firewall-cmd --permanent --add-port=3260/tcp Success firewall-cmd --reload Success targetcli /> ls o- /........................................[...] o- backstores.............................[...] | o- block.................[Storage Objects: 0] | o- fileio................[Storage Objects: 0] | o- pscsi.................[Storage Objects: 0] | o- ramdisk...............[Storage Objects: 0] o- iscsi...........................[Targets: 0] o- loopback........................[Targets: 0] /> iscsi/ /iscsi> create Created target iqn.2003-01.org.linux-iscsi.hostname.x8664:sn.78b473f296ff Created TPG1 /iscsi> create iqn.2006-04.com.example:444 Created target iqn.2006-04.com.example:444 Created TPG1 Here iqn.2006-04.com.example:444 is target_iqn_name /iscsi> ls o- iscsi.......................................[1 Target] o- iqn.2006-04.com.example:444................[1 TPG] o- tpg1...........................[enabled, auth] o- acls...............................[0 ACL] o- luns...............................[0 LUN] o- portals.........................[0 Portal] /> backstores/fileio /backstores/fileio> create file1 /tmp/disk1 .img 200M write_back=false Created fileio file1 with size 209715200 /backstores/fileio> ls /> backstores/block/ /backstores/block> create name= block_backend dev=/dev/ sdb Generating a wwn serial. Created block storage object block_backend using /dev/vdb. /backstores/block> ls /> backstores/pscsi/ /backstores/pscsi> create name= pscsi_backend dev=/dev/sr0 Generating a wwn serial. Created pscsi storage object pscsi_backend using /dev/sr0 /backstores/pscsi> ls /> backstores/ramdisk/ /backstores/ramdisk> create name= rd_backend size= 1GB Generating a wwn serial. Created rd_mcp ramdisk rd_backend with size 1GB. /backstores/ramdisk> ls /iscsi> iqn.2006-04.example:444/tpg1/ /iscsi/iqn.20...mple:444/tpg1> portals/ create Using default IP port 3260 Binding to INADDR_Any (0.0.0.0) Created network portal 0.0.0.0:3260 /iscsi/iqn.20...mple:444/tpg1> portals/ create 192.168.122.137 Using default IP port 3260 Created network portal 192.168.122.137:3260 /iscsi/iqn.20...mple:444/tpg1> ls o- tpg.................................. [enambled, auth] o- acls ......................................[0 ACL] o- luns ......................................[0 LUN] o- portals ................................[1 Portal] o- 192.168.122.137:3260......................[OK] /iscsi/iqn.20...mple:444/tpg1> luns/ create /backstores/ramdisk/rd_backend Created LUN 0. /iscsi/iqn.20...mple:444/tpg1> luns/ create /backstores/block/block_backend Created LUN 1. /iscsi/iqn.20...mple:444/tpg1> luns/ create /backstores/fileio/file1 Created LUN 2. /iscsi/iqn.20...mple:444/tpg1> ls o- tpg.................................. [enambled, auth] o- acls ......................................[0 ACL] o- luns .....................................[3 LUNs] | o- lun0.........................[ramdisk/ramdisk1] | o- lun1.................[block/block1 (/dev/vdb1)] | o- lun2...................[fileio/file1 (/foo.img)] o- portals ................................[1 Portal] o- 192.168.122.137:3260......................[OK] /> set global auto_add_mapped_luns=false Parameter auto_add_mapped_luns is now 'false'. /> iscsi/target_iqn_name/tpg1/acls/ initiator_iqn_name / /iscsi/target_iqn_name/tpg1/acls/ initiator_iqn_name > create mapped_lun=next_sequential_LUN_number tpg_lun_or_backstore=backstore write_protect=1 /iscsi/target_iqn_name/tpg1/acls/2006-04.com.example:888> create mapped_lun=1 tpg_lun_or_backstore=/backstores/block/block2 write_protect=1 Created LUN 1. Created Mapped LUN 1. /iscsi/target_iqn_name/tpg1/acls/2006-04.com.example:888> ls o- 2006-04.com.example:888 .. [Mapped LUNs: 2] | o- mapped_lun0 .............. [lun0 block/disk1 (rw)] | o- mapped_lun1 .............. [lun1 block/disk2 (ro)] /> iscsi/ target_iqn_name / tpg_name /acls/ iscsi/ target_iqn_name / tpg_name /acls> create initiator_iqn_name Created Node ACL for initiator_iqn_name Created mapped LUN 2. Created mapped LUN 1. Created mapped LUN 0. iscsi/ target_iqn_name / tpg_name /acls> create custom_name Created Node ACL for custom_name Created mapped LUN 2. Created mapped LUN 1. Created mapped LUN 0. iscsi/ target_iqn_name / tpg_name /acls> ls o- acls .................................................[1 ACL] o- target_iqn_name ....[3 Mapped LUNs, auth] o- mapped_lun0 .............[lun0 ramdisk/ramdisk1 (rw)] o- mapped_lun1 .................[lun1 block/block1 (rw)] o- mapped_lun2 .................[lun2 fileio/file1 (rw)] /iscsi/iqn.20...mple:444/tpg1> set attribute authentication=1 Parameter authentication is now '1'. /tpg1> set auth userid= redhat Parameter userid is now ' redhat '. /iscsi/iqn.20...689dcbb3/tpg1> set auth password= redhat_passwd Parameter password is now ' redhat_passwd '. iscsiadm -m node -T iqn.2006-04.example:444 -u /> iscsi/ delete iqn.2006-04.com.example:444 /> backstores/ backstore-type / delete block_backend /> /iscsi/iqn-name/tpg/acls/ delete iqn.2006-04.com.example:444 /> iscsi/ ls dnf install iscsi-initiator-utils cat /etc/iscsi/initiatorname.iscsi InitiatorName=iqn.2006-04.com.example:888 vi /etc/iscsi/initiatorname.iscsi InitiatorName= custom-name systemctl restart iscsid iscsiadm -m discovery -t st -p 10.64.24.179 10.64.24.179:3260,1 iqn.2006-04.example:444 iscsiadm -m node -T iqn.2006-04.example:444 -l Logging in to [iface: default, target: iqn.2006-04.example:444, portal: 10.64.24.179,3260] (multiple) Login to [iface: default, target: iqn.2006-04.example:444, portal: 10.64.24.179,3260] successful. grep "Attached SCSI" /var/log/messages mkfs.ext4 /dev/ disk_name mkdir /mount/point mount /dev/ disk_name /mount/point vi /etc/fstab /dev/ disk_name /mount/point ext4 _netdev 0 0 vi /etc/iscsi/iscsid.conf node.session.auth.authmethod = CHAP node.session.auth.username = redhat node.session.auth.password = redhat_passwd systemctl start iscsid.service dnf install iscsi-initiator-utils iscsiadm -m session -P 3 iscsiadm -m session -P 0 or iscsiadm -m session tcp [2] 10.15.84.19:3260,2 iqn.1992-08.com.netapp:sn.33615311 tcp [3] 10.15.85.19:3260,3 iqn.1992-08.com.netapp:sn.33615311 multipath -ll echo 1 > /sys/block/ sdX /device/rescan cat /sys/class/fc_remote_ports/rport- host : bus : remote-port /port_state modprobe nvmet-rdma mkdir /sys/kernel/config/nvmet/subsystems/ testnqn cd /sys/kernel/config/nvmet/subsystems/ testnqn echo 1 > attr_allow_any_host mkdir namespaces/ 10 cd namespaces/ 10 echo -n /dev/ nvme0n1 > device_path echo 1 > enable mkdir /sys/kernel/config/nvmet/ports/ 1 cd /sys/kernel/config/nvmet/ports/ 1 ip addr show mlx5_ib0 8: mlx5_ib0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 4092 qdisc mq state UP group default qlen 256 link/infiniband 00:00:06:2f:fe:80:00:00:00:00:00:00:e4:1d:2d:03:00:e7:0f:f6 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff inet 172.31.0.202/24 brd 172.31.0.255 scope global noprefixroute mlx5_ib0 valid_lft forever preferred_lft forever inet6 fe80::e61d:2d03:e7:ff6/64 scope link noprefixroute valid_lft forever preferred_lft forever echo -n 172.31.0.202 > addr_traddr echo rdma > addr_trtype echo 4420 > addr_trsvcid echo ipv4 > addr_adrfam ln -s /sys/kernel/config/nvmet/subsystems/ testnqn /sys/kernel/config/nvmet/ports/1/subsystems/ testnqn dmesg | grep "enabling port" [ 1091.413648] nvmet_rdma: enabling port 1 (172.31.0.202:4420) dnf install nvmetcli wget http://git.infradead.org/users/hch/nvmetcli.git/blob_plain/0a6b088db2dc2e5de11e6f23f1e890e4b54fee64:/rdma.json nvmetcli restore rdma.json dmesg | tail -1 [ 4797.132647] nvmet_rdma: enabling port 2 (172.31.0.202:4420) nvmetcli clear dnf install nvme-cli modprobe nvme-rdma nvme discover -t rdma -a 172.31.0.202 -s 4420 Discovery Log Number of Records 1, Generation counter 2 =====Discovery Log Entry 0====== trtype: rdma adrfam: ipv4 subtype: nvme subsystem treq: not specified, sq flow control disable supported portid: 1 trsvcid: 4420 subnqn: testnqn traddr: 172.31.0.202 rdma_prtype: not specified rdma_qptype: connected rdma_cms: rdma-cm rdma_pkey: 0x0000 nvme connect -t rdma -n testnqn -a 172.31.0.202 -s 4420 lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sda 8:0 0 465.8G 0 disk ├─sda1 8:1 0 1G 0 part /boot └─sda2 8:2 0 464.8G 0 part ├─rhel_rdma--virt--03-root 253:0 0 50G 0 lvm / ├─rhel_rdma--virt--03-swap 253:1 0 4G 0 lvm [SWAP] └─rhel_rdma--virt--03-home 253:2 0 410.8G 0 lvm /home nvme0n1 cat /sys/class/nvme/nvme0/transport rdma nvme list nvme disconnect -n testnqn NQN:testnqn disconnected 1 controller(s) lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sda 8:0 0 465.8G 0 disk ├─sda1 8:1 0 1G 0 part /boot └─sda2 8:2 0 464.8G 0 part ├─rhel_rdma--virt--03-root 253:0 0 50G 0 lvm / ├─rhel_rdma--virt--03-swap 253:1 0 4G 0 lvm [SWAP] └─rhel_rdma--virt--03-home 253:2 0 410.8G 0 lvm /home dnf install nvme-cli cat /sys/class/scsi_host/host*/nvme_info NVME Host Enabled XRI Dist lpfc0 Total 6144 IO 5894 ELS 250 NVME LPORT lpfc0 WWPN x10000090fae0b5f5 WWNN x20000090fae0b5f5 DID x010f00 ONLINE NVME RPORT WWPN x204700a098cbcac6 WWNN x204600a098cbcac6 DID x01050e TARGET DISCSRVC ONLINE NVME Statistics LS: Xmt 000000000e Cmpl 000000000e Abort 00000000 LS XMIT: Err 00000000 CMPL: xb 00000000 Err 00000000 Total FCP Cmpl 00000000000008ea Issue 00000000000008ec OutIO 0000000000000002 abort 00000000 noxri 00000000 nondlp 00000000 qdepth 00000000 wqerr 00000000 err 00000000 FCP CMPL: xb 00000000 Err 00000000 nvme discover --transport fc \ --traddr nn-0x204600a098cbcac6:pn-0x204700a098cbcac6 \ --host-traddr nn-0x20000090fae0b5f5:pn-0x10000090fae0b5f5 Discovery Log Number of Records 2, Generation counter 49530 =====Discovery Log Entry 0====== trtype: fc adrfam: fibre-channel subtype: nvme subsystem treq: not specified portid: 0 trsvcid: none subnqn: nqn.1992-08.com.netapp:sn.e18bfca87d5e11e98c0800a098cbcac6:subsystem.st14_nvme_ss_1_1 traddr: nn-0x204600a098cbcac6:pn-0x204700a098cbcac6 nvme connect --transport fc \ --traddr nn-0x204600a098cbcac6:pn-0x204700a098cbcac6 \ --host-traddr nn-0x20000090fae0b5f5:pn-0x10000090fae0b5f5 \ -n nqn.1992-08.com.netapp:sn.e18bfca87d5e11e98c0800a098cbcac6:subsystem.st14_nvme_ss_1_1 \ -k 5 nvme list Node SN Model Namespace Usage Format FW Rev ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- /dev/nvme0n1 80BgLFM7xMJbAAAAAAAC NetApp ONTAP Controller 1 107.37 GB / 107.37 GB 4 KiB + 0 B FFFFFFFF lsblk |grep nvme nvme0n1 259:0 0 100G 0 disk dnf install nvme-cli modprobe -r qla2xxx modprobe qla2xxx dmesg |grep traddr [ 6.139862] qla2xxx [0000:04:00.0]-ffff:0: register_localport: host-traddr=nn-0x20000024ff19bb62:pn-0x21000024ff19bb62 on portID:10700 [ 6.241762] qla2xxx [0000:04:00.0]-2102:0: qla_nvme_register_remote: traddr=nn-0x203b00a098cbcac6:pn-0x203d00a098cbcac6 PortID:01050d nvme discover --transport fc \ --traddr nn-0x203b00a098cbcac6:pn-0x203d00a098cbcac6 \ --host-traddr nn-0x20000024ff19bb62:pn-0x21000024ff19bb62 Discovery Log Number of Records 2, Generation counter 49530 =====Discovery Log Entry 0====== trtype: fc adrfam: fibre-channel subtype: nvme subsystem treq: not specified portid: 0 trsvcid: none subnqn: nqn.1992-08.com.netapp:sn.c9ecc9187b1111e98c0800a098cbcac6:subsystem.vs_nvme_multipath_1_subsystem_468 traddr: nn-0x203b00a098cbcac6:pn-0x203d00a098cbcac6 nvme connect --transport fc \ --traddr nn-0x203b00a098cbcac6:pn-0x203d00a098cbcac6 \ --host-traddr nn-0x20000024ff19bb62:pn-0x21000024ff19bb62 \ -n nqn.1992-08.com.netapp:sn.c9ecc9187b1111e98c0800a098cbcac6:subsystem.vs_nvme_multipath_1_subsystem_468 \ -k 5 nvme list Node SN Model Namespace Usage Format FW Rev ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- /dev/nvme0n1 80BgLFM7xMJbAAAAAAAC NetApp ONTAP Controller 1 107.37 GB / 107.37 GB 4 KiB + 0 B FFFFFFFF lsblk |grep nvme nvme0n1 259:0 0 100G 0 disk dnf install nvme-cli cat /etc/nvme/hostnqn nqn.2014-08.org.nvmexpress:uuid:8ae2b12c-3d28-4458-83e3-658e571ed4b8 cat /etc/nvme/hostid 09e2ce17-ccc9-412d-8dcf-2b0a1d581ee3 nmcli device show ens6 GENERAL.DEVICE: ens6 GENERAL.TYPE: ethernet GENERAL.HWADDR: 52:57:02:12:02:02 GENERAL.MTU: 1500 GENERAL.STATE: 30 (disconnected) GENERAL.CONNECTION: -- GENERAL.CON-PATH: -- WIRED-PROPERTIES.CARRIER: on nmcli connection add con-name ens6 ifname ens6 type ethernet ip4 192.168.101.154/24 gw4 192.168.101.1 nmcli connection mod ens6 ipv4.method manual nmcli connection up ens6 nmcli device show ens6 GENERAL.DEVICE: ens6 GENERAL.TYPE: ethernet GENERAL.HWADDR: 52:57:02:12:02:02 GENERAL.MTU: 1500 GENERAL.STATE: 100 (connected) GENERAL.CONNECTION: ens6 GENERAL.CON-PATH: /org/freedesktop/NetworkManager/ActiveConnection/5 WIRED-PROPERTIES.CARRIER: on IP4.ADDRESS[1]: 192.168.101.154/24 IP4.GATEWAY: 192.168.101.1 IP4.ROUTE[1]: dst = 192.168.101.0/24, nh = 0.0.0.0, mt = 101 IP4.ROUTE[2]: dst = 192.168.1.1/32, nh = 0.0.0.0, mt = 101 IP4.ROUTE[3]: dst = 0.0.0.0/0, nh = 192.168.1.1, mt = 101 IP6.ADDRESS[1]: fe80::27ce:dde1:620:996c/64 IP6.GATEWAY: -- IP6.ROUTE[1]: dst = fe80::/64, nh = ::, mt = 101 modprobe nvme-tcp nvme discover --transport=tcp --traddr= 192.168.101.55 --host-traddr= 192.168.101.154 --trsvcid=8009 Discovery Log Number of Records 2, Generation counter 7 =====Discovery Log Entry 0====== trtype: tcp adrfam: ipv4 subtype: current discovery subsystem treq: not specified, sq flow control disable supported portid: 2 trsvcid: 8009 subnqn: nqn.2014-08.org.nvmexpress.discovery traddr: 192.168.101.55 eflags: not specified sectype: none =====Discovery Log Entry 1====== trtype: tcp adrfam: ipv4 subtype: nvme subsystem treq: not specified, sq flow control disable supported portid: 2 trsvcid: 8009 subnqn: nqn.2014-08.org.nvmexpress:uuid:0c468c4d-a385-47e0-8299-6e95051277db traddr: 192.168.101.55 eflags: not specified sectype: none echo "--transport=tcp --traddr= 192.168.101.55 --host-traddr= 192.168.101.154 --trsvcid=8009" >> /etc/nvme/discovery.conf nvme connect-all nvme list-subsys nvme-subsys3 - NQN= nqn.2014-08.org.nvmexpress:uuid:0c468c4d-a385-47e0-8299-6e95051277db +- nvme3 tcp traddr= 192.168.101.55 ,trsvcid=8009,host_traddr= 192.168.101.154 live optimized nvme list Node Generic SN Model Namespace Usage Format FW Rev --------------------- --------------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- /dev/nvme3n1 /dev/ng3n1 d93a63d394d043ab4b74 Linux 1 21.47 GB / 21.47 GB 512 B + 0 B 5.18.5-2 cat /sys/module/nvme_core/parameters/multipath grubby --update-kernel=ALL --args="nvme_core.multipath=N" zipl options nvme_core multipath=N cp /boot/initramfs-USD(uname -r).img /boot/initramfs-USD(uname -r).bak.USD(date +%m%d-%H%M%S).img cp /boot/initramfs-USD(uname -r).img /boot/initramfs-USD(uname -r).bak.USD(date +%m-%d-%H%M%S).img dracut --force --verbose systemctl enable --now multipathd.service devices { device { vendor "NVME" product ".*" path_grouping_policy group_by_prio } } multipath -r cat /sys/module/nvme_core/parameters/multipath N multipath -l eui.00007a8962ab241100a0980000d851c8 dm-6 NVME,NetApp E-Series size=20G features='0' hwhandler='0' wp=rw `-+- policy='service-time 0' prio=0 status=active |- 0:10:2:2 nvme0n2 259:3 active undef running `-+- policy='service-time 0' prio=0 status=enabled |- 4:11:2:2 nvme4n2 259:28 active undef running `-+- policy='service-time 0' prio=0 status=enabled |- 5:32778:2:2 nvme5n2 259:38 active undef running `-+- policy='service-time 0' prio=0 status=enabled |- 6:32779:2:2 nvme6n2 259:44 active undef running cat /sys/module/nvme_core/parameters/multipath grubby --update-kernel=ALL --remove-args="nvme_core.multipath=N" zipl rm /etc/modprobe.d/nvme_core.conf cp /boot/initramfs-USD(uname -r).img /boot/initramfs-USD(uname -r).bak.USD(date +%m-%d-%H%M%S).img dracut --force --verbose echo "round-robin" > /sys/class/nvme-subsystem/nvme-subsys0/iopolicy ACTION=="add|change", SUBSYSTEM=="nvme-subsystem", ATTR{iopolicy}="round-robin" nvme list Node SN Model Namespace Usage Format FW Rev ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- /dev/nvme0n1 a34c4f3a0d6f5cec Linux 1 250.06 GB / 250.06 GB 512 B + 0 B 4.18.0-2 /dev/nvme0n2 a34c4f3a0d6f5cec Linux 2 250.06 GB / 250.06 GB 512 B + 0 B 4.18.0-2 nvme list-subsys nvme-subsys0 - NQN=testnqn +- nvme0 fc traddr=nn-0x20000090fadd597a:pn-0x10000090fadd597a host_traddr=nn-0x20000090fac7e1dd:pn-0x10000090fac7e1dd live +- nvme1 fc traddr=nn-0x20000090fadd5979:pn-0x10000090fadd5979 host_traddr=nn-0x20000090fac7e1dd:pn-0x10000090fac7e1dd live +- nvme2 fc traddr=nn-0x20000090fadd5979:pn-0x10000090fadd5979 host_traddr=nn-0x20000090fac7e1de:pn-0x10000090fac7e1de live +- nvme3 fc traddr=nn-0x20000090fadd597a:pn-0x10000090fadd597a host_traddr=nn-0x20000090fac7e1de:pn-0x10000090fac7e1de live cat /proc/cmdline BOOT_IMAGE=[...] nvme_core.multipath=Y cat /sys/class/nvme-subsystem/nvme-subsys0/iopolicy round-robin dnf install dracut-network add_dracutmodules+=" nfs " systemctl enable --now tftp mkdir -p /var/lib/tftpboot/pxelinux/ cp /usr/share/syslinux/pxelinux.0 /var/lib/tftpboot/pxelinux/ cp /usr/share/syslinux/ldlinux.c32 /var/lib/tftpboot/pxelinux/ mkdir -p /var/lib/tftpboot/pxelinux/pxelinux.cfg/ systemctl status tftp Active: active (running) option space pxelinux; option pxelinux.magic code 208 = string; option pxelinux.configfile code 209 = text; option pxelinux.pathprefix code 210 = text; option pxelinux.reboottime code 211 = unsigned integer 32; option architecture-type code 93 = unsigned integer 16; subnet 192.168.205.0 netmask 255.255.255.0 { option routers 192.168.205.1; range 192.168.205.10 192.168.205.25; class "pxeclients" { match if substring (option vendor-class-identifier, 0, 9) = "PXEClient"; next-server 192.168.205.1; if option architecture-type = 00:07 { filename "BOOTX64.efi"; } else { filename "pxelinux/pxelinux.0"; } } } systemctl enable --now dhcpd.service systemctl status dhcpd.service Active: active (running) dnf install @Base kernel dracut-network nfs-utils --installroot= exported-root-directory --releasever=/ rsync -a -e ssh --exclude='/proc/' --exclude='/sys/' example.com :/ exported-root-directory cp / exported-root-directory /boot/vmlinuz-kernel-version /var/lib/tftpboot/pxelinux/ dracut --add nfs initramfs-kernel-version.img kernel-version dracut --add nfs /exports/root/boot/initramfs-5.14.0-202.el9.x86_64.img 5.14.0-202.el9.x86_64 dracut -f --add nfs "boot/initramfs-USD(uname -r).img" "USD(uname -r)" chmod 0644 / exported-root-directory /boot/initramfs- kernel-version .img cp / exported-root-directory /boot/initramfs- kernel-version .img /var/lib/tftpboot/pxelinux/ default rhel9 label rhel9 kernel vmlinuz- kernel-version append initrd=initramfs- kernel-version .img root=nfs:_server-ip_:/ exported-root-directory rw default rhel9 label rhel9 kernel vmlinuz- kernel-version append initrd=initramfs- kernel-version .img root=nfs: server-ip :/ exported-root-directory ro systemctl restart nfs-server.service chroot /exported/root/directory /bin/bash passwd <username> dnf install <package> --installroot= /exported/root/directory --releasever=/ --config /etc/dnf/dnf.conf --setopt=reposdir=/etc/yum.repos.d/ systemctl status dhcpd.service systemctl enable dhcpd.service systemctl start dhcpd.service firewall-cmd --get-active-zones firewall-cmd --info-zone=public firewall-cmd --add-service=dhcp --permanent firewall-cmd --get-active-zones firewall-cmd --info-zone=public firewall-cmd --add-service=nfs --permanent chmod 644 pxelinux.0 systemctl status nfs.service systemctl start nfs.service systemctl enable nfs.service lvcreate VolGroup00 -n LogVol02 -L 2G mkswap /dev/VolGroup00/LogVol02 /dev/VolGroup00/LogVol02 none swap defaults 0 0 systemctl daemon-reload swapon -v /dev/VolGroup00/LogVol02 cat /proc/swaps total used free shared buff/cache available Mem: 30Gi 1.2Gi 28Gi 12Mi 994Mi 28Gi Swap: 22Gi 0B 22Gi free -h total used free shared buff/cache available Mem: 30Gi 1.2Gi 28Gi 12Mi 995Mi 28Gi Swap: 17Gi 0B 17Gi dd if=/dev/zero of=/swapfile bs=1024 count= 65536 mkswap /swapfile chmod 0600 /swapfile /swapfile none swap defaults 0 0 systemctl daemon-reload swapon /swapfile cat /proc/swaps free -h --- - name: Create a disk device with swap hosts: managed-node-01.example.com roles: - rhel-system-roles.storage vars: storage_volumes: - name: swap_fs type: disk disks: - /dev/sdb size: 15 GiB fs_type: swap ansible-playbook --syntax-check ~/playbook.yml ansible-playbook ~/playbook.yml swapoff -v /dev/VolGroup00/LogVol01 lvresize /dev/VolGroup00/LogVol01 -L +2G mkswap /dev/VolGroup00/LogVol01 swapon -v /dev/VolGroup00/LogVol01 cat /proc/swaps Filename Type Size Used Priority /dev/dm-1 partition 16322556 0 -2 /dev/dm-4 partition 7340028 0 -3 free -h total used free shared buff/cache available Mem: 30Gi 1.2Gi 28Gi 12Mi 994Mi 28Gi Swap: 22Gi 0B 22Gi swapoff -v /dev/VolGroup00/LogVol01 wipefs -a /dev/VolGroup00/LogVol01 lvreduce /dev/VolGroup00/LogVol01 -L -512M mkswap /dev/VolGroup00/LogVol01 swapon -v /dev/VolGroup00/LogVol01 cat /proc/swaps free -h swapoff -v /dev/VolGroup00/LogVol02 lvremove /dev/VolGroup00/LogVol02 /dev/VolGroup00/LogVol02 none swap defaults 0 0 systemctl daemon-reload cat /proc/swaps free -h swapoff -v /swapfile systemctl daemon-reload rm /swapfile dnf install fcoe-utils cp /etc/fcoe/cfg-ethx /etc/fcoe/cfg- enp1s0 systemctl enable --now fcoe fipvlan -s -c enp1s0 Created VLAN device enp1s0.200 Starting FCoE on interface enp1s0.200 Fibre Channel Forwarders Discovered interface | VLAN | FCF MAC ------------------------------------------ enp1s0 | 200 | 00:53:00:a7:e7:1b fcoeadm -t Interface: enp1s0.200 Roles: FCP Target Node Name: 0x500a0980824acd15 Port Name: 0x500a0982824acd15 Target ID: 0 MaxFrameSize: 2048 bytes OS Device Name: rport-11:0-1 FC-ID (Port ID): 0xba00a0 State: Online LUN ID Device Name Capacity Block Size Description ------ ----------- ---------- ---------- --------------------- 0 sdb 28.38 GiB 512 NETAPP LUN (rev 820a) fcoeadm -i Description: BCM57840 NetXtreme II 10 Gigabit Ethernet Revision: 11 Manufacturer: Broadcom Inc. and subsidiaries Serial Number: 000AG703A9B7 Driver: bnx2x Unknown Number of Ports: 1 Symbolic Name: bnx2fc (QLogic BCM57840) v2.12.13 over enp1s0.200 OS Device Name: host11 Node Name: 0x2000000af70ae935 Port Name: 0x2001000af70ae935 Fabric Name: 0x20c8002a6aa7e701 Speed: 10 Gbit Supported Speed: 1 Gbit, 10 Gbit MaxFrameSize: 2048 bytes FC-ID (Port ID): 0xba02c0 State: Online dnf install mt-st mt -f /dev/st0 load mt -f /dev/st0 status SCSI 2 tape drive: File number=-1, block number=-1, partition=0. Tape block size 0 bytes. Density code 0x0 (default). Soft error count since last status=0 General status bits on (50000): DR_OPEN IM_REP_EN tar -czf /dev/st0 _/source/directory tar -czf /dev/st0 _/source/directory tar: Removing leading `/' from member names /source/directory /source/directory /man_db.conf /source/directory /DIR_COLORS /source/directory /rsyslog.conf [...] mt -f /dev/st0 status tar -tzf /dev/st0 /source/directory / /source/directory /man_db.conf /source/directory /DIR_COLORS /source/directory /rsyslog.conf [...] mt -f /dev/nst0 load mt -f /dev/nst0 status mt -f /dev/nst0 rewind mt -f /dev/nst0 eod tar -czf /dev/nst0 /source/directory / tar -czf /dev/nst0 /source/directory / tar: Removing leading `/' from member names /source/directory / /source/directory /man_db.conf /source/directory /DIR_COLORS /source/directory /rsyslog.conf [...] mt -f /dev/nst0 status tar -tzf /dev/nst0 /source/directory / /source/directory /man_db.conf /source/directory /DIR_COLORS /source/directory /rsyslog.conf [...] mt -f /dev/nst0 tell mt -f /dev/nst0 eod mt -f /dev/nst0 bsfm 1 mt -f /dev/nst0 fsf 1 tar -xzf /dev/st0 /source/directory / mt -f /dev/nst0 rewind tar -xzf /dev/nst0 /source/directory / mt -f /dev/st0 erase mt -f /dev/st0 offline mdadm --grow /dev/md0 --level=4 -n3 --add /dev/vdd mdadm --grow /dev/md0 --level=5 -n3 --add /dev/vdd mdadm --grow /dev/md0 --level 10 -n 4 --add /dev/vd[ef] mdadm --grow /dev/md0 -l0 mdadm --grow /dev/md0 --level=5 mdadm --grow /dev/md0 --level=0 mdadm --grow /dev/md0 --level=5 mdadm --grow /dev/md0 --level=0 mdadm -CR /dev/md0 -l5 -n3 /dev/sd[abc] --assume-clean --size 1G mdadm -D /dev/md0 | grep Level mdadm --grow /dev/md0 --array-size 1048576 mdadm --grow -n 2 /dev/md0 --backup=internal mdadm --grow -l1 /dev/md0 mdadm -D /dev/md0 | grep Level mdadm --grow /dev/md0 --level=4 mdadm --grow /dev/md0 --level=6 --add /dev/vde mdadm --grow /dev/md0 --level=0 # mdadm --grow /dev/md0 --level=10 --add /dev/vde /dev/vdf mdadm --grow /dev/md0 --level=5 mdadm --grow /dev/md0 --level=0 mdadm --create /dev/md0 --level= 0 --raid-devices=2 /dev/sda1 /dev/sdc1 mdadm: Defaulting to version 1.2 metadata mdadm: array /dev/md0 started. mdadm --detail /dev/md0 /dev/md0: Version : 1.2 Creation Time : Thu Oct 13 15:17:39 2022 Raid Level : raid0 Array Size : 18649600 (17.79 GiB 19.10 GB) Raid Devices : 2 Total Devices : 2 Persistence : Superblock is persistent Update Time : Thu Oct 13 15:17:39 2022 State : clean Active Devices : 2 Working Devices : 2 Failed Devices : 0 Spare Devices : 0 [...] mdadm --examine /dev/sda1 /dev/sdc1 /dev/sda1: Magic : a92b4efc Version : 1.2 Feature Map : 0x1000 Array UUID : 77ddfb0a:41529b0e:f2c5cde1:1d72ce2c Name : 0 Creation Time : Thu Oct 13 15:17:39 2022 Raid Level : raid0 Raid Devices : 2 [...] mkfs -t xfs /dev/md0 mkdir /mnt/raid1 mount /dev/md0 /mnt/raid1 /dev/md0 /mnt/raid1 xfs defaults 0 0 --- - name: Manage local storage hosts: managed-node-01.example.com tasks: - name: Create a RAID on sdd, sde, sdf, and sdg ansible.builtin.include_role: name: rhel-system-roles.storage vars: storage_safe_mode: false storage_volumes: - name: data type: raid disks: [sdd, sde, sdf, sdg] raid_level: raid0 raid_chunk_size: 32 KiB mount_point: /mnt/data state: present ansible-playbook --syntax-check ~/playbook.yml ansible-playbook ~/playbook.yml ansible managed-node-01.example.com -m command -a 'mdadm --detail /dev/md/data' mdadm --grow --size=max /dev/md0 xfs_growfs /mnt/ mdadm --grow --size= 524228 /dev/md0 dd if= /dev/sda1 of= /dev/sdb1 parted /dev/sda set 1 prep on parted /dev/sda set 1 boot on parted /dev/sdb set 1 prep on parted /dev/sdb set 1 boot on mdadm --detail --scan >> /etc/mdadm.conf MAILADDR example@example.com journalctl -k -f md/raid:md0: Disk failure on sdd, disabling device. md/raid:md0: Operation continuing on 3 devices. mdadm --manage /dev/md0 --fail /dev/sdd mdadm --detail /dev/md0 Number Major Minor RaidDevice State 0 8 16 0 active sync /dev/sdb 1 8 32 1 active sync /dev/sdc - 0 0 2 removed 3 8 64 3 active sync /dev/sde 2 8 48 - faulty /dev/sdd mdadm --manage /dev/md0 --remove /dev/sdd mdadm --manage /dev/md0 --add /dev/sdf mdadm --detail /dev/md0 Number Major Minor RaidDevice State 0 8 16 0 active sync /dev/sdb 1 8 32 1 active sync /dev/sdc 4 8 80 2 spare rebuilding /dev/sdf 3 8 64 3 active sync /dev/sde echo check > /sys/block/ md0 /md/sync_action echo repair > /sys/block/ md0 /md/sync_action cat /sys/block/ md0 /md/sync_action repair cat /proc/mdstat Personalities : [raid0] [raid6] [raid5] [raid4] [raid1] md0 : active raid1 sdg[1] dm-3[0] 511040 blocks super 1.2 [2/2] [UU] unused devices: &lt;none&gt; umount /dev/mapper/vg00-lv00 lvextend -L+ 32M /dev/mapper/vg00-lv00 cryptsetup reencrypt --encrypt --init-only --reduce-device-size 32M /dev/mapper/ vg00-lv00 lv00_encrypted /dev/mapper/ lv00_encrypted is now active and ready for online encryption. mount /dev/mapper/ lv00_encrypted /mnt/lv00_encrypted cryptsetup luksUUID /dev/mapper/ vg00-lv00 a52e2cc9-a5be-47b8-a95d-6bdf4f2d9325 vi /etc/crypttab lv00_encrypted UUID= a52e2cc9-a5be-47b8-a95d-6bdf4f2d9325 none dracut -f --regenerate-all blkid -p /dev/mapper/ lv00_encrypted /dev/mapper/ lv00-encrypted : UUID=" 37bc2492-d8fa-4969-9d9b-bb64d3685aa9 " BLOCK_SIZE="4096" TYPE="xfs" USAGE="filesystem" vi /etc/fstab UUID= 37bc2492-d8fa-4969-9d9b-bb64d3685aa9 /home auto rw,user,auto 0 cryptsetup reencrypt --resume-only /dev/mapper/ vg00-lv00 Enter passphrase for /dev/mapper/ vg00-lv00 : Auto-detected active dm device ' lv00_encrypted ' for data device /dev/mapper/ vg00-lv00 . Finished, time 00:31.130, 10272 MiB written, speed 330.0 MiB/s cryptsetup luksDump /dev/mapper/ vg00-lv00 LUKS header information Version: 2 Epoch: 4 Metadata area: 16384 [bytes] Keyslots area: 16744448 [bytes] UUID: a52e2cc9-a5be-47b8-a95d-6bdf4f2d9325 Label: (no label) Subsystem: (no subsystem) Flags: (no flags) Data segments: 0: crypt offset: 33554432 [bytes] length: (whole device) cipher: aes-xts-plain64 [...] cryptsetup status lv00_encrypted /dev/mapper/ lv00_encrypted is active and is in use. type: LUKS2 cipher: aes-xts-plain64 keysize: 512 bits key location: keyring device: /dev/mapper/ vg00-lv00 umount /dev/ nvme0n1p1 cryptsetup reencrypt --encrypt --init-only --header /home/header /dev/ nvme0n1p1 nvme_encrypted WARNING! ======== Header file does not exist, do you want to create it? Are you sure? (Type 'yes' in capital letters): YES Enter passphrase for /home/header : Verify passphrase: /dev/mapper/ nvme_encrypted is now active and ready for online encryption. mount /dev/mapper/ nvme_encrypted /mnt/nvme_encrypted cryptsetup reencrypt --resume-only --header /home/header /dev/ nvme0n1p1 Enter passphrase for /dev/ nvme0n1p1 : Auto-detected active dm device 'nvme_encrypted' for data device /dev/ nvme0n1p1 . Finished, time 00m51s, 10 GiB written, speed 198.2 MiB/s cryptsetup luksDump /home/header LUKS header information Version: 2 Epoch: 88 Metadata area: 16384 [bytes] Keyslots area: 16744448 [bytes] UUID: c4f5d274-f4c0-41e3-ac36-22a917ab0386 Label: (no label) Subsystem: (no subsystem) Flags: (no flags) Data segments: 0: crypt offset: 0 [bytes] length: (whole device) cipher: aes-xts-plain64 sector: 512 [bytes] [...] cryptsetup status nvme_encrypted /dev/mapper/ nvme_encrypted is active and is in use. type: LUKS2 cipher: aes-xts-plain64 keysize: 512 bits key location: keyring device: /dev/ nvme0n1p1 cryptsetup luksFormat /dev/ nvme0n1p1 WARNING! ======== This will overwrite data on /dev/nvme0n1p1 irrevocably. Are you sure? (Type 'yes' in capital letters): YES Enter passphrase for /dev/ nvme0n1p1 : Verify passphrase: cryptsetup open /dev/ nvme0n1p1 nvme0n1p1_encrypted Enter passphrase for /dev/ nvme0n1p1 : mkfs -t ext4 /dev/mapper/ nvme0n1p1_encrypted mount /dev/mapper/ nvme0n1p1_encrypted mount-point cryptsetup luksDump /dev/ nvme0n1p1 LUKS header information Version: 2 Epoch: 3 Metadata area: 16384 [bytes] Keyslots area: 16744448 [bytes] UUID: 34ce4870-ffdf-467c-9a9e-345a53ed8a25 Label: (no label) Subsystem: (no subsystem) Flags: (no flags) Data segments: 0: crypt offset: 16777216 [bytes] length: (whole device) cipher: aes-xts-plain64 sector: 512 [bytes] [...] cryptsetup status nvme0n1p1_encrypted /dev/mapper/ nvme0n1p1_encrypted is active and is in use. type: LUKS2 cipher: aes-xts-plain64 keysize: 512 bits key location: keyring device: /dev/ nvme0n1p1 sector size: 512 offset: 32768 sectors size: 20938752 sectors mode: read/write ansible-vault create vault.yml New Vault password: <vault_password> Confirm New Vault password: <vault_password> luks_password: <password> --- - name: Manage local storage hosts: managed-node-01.example.com vars_files: - vault.yml tasks: - name: Create and configure a volume encrypted with LUKS ansible.builtin.include_role: name: rhel-system-roles.storage vars: storage_volumes: - name: barefs type: disk disks: - sdb fs_type: xfs fs_label: <label> mount_point: /mnt/data encryption: true encryption_password: "{{ luks_password }}" ansible-playbook --ask-vault-pass --syntax-check ~/playbook.yml ansible-playbook --ask-vault-pass ~/playbook.yml ansible managed-node-01.example.com -m command -a 'cryptsetup luksUUID /dev/sdb' 4e4e7970-1822-470e-b55a-e91efe5d0f5c ansible managed-node-01.example.com -m command -a 'cryptsetup status luks-4e4e7970-1822-470e-b55a-e91efe5d0f5c' /dev/mapper/luks-4e4e7970-1822-470e-b55a-e91efe5d0f5c is active and is in use. type: LUKS2 cipher: aes-xts-plain64 keysize: 512 bits key location: keyring device: /dev/sdb ansible managed-node-01.example.com -m command -a 'cryptsetup luksDump /dev/sdb' LUKS header information Version: 2 Epoch: 3 Metadata area: 16384 [bytes] Keyslots area: 16744448 [bytes] UUID: 4e4e7970-1822-470e-b55a-e91efe5d0f5c Label: (no label) Subsystem: (no subsystem) Flags: (no flags) Data segments: 0: crypt offset: 16777216 [bytes] length: (whole device) cipher: aes-xts-plain64 sector: 512 [bytes] dnf install ndctl ndctl list --namespaces --idle [ { "dev":"namespace1.0", "mode":"raw", "size":34359738368, "state":"disabled", "numa_node":1 }, { "dev":"namespace0.0", "mode":"raw", "size":34359738368, "state":"disabled", "numa_node":0 } ] ndctl create-namespace --force --reconfig= namespace-ID --mode=sector ndctl create-namespace --force --reconfig=namespace1.0 --mode=sector { "dev":"namespace1.0", "mode":"sector", "size":"755.26 GiB (810.95 GB)", "uuid":"2509949d-1dc4-4ee0-925a-4542b28aa616", "sector_size":4096, "blockdev":"pmem1s" } ndctl list --namespace namespace1.0 [ { "dev":"namespace1.0", "mode":"sector", "size":810954706944, "uuid":"2509949d-1dc4-4ee0-925a-4542b28aa616", "sector_size":4096, "blockdev":"pmem1s" } ] ndctl read-labels nmem0 >/dev/null read 1 nmem ndctl list --regions [ { "dev":"region1", "size":2156073582592, "align":16777216, "available_size":2117418876928, "max_available_extent":2117418876928, "type":"pmem", "iset_id":-9102197055295954944, "badblock_count":1, "persistence_domain":"memory_controller" }, { "dev":"region0", "size":2156073582592, "align":16777216, "available_size":2143188680704, "max_available_extent":2143188680704, "type":"pmem", "iset_id":736272362787276936, "badblock_count":3, "persistence_domain":"memory_controller" } ] ndctl create-namespace --mode=sector --region=region N --size=namespace-size ndctl create-namespace --mode=sector --region=region0 --size=36G { "dev":"namespace0.1", "mode":"sector", "size":"35.96 GiB (38.62 GB)", "uuid":"ff5a0a16-3495-4ce8-b86b-f0e3bd9d1817", "sector_size":4096, "blockdev":"pmem0.1s" } ndctl list -RN -n namespace0.1 { "regions":[ { "dev":"region0", "size":2156073582592, "align":16777216, "available_size":2104533975040, "max_available_extent":2104533975040, "type":"pmem", "iset_id":736272362787276936, "badblock_count":3, "persistence_domain":"memory_controller", "namespaces":[ { "dev":"namespace0.1", "mode":"sector", "size":38615912448, "uuid":"ff5a0a16-3495-4ce8-b86b-f0e3bd9d1817", "sector_size":4096, "blockdev":"pmem0.1s" } ] } ] } ndctl list --namespaces --idle [ { "dev":"namespace1.0", "mode":"raw", "size":34359738368, "uuid":"ac951312-b312-4e76-9f15-6e00c8f2e6f4" "state":"disabled", "numa_node":1 }, { "dev":"namespace0.0", "mode":"raw", "size":38615912448, "uuid":"ff5a0a16-3495-4ce8-b86b-f0e3bd9d1817", "state":"disabled", "numa_node":0 } ] ndctl create-namespace --force --mode=devdax --reconfig= namespace-ID ndctl create-namespace --force --mode=devdax --align=2M --reconfig=namespace0.1 { "dev":"namespace0.1", "mode":"devdax", "map":"dev", "size":"35.44 GiB (38.05 GB)", "uuid":"426d6a52-df92-43d2-8cc7-046241d6d761", "daxregion":{ "id":0, "size":"35.44 GiB (38.05 GB)", "align":2097152, "devices":[ { "chardev":"dax0.1", "size":"35.44 GiB (38.05 GB)", "target_node":4, "mode":"devdax" } ] }, "align":2097152 } ndctl list --namespace namespace0.1 [ { "dev":"namespace0.1", "mode":"devdax", "map":"dev", "size":38048628736, "uuid":"426d6a52-df92-43d2-8cc7-046241d6d761", "chardev":"dax0.1", "align":2097152 } ] ndctl read-labels nmem0 >/dev/null read 1 nmem ndctl list --regions [ { "dev":"region1", "size":2156073582592, "align":16777216, "available_size":2117418876928, "max_available_extent":2117418876928, "type":"pmem", "iset_id":-9102197055295954944, "badblock_count":1, "persistence_domain":"memory_controller" }, { "dev":"region0", "size":2156073582592, "align":16777216, "available_size":2143188680704, "max_available_extent":2143188680704, "type":"pmem", "iset_id":736272362787276936, "badblock_count":3, "persistence_domain":"memory_controller" } ] ndctl create-namespace --mode=devdax --region=region N --size= namespace-size ndctl create-namespace --mode=devdax --region=region0 --align=2M --size=36G { "dev":"namespace0.2", "mode":"devdax", "map":"dev", "size":"35.44 GiB (38.05 GB)", "uuid":"89d13f41-be6c-425b-9ec7-1e2a239b5303", "daxregion":{ "id":0, "size":"35.44 GiB (38.05 GB)", "align":2097152, "devices":[ { "chardev":"dax0.2", "size":"35.44 GiB (38.05 GB)", "target_node":4, "mode":"devdax" } ] }, "align":2097152 } ndctl list -RN -n namespace0.2 { "regions":[ { "dev":"region0", "size":2156073582592, "align":16777216, "available_size":2065879269376, "max_available_extent":2065879269376, "type":"pmem", "iset_id":736272362787276936, "badblock_count":3, "persistence_domain":"memory_controller", "namespaces":[ { "dev":"namespace0.2", "mode":"devdax", "map":"dev", "size":38048628736, "uuid":"89d13f41-be6c-425b-9ec7-1e2a239b5303", "chardev":"dax0.2", "align":2097152 } ] } ] } ndctl list --namespaces --idle [ { "dev":"namespace1.0", "mode":"raw", "size":34359738368, "uuid":"ac951312-b312-4e76-9f15-6e00c8f2e6f4" "state":"disabled", "numa_node":1 }, { "dev":"namespace0.0", "mode":"raw", "size":38615912448, "uuid":"ff5a0a16-3495-4ce8-b86b-f0e3bd9d1817", "state":"disabled", "numa_node":0 } ] ndctl create-namespace --force --mode=fsdax --reconfig= namespace-ID ndctl create-namespace --force --mode=fsdax --reconfig=namespace0.0 { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":"11.81 GiB (12.68 GB)", "uuid":"f8153ee3-c52d-4c6e-bc1d-197f5be38483", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ndctl list --namespace namespace0.0 [ { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":12681478144, "uuid":"f8153ee3-c52d-4c6e-bc1d-197f5be38483", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ] ndctl read-labels nmem0 >/dev/null read 1 nmem ndctl list --regions [ { "dev":"region1", "size":2156073582592, "align":16777216, "available_size":2117418876928, "max_available_extent":2117418876928, "type":"pmem", "iset_id":-9102197055295954944, "badblock_count":1, "persistence_domain":"memory_controller" }, { "dev":"region0", "size":2156073582592, "align":16777216, "available_size":2143188680704, "max_available_extent":2143188680704, "type":"pmem", "iset_id":736272362787276936, "badblock_count":3, "persistence_domain":"memory_controller" } ] ndctl create-namespace --mode=fsdax --region=region N --size= namespace-size ndctl create-namespace --mode=fsdax --region=region0 --size=36G { "dev":"namespace0.3", "mode":"fsdax", "map":"dev", "size":"35.44 GiB (38.05 GB)", "uuid":"99e77865-42eb-4b82-9db6-c6bc9b3959c2", "sector_size":512, "align":2097152, "blockdev":"pmem0.3" } ndctl list -RN -n namespace0.3 { "regions":[ { "dev":"region0", "size":2156073582592, "align":16777216, "available_size":2027224563712, "max_available_extent":2027224563712, "type":"pmem", "iset_id":736272362787276936, "badblock_count":3, "persistence_domain":"memory_controller", "namespaces":[ { "dev":"namespace0.3", "mode":"fsdax", "map":"dev", "size":38048628736, "uuid":"99e77865-42eb-4b82-9db6-c6bc9b3959c2", "sector_size":512, "align":2097152, "blockdev":"pmem0.3" } ] } ] } mkfs.xfs -d su=2m,sw=1 fsdax-partition-or-device mount f_sdax-partition-or-device mount-point_ modprobe acpi_ipmi ndctl list --dimms --health [ { "dev":"nmem1", "id":"8089-a2-1834-00001f13", "handle":17, "phys_id":32, "security":"disabled", "health":{ "health_state":"ok", "temperature_celsius":36.0, "controller_temperature_celsius":37.0, "spares_percentage":100, "alarm_temperature":false, "alarm_controller_temperature":false, "alarm_spares":false, "alarm_enabled_media_temperature":true, "temperature_threshold":82.0, "alarm_enabled_ctrl_temperature":true, "controller_temperature_threshold":98.0, "alarm_enabled_spares":true, "spares_threshold":50, "shutdown_state":"clean", "shutdown_count":4 } }, [...] ] ndctl list --dimms --regions --health { "dimms":[ { "dev":"nmem1", "id":"8089-a2-1834-00001f13", "handle":17, "phys_id":32, "security":"disabled", "health":{ "health_state":"ok", "temperature_celsius":35.0, [...] } [...] } ndctl list --dimms --human ndctl list --dimms --human [ { "dev":"nmem1", "id":"XXXX-XX-XXXX-XXXXXXXX", "handle":"0x120", "phys_id":"0x1c" }, { "dev":"nmem0", "id":"XXXX-XX-XXXX-XXXXXXXX", "handle":"0x20", "phys_id":"0x10", "flag_failed_flush":true, "flag_smart_event":true } ] dmidecode dmidecode Handle 0x0010, DMI type 17, 40 bytes Memory Device Array Handle: 0x0004 Error Information Handle: Not Provided Total Width: 72 bits Data Width: 64 bits Size: 125 GB Form Factor: DIMM Set: 1 Locator: DIMM-XXX-YYYY Bank Locator: Bank0 Type: Other Type Detail: Non-Volatile Registered (Buffered) ndctl list --namespaces --dimm= DIMM-ID-number ndctl list --namespaces --dimm=0 [ { "dev":"namespace0.2", "mode":"sector", "size":67042312192, "uuid":"XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", "raw_uuid":"XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", "sector_size":4096, "blockdev":"pmem0.2s", "numa_node":0 }, { "dev":"namespace0.0", "mode":"sector", "size":67042312192, "uuid":"XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", "raw_uuid":"XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", "sector_size":4096, "blockdev":"pmem0s", "numa_node":0 } ] fstrim mount-point fstrim --all fstrim /mnt/non_discard fstrim: /mnt/non_discard : the discard operation is not supported mount -o discard device mount-point --- - name: Manage local storage hosts: managed-node-01.example.com tasks: - name: Enable online block discard ansible.builtin.include_role: name: rhel-system-roles.storage vars: storage_volumes: - name: barefs type: disk disks: - sdb fs_type: xfs mount_point: /mnt/data mount_options: discard ansible-playbook --syntax-check ~/playbook.yml ansible-playbook ~/playbook.yml ansible managed-node-01.example.com -m command -a 'findmnt /mnt/data' systemctl enable --now fstrim.timer Created symlink /etc/systemd/system/timers.target.wants/fstrim.timer -> /usr/lib/systemd/system/fstrim.timer. systemctl status fstrim.timer fstrim.timer - Discard unused blocks once a week Loaded: loaded (/usr/lib/systemd/system/fstrim.timer; enabled; vendor preset: disabled) Active: active (waiting) since Wed 2023-05-17 13:24:41 CEST; 3min 15s ago Trigger: Mon 2023-05-22 01:20:46 CEST; 4 days left Docs: man:fstrim May 17 13:24:41 localhost.localdomain systemd[1]: Started Discard unused blocks once a week. vmstat 1 100 free multipathd disablequeueing map multipath-device umount /mnt/mount-point wipefs -a /dev/vg0/myvol lvremove vg0/myvol vgremove vg0 pvremove /dev/sdc1 wipefs -a /dev/sdc1 parted /dev/sdc rm 1 wipefs -a /dev/sdc multipath -l multipath -f multipath-device blockdev --flushbufs device echo 1 > /sys/block/ device-name /device/delete lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sda 8:0 0 5G 0 disk sr0 11:0 1 1024M 0 rom vda 252:0 0 10G 0 disk |-vda1 252:1 0 1M 0 part |-vda2 252:2 0 100M 0 part /boot/efi `-vda3 252:3 0 9.9G 0 part / dnf install stratisd stratis-cli systemctl enable --now stratisd wipefs --all block-device stratis pool create my-pool block-device stratis pool create my-pool block-device-1 block-device-2 stratis pool list wipefs --all block-device stratis key set --capture-key key-description stratis pool create --key-desc key-description my-pool block-device stratis pool create --key-desc key-description my-pool block-device-1 block-device-2 stratis pool list ansible-vault create vault.yml New Vault password: <vault_password> Confirm New Vault password: <vault_password> luks_password: <password> --- - name: Manage local storage hosts: managed-node-01.example.com vars_files: - vault.yml tasks: - name: Create a new encrypted Stratis pool with Clevis and Tang ansible.builtin.include_role: name: rhel-system-roles.storage vars: storage_pools: - name: mypool disks: - sdd - sde type: stratis encryption: true encryption_password: "{{ luks_password }}" encryption_clevis_pin: tang encryption_tang_url: tang-server.example.com:7500 ansible-playbook --ask-vault-pass --syntax-check ~/playbook.yml ansible-playbook --ask-vault-pass ~/playbook.yml ansible managed-node-01.example.com -m command -a 'sudo stratis report' "clevis_config": { "thp": "j-G4ddvdbVfxpnUbgxlpbe3KutSKmcHttILAtAkMTNA", "url": "tang-server.example.com:7500" }, "clevis_pin": "tang", "in_use": true, "key_description": "blivet-mypool", stratis pool create --no-overprovision pool-name /dev/sdb stratis pool overprovision pool-name <yes|no> stratis pool list Name Total Physical Properties UUID Alerts pool-name 1.42 TiB / 23.96 MiB / 1.42 TiB ~Ca,~Cr,~Op cb7cb4d8-9322-4ac4-a6fd-eb7ae9e1e540 stratis pool overprovision pool-name yes stratis pool list Name Total Physical Properties UUID Alerts pool-name 1.42 TiB / 23.96 MiB / 1.42 TiB ~Ca,~Cr,~Op cb7cb4d8-9322-4ac4-a6fd-eb7ae9e1e540 stratis pool bind nbde --trust-url my-pool tang-server stratis pool bind tpm my-pool key-description stratis key set --capture-key key-description stratis pool list stratis pool unbind clevis my-pool stratis pool start pool-uuid --unlock-method <keyring|clevis> stratis pool stop pool-name stratis pool list stratis pool list --stopped --uuid UUID stratis filesystem create --size number-and-unit my-pool my-fs stratis filesystem create --size 10GiB pool1 filesystem1 stratis fs list my-pool mount /dev/stratis/ my-pool / my-fs mount-point /dev/stratis/ my-pool/my-fs mount-point xfs defaults,x-systemd.requires=stratis-fstab-setup@ pool-uuid .service,x-systemd.after=stratis-fstab-setup@ pool-uuid .service dump-value fsck_value stratis pool add-data my-pool device-1 device-2 device-n stratis blockdev Pool Name Device Node Physical Size State Tier my-pool /dev/sdb 9.10 TiB In-use Data stratis pool Name Total Physical Size Total Physical Used my-pool 9.10 TiB 598 MiB stratis filesystem Pool Name Name Used Created Device my-pool my-fs 546 MiB Nov 08 2018 08:03 /dev/stratis/ my-pool/my-fs stratis fs snapshot my-pool my-fs my-fs-snapshot mount /dev/stratis/ my-pool / my-fs-snapshot mount-point stratis filesystem snapshot my-pool my-fs my-fs-backup umount /dev/stratis/ my-pool / my-fs stratis filesystem destroy my-pool my-fs stratis filesystem snapshot my-pool my-fs-snapshot my-fs mount /dev/stratis/ my-pool / my-fs mount-point umount /dev/stratis/ my-pool / my-fs-snapshot stratis filesystem destroy my-pool my-fs-snapshot umount /dev/stratis/ my-pool / my-fs stratis filesystem destroy my-pool my-fs stratis filesystem list my-pool stratis filesystem list my-pool umount /dev/stratis/ my-pool / my-fs-1 /dev/stratis/ my-pool / my-fs-2 /dev/stratis/ my-pool / my-fs-n stratis filesystem destroy my-pool my-fs-1 my-fs-2 stratis pool destroy my-pool stratis pool list