File System Guide
File System Guide Red Hat Ceph Storage 8 Configuring and Mounting Ceph File Systems Red Hat Ceph Storage Documentation Team
cephadm shell ceph fs volume create FILESYSTEM_NAME --placement=" NUMBER_OF_DAEMONS HOST_NAME_1 HOST_NAME_2 HOST_NAME_3 " ceph fs volume create test --placement="2 host01 host02" ceph osd pool create DATA_POOL [ PG_NUM ] ceph osd pool create METADATA_POOL [ PG_NUM ] ceph osd pool create cephfs_data 64 ceph osd pool create cephfs_metadata 64 ceph fs new FILESYSTEM_NAME METADATA_POOL DATA_POOL ceph fs new test cephfs_metadata cephfs_data ceph orch apply mds FILESYSTEM_NAME --placement=" NUMBER_OF_DAEMONS HOST_NAME_1 HOST_NAME_2 HOST_NAME_3 " ceph orch apply mds test --placement="2 host01 host02" ceph orch ls ceph fs ls ceph fs status ceph orch ps --daemon_type= DAEMON_NAME ceph orch ps --daemon_type=mds touch mds.yaml service_type: mds service_id: FILESYSTEM_NAME placement: hosts: - HOST_NAME_1 - HOST_NAME_2 - HOST_NAME_3 service_type: mds service_id: fs_name placement: hosts: - host01 - host02 cephadm shell --mount mds.yaml:/var/lib/ceph/mds/mds.yaml cd /var/lib/ceph/mds/ cephadm shell cd /var/lib/ceph/mds/ ceph orch apply -i FILE_NAME .yaml ceph orch apply -i mds.yaml ceph fs new CEPHFS_NAME METADATA_POOL DATA_POOL ceph fs new test metadata_pool data_pool ceph orch ls ceph orch ps --daemon_type= DAEMON_NAME ceph orch ps --daemon_type=mds cephadm shell ceph config set mon mon_allow_pool_delete true ceph fs volume rm FILESYSTEM_NAME --yes-i-really-mean-it ceph fs volume rm cephfs-new --yes-i-really-mean-it ceph orch ls ceph orch rm SERVICE_NAME ceph orch rm mds.test ceph orch ps ceph orch ps ceph fs dump dumped fsmap epoch 399 Filesystem 'cephfs01' (27) e399 max_mds 1 in 0 up {0=20384} failed damaged stopped [mds.a{0:20384} state up:active seq 239 addr [v2:127.0.0.1:6854/966242805,v1:127.0.0.1:6855/966242805]] Standby daemons: [mds.b{-1:10420} state up:standby seq 2 addr [v2:127.0.0.1:6856/2745199145,v1:127.0.0.1:6857/2745199145]] ceph config set STANDBY_DAEMON mds_join_fs FILE_SYSTEM_NAME ceph config set mds.b mds_join_fs cephfs01 ceph fs dump dumped fsmap epoch 405 e405 Filesystem 'cephfs01' (27) max_mds 1 in 0 up {0=10420} failed damaged stopped [mds.b{0:10420} state up:active seq 274 join_fscid=27 addr [v2:127.0.0.1:6856/2745199145,v1:127.0.0.1:6857/2745199145]] 1 Standby daemons: [mds.a{-1:10720} state up:standby seq 2 addr [v2:127.0.0.1:6854/1340357658,v1:127.0.0.1:6855/1340357658]] ceph fs set NAME max_mds NUMBER ceph fs set cephfs max_mds 2 ceph fs status NAME ceph fs status cephfs cephfs - 0 clients ====== +------+--------+-------+---------------+-------+-------+--------+--------+ | RANK | STATE | MDS | ACTIVITY | DNS | INOS | DIRS | CAPS | +------+--------+-------+---------------+-------+-------+--------+--------+ | 0 | active | node1 | Reqs: 0 /s | 10 | 12 | 12 | 0 | | 1 | active | node2 | Reqs: 0 /s | 10 | 12 | 12 | 0 | +------+--------+-------+---------------+-------+-------+--------+--------+ +-----------------+----------+-------+-------+ | POOL | TYPE | USED | AVAIL | +-----------------+----------+-------+-------+ | cephfs_metadata | metadata | 4638 | 26.7G | | cephfs_data | data | 0 | 26.7G | +-----------------+----------+-------+-------+ +-------------+ | STANDBY MDS | +-------------+ | node3 | +-------------+ ceph fs set FS_NAME standby_count_wanted NUMBER ceph fs set cephfs standby_count_wanted 2 ceph fs set FS_NAME allow_standby_replay 1 ceph fs set cephfs allow_standby_replay 1 setfattr -n ceph.dir.pin.distributed -v 1 DIRECTORY_PATH setfattr -n ceph.dir.pin.distributed -v 1 dir1/ setfattr -n ceph.dir.pin.random -v PERCENTAGE_IN_DECIMAL DIRECTORY_PATH setfattr -n ceph.dir.pin.random -v 0.01 dir1/ getfattr -n ceph.dir.pin.random DIRECTORY_PATH getfattr -n ceph.dir.pin.distributed DIRECTORY_PATH getfattr -n ceph.dir.pin.distributed dir1/ file: dir1/ ceph.dir.pin.distributed="1" getfattr -n ceph.dir.pin.random dir1/ file: dir1/ ceph.dir.pin.random="0.01" ceph tell mds.a get subtrees | jq '.[] | [.dir.path, .auth_first, .export_pin]' setfattr -n ceph.dir.pin.distributed -v 0 DIRECTORY_PATH setfattr -n ceph.dir.pin.distributed -v 0 dir1/ getfattr -n ceph.dir.pin.distributed DIRECTORY_PATH getfattr -n ceph.dir.pin.distributed dir1/ setfattr -n ceph.dir.pin -v -1 DIRECTORY_PATH setfattr -n ceph.dir.pin -v -1 dir1/ mkdir -p a/b 1 setfattr -n ceph.dir.pin -v 1 a/ 2 setfattr -n ceph.dir.pin -v 0 a/b 3 setfattr -n ceph.dir.pin -v RANK PATH_TO_DIRECTORY setfattr -n ceph.dir.pin -v 2 cephfs/home ceph fs status NAME ceph fs status cephfs cephfs - 0 clients +------+--------+-------+---------------+-------+-------+--------+--------+ | RANK | STATE | MDS | ACTIVITY | DNS | INOS | DIRS | CAPS | +------+--------+-------+---------------+-------+-------+--------+--------+ | 0 | active | node1 | Reqs: 0 /s | 10 | 12 | 12 | 0 | | 1 | active | node2 | Reqs: 0 /s | 10 | 12 | 12 | 0 | +------+--------+-------+---------------+-------+-------+--------+--------+ +-----------------+----------+-------+-------+ | POOL | TYPE | USED | AVAIL | +-----------------+----------+-------+-------+ | cephfs_metadata | metadata | 4638 | 26.7G | | cephfs_data | data | 0 | 26.7G | +-----------------+----------+-------+-------+ +-------------+ | Standby MDS | +-------------+ | node3 | +-------------+ ceph fs set NAME max_mds NUMBER ceph fs set cephfs max_mds 1 ceph fs status NAME ceph fs status cephfs cephfs - 0 clients +------+--------+-------+---------------+-------+-------+--------+--------+ | RANK | STATE | MDS | ACTIVITY | DNS | INOS | DIRS | CAPS | +------+--------+-------+---------------+-------+-------+--------+--------+ | 0 | active | node1 | Reqs: 0 /s | 10 | 12 | 12 | 0 | +------+--------+-------+---------------+-------+-------+--------|--------+ +-----------------+----------+-------+-------+ | POOl | TYPE | USED | AVAIL | +-----------------+----------+-------+-------+ | cephfs_metadata | metadata | 4638 | 26.7G | | cephfs_data | data | 0 | 26.7G | +-----------------+----------+-------+-------+ +-------------+ | Standby MDS | +-------------+ | node3 | | node2 | +-------------+ ceph orch ps | grep mds ceph tell MDS_SERVICE_NAME counter dump ceph tell mds.cephfs.ceph2-hk-n-0mfqao-node4.isztbk counter dump [ { "key": "mds_client_metrics", "value": [ { "labels": { "fs_name": "cephfs", "id": "24379" }, "counters": { "num_clients": 4 } } ] }, { "key": "mds_client_metrics-cephfs", "value": [ { "labels": { "client": "client.24413", "rank": "0" }, "counters": { "cap_hits": 56, "cap_miss": 9, "avg_read_latency": 0E-9, "avg_write_latency": 0E-9, "avg_metadata_latency": 0E-9, "dentry_lease_hits": 2, "dentry_lease_miss": 12, "opened_files": 0, "opened_inodes": 9, "pinned_icaps": 4, "total_inodes": 9, "total_read_ops": 0, "total_read_size": 0, "total_write_ops": 0, "total_write_size": 0 } }, { "labels": { "client": "client.24502", "rank": "0" }, "counters": { "cap_hits": 921403, "cap_miss": 102382, "avg_read_latency": 0E-9, "avg_write_latency": 0E-9, "avg_metadata_latency": 0E-9, "dentry_lease_hits": 17117, "dentry_lease_miss": 204710, "opened_files": 0, "opened_inodes": 9, "pinned_icaps": 7, "total_inodes": 9, "total_read_ops": 0, "total_read_size": 0, "total_write_ops": 1, "total_write_size": 132 } }, { "labels": { "client": "client.24508", "rank": "0" }, "counters": { "cap_hits": 928694, "cap_miss": 103183, "avg_read_latency": 0E-9, "avg_write_latency": 0E-9, "avg_metadata_latency": 0E-9, "dentry_lease_hits": 17217, "dentry_lease_miss": 206348, "opened_files": 0, "opened_inodes": 9, "pinned_icaps": 7, "total_inodes": 9, "total_read_ops": 0, "total_read_size": 0, "total_write_ops": 1, "total_write_size": 132 } }, { "labels": { "client": "client.24520", "rank": "0" }, "counters": { "cap_hits": 56, "cap_miss": 9, "avg_read_latency": 0E-9, "avg_write_latency": 0E-9, "avg_metadata_latency": 0E-9, "dentry_lease_hits": 2, "dentry_lease_miss": 12, "opened_files": 0, "opened_inodes": 9, "pinned_icaps": 4, "total_inodes": 9, "total_read_ops": 0, "total_read_size": 0, "total_write_ops": 0, "total_write_size": 0 } } ] } ] client.0 key: AQAz7EVWygILFRAAdIcuJ10opU/JKyfFmxhuaw== caps: [mds] allow rwp caps: [mon] allow r caps: [osd] allow rw tag cephfs data=cephfs_a client.1 key: AQAz7EVWygILFRAAdIcuJ11opU/JKyfFmxhuaw== caps: [mds] allow rw caps: [mon] allow r caps: [osd] allow rw tag cephfs data=cephfs_a client.0 key: AQAz7EVWygILFRAAdIcuJ10opU/JKyfFmxhuaw== caps: [mds] allow rw, allow rws path=/temp caps: [mon] allow r caps: [osd] allow rw tag cephfs data=cephfs_a client.0 key: AQAz7EVWygILFRAAdIcuJ10opU/JKyfFmxhuaw== caps: [mds] allow r network 10.0.0.0/8, allow rw path=/bar network 10.0.0.0/8 caps: [mon] allow r network 10.0.0.0/8 caps: [osd] allow rw tag cephfs data=cephfs_a network 10.0.0.0/8 subscription-manager repos --enable=rhceph-6-tools-for-rhel-8-x86_64-rpms subscription-manager repos --enable=rhceph-6-tools-for-rhel-9-x86_64-rpms dnf install ceph-fuse scp root@ MONITOR_NODE_NAME :/etc/ceph/ KEYRING_FILE /etc/ceph/ scp root@192.168.0.1:/etc/ceph/ceph.client.1.keyring /etc/ceph/ scp root@ MONITOR_NODE_NAME :/etc/ceph/ceph.conf /etc/ceph/ceph.conf scp root@192.168.0.1:/etc/ceph/ceph.conf /etc/ceph/ceph.conf chmod 644 /etc/ceph/ceph.conf ceph fs volume create FILE_SYSTEM_NAME ceph fs volume create cephfs01 ceph fs authorize FILE_SYSTEM_NAME CLIENT_NAME DIRECTORY PERMISSIONS ceph fs authorize cephfs01 client.1 / rw [client.1] key = BQAmthpf81M+JhAAiHDYQkMiCq3x+J0n9e8REK== ceph auth get client.1 exported keyring for client.1 [client.1] key = BQAmthpf81M+JhAAiHDYQkMiCq3x+J0n9e8REK== caps mds = "allow rw fsname=cephfs01" caps mon = "allow r fsname=cephfs01" caps osd = "allow rw tag cephfs data=cephfs01" ceph fs authorize cephfs01 client.1 / rw root_squash /volumes rw [client.1] key = BQAmthpf81M+JhAAiHDYQkMiCq3x+J0n9e8REK== ceph auth get client.1 [client.1] key = BQAmthpf81M+JhAAiHDYQkMiCq3x+J0n9e8REK== caps mds = "allow rw fsname=cephfs01 root_squash, allow rw fsname=cephfs01 path=/volumes" caps mon = "allow r fsname=cephfs01" caps osd = "allow rw tag cephfs data=cephfs01" ceph auth get CLIENT_NAME > OUTPUT_FILE_NAME scp OUTPUT_FILE_NAME TARGET_NODE_NAME :/etc/ceph ceph auth get client.1 > ceph.client.1.keyring exported keyring for client.1 scp ceph.client.1.keyring client:/etc/ceph root@client's password: ceph.client.1.keyring 100% 178 333.0KB/s 00:00 mkdir PATH_TO_NEW_DIRECTORY_NAME mkdir /mnt/mycephfs ceph-fuse PATH_TO_NEW_DIRECTORY_NAME -n CEPH_USER_NAME --client-fs=_FILE_SYSTEM_NAME ceph-fuse /mnt/mycephfs/ -n client.1 --client-fs=cephfs01 ceph-fuse[555001]: starting ceph client 2022-05-09T07:33:27.158+0000 7f11feb81200 -1 init, newargv = 0x55fc4269d5d0 newargc=15 ceph-fuse[555001]: starting fuse ceph osd pool create DATA_POOL_NAME erasure ceph osd pool create cephfs-data-ec01 erasure pool 'cephfs-data-ec01' created ceph osd lspools ceph osd pool set DATA_POOL_NAME allow_ec_overwrites true ceph osd pool set cephfs-data-ec01 allow_ec_overwrites true set pool 15 allow_ec_overwrites to true ceph fs status FILE_SYSTEM_NAME ceph fs status cephfs-ec cephfs-ec - 14 clients ========= RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active cephfs-ec.example.ooymyq Reqs: 0 /s 8231 8233 891 921 POOL TYPE USED AVAIL cephfs-metadata-ec metadata 787M 8274G cephfs-data-ec data 2360G 12.1T STANDBY MDS cephfs-ec.example.irsrql cephfs-ec.example.cauuaj ceph fs add_data_pool FILE_SYSTEM_NAME DATA_POOL_NAME ceph fs add_data_pool cephfs-ec cephfs-data-ec01 ceph fs status FILE_SYSTEM_NAME ceph fs status cephfs-ec cephfs-ec - 14 clients ========= RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active cephfs-ec.example.ooymyq Reqs: 0 /s 8231 8233 891 921 POOL TYPE USED AVAIL cephfs-metadata-ec metadata 787M 8274G cephfs-data-ec data 2360G 12.1T cephfs-data-ec01 data 0 12.1T STANDBY MDS cephfs-ec.example.irsrql cephfs-ec.example.cauuaj mkdir PATH_TO_DIRECTORY setfattr -n ceph.dir.layout.pool -v DATA_POOL_NAME PATH_TO_DIRECTORY mkdir /mnt/cephfs/newdir setfattr -n ceph.dir.layout.pool -v cephfs-data-ec01 /mnt/cephfs/newdir cephadm shell ceph fs authorize FILE_SYSTEM_NAME client. CLIENT_NAME / DIRECTORY CAPABILITY [/ DIRECTORY CAPABILITY ] PERMISSIONS ceph fs authorize cephfs_a client.1 / r /temp rw client.1 key = AQBSdFhcGZFUDRAAcKhG9Cl2HPiDMMRv4DC43A== ceph fs authorize cephfs_a client.1 /temp rw ceph auth get client. ID ceph auth get client.1 client.1 key = AQBSdFhcGZFUDRAAcKhG9Cl2HPiDMMRv4DC43A== caps mds = "allow r, allow rw path=/temp" caps mon = "allow r" caps osd = "allow rw tag cephfs data=cephfs_a" ceph auth get client. ID -o ceph.client. ID .keyring ceph auth get client.1 -o ceph.client.1.keyring exported keyring for client.1 scp /ceph.client. ID .keyring root@ CLIENT_NODE_NAME :/etc/ceph/ceph.client. ID .keyring scp /ceph.client.1.keyring root@client01:/etc/ceph/ceph.client.1.keyring chmod 644 ceph.client. ID .keyring chmod 644 /etc/ceph/ceph.client.1.keyring subscription-manager repos --enable=rhceph-6-tools-for-rhel-9-x86_64-rpms dnf install ceph-common cephadm shell scp /ceph.client. ID .keyring root@ CLIENT_NODE_NAME :/etc/ceph/ceph.client. ID .keyring scp /ceph.client.1.keyring root@client01:/etc/ceph/ceph.client.1.keyring scp /etc/ceph/ceph.conf root@ CLIENT_NODE_NAME :/etc/ceph/ceph.conf scp /etc/ceph/ceph.conf root@client01:/etc/ceph/ceph.conf chmod 644 /etc/ceph/ceph.conf mkdir -p MOUNT_POINT mkdir -p /mnt/cephfs mount -t ceph MONITOR-1_NAME :6789, MONITOR-2_NAME :6789, MONITOR-3_NAME :6789:/ MOUNT_POINT -o name= CLIENT_ID ,fs= FILE_SYSTEM_NAME mount -t ceph mon1:6789,mon2:6789,mon3:6789:/ /mnt/cephfs -o name=1,fs=cephfs01 mount -t ceph mon1:6789,mon2:6789,mon3:6789:/ /mnt/cephfs -o nowsync,name=1,fs=cephfs01 stat -f MOUNT_POINT stat -f /mnt/cephfs mkdir -p MOUNT_POINT mkdir -p /mnt/cephfs #DEVICE PATH TYPE OPTIONS MON_0_HOST : PORT , MOUNT_POINT ceph name= CLIENT_ID , MON_1_HOST : PORT , ceph.client_mountpoint=/ VOL / SUB_VOL_GROUP / SUB_VOL / UID_SUB_VOL , fs= FILE_SYSTEM_NAME , MON_2_HOST : PORT :/q[_VOL_]/ SUB_VOL / UID_SUB_VOL , [ ADDITIONAL_OPTIONS ] #DEVICE PATH TYPE OPTIONS DUMP FSCK mon1:6789, /mnt/cephfs ceph name=1, 0 0 mon2:6789, ceph.client_mountpoint=/my_vol/my_sub_vol_group/my_sub_vol/0, mon3:6789:/ fs=cephfs01, _netdev,noatime subscription-manager repos --enable=6-tools-for-rhel-8-x86_64-rpms subscription-manager repos --enable=6-tools-for-rhel-9-x86_64-rpms dnf install ceph-fuse cephadm shell scp /ceph.client. ID .keyring root@ CLIENT_NODE_NAME :/etc/ceph/ceph.client. ID .keyring scp /ceph.client.1.keyring root@client01:/etc/ceph/ceph.client.1.keyring scp /etc/ceph/ceph.conf root@ CLIENT_NODE_NAME :/etc/ceph/ceph.conf scp /etc/ceph/ceph.conf root@client01:/etc/ceph/ceph.conf chmod 644 /etc/ceph/ceph.conf mkdir PATH_TO_MOUNT_POINT mkdir /mnt/mycephfs ceph-fuse -n client. CLIENT_ID --client_fs FILE_SYSTEM_NAME MOUNT_POINT ceph-fuse -n client.1 --client_fs cephfs01 /mnt/mycephfs ceph-fuse -n client.1 --keyring=/etc/ceph/client.1.keyring /mnt/mycephfs ceph-fuse -n client. CLIENT_ID MOUNT_POINT -r PATH ceph-fuse -n client.1 /mnt/cephfs -r /home/cephfs ceph-fuse -n client.1 /mnt/cephfs --client_reconnect_stale=true stat -f MOUNT_POINT stat -f /mnt/cephfs mkdir PATH_TO_MOUNT_POINT mkdir /mnt/mycephfs #DEVICE PATH TYPE OPTIONS DUMP FSCK HOST_NAME : PORT , MOUNT_POINT fuse.ceph ceph.id= CLIENT_ID , 0 0 HOST_NAME : PORT , ceph.client_mountpoint=/ VOL / SUB_VOL_GROUP / SUB_VOL / UID_SUB_VOL , HOST_NAME : PORT :/ ceph.client_fs= FILE_SYSTEM_NAME ,ceph.name= USERNAME ,ceph.keyring=/etc/ceph/ KEYRING_FILE , [ ADDITIONAL_OPTIONS ] #DEVICE PATH TYPE OPTIONS DUMP FSCK mon1:6789, /mnt/mycephfs fuse.ceph ceph.id=1, 0 0 mon2:6789, ceph.client_mountpoint=/my_vol/my_sub_vol_group/my_sub_vol/0, mon3:6789:/ ceph.client_fs=cephfs01,ceph.name=client.1,ceph.keyring=/etc/ceph/client1.keyring, _netdev,defaults ceph fs volume create VOLUME_NAME ceph fs volume create cephfs ceph fs volume ls ceph fs volume info VOLUME_NAME ceph fs volume info cephfs { "mon_addrs": [ "192.168.1.7:40977", ], "pending_subvolume_deletions": 0, "pools": { "data": [ { "avail": 106288709632, "name": "cephfs.cephfs.data", "used": 4096 } ], "metadata": [ { "avail": 106288709632, "name": "cephfs.cephfs.meta", "used": 155648 } ] }, "used_size": 0 } ceph config set mon mon_allow_pool_delete true ceph fs volume rm VOLUME_NAME [--yes-i-really-mean-it] ceph fs volume rm cephfs --yes-i-really-mean-it ceph fs subvolumegroup create VOLUME_NAME GROUP_NAME [--pool_layout DATA_POOL_NAME --uid UID --gid GID --mode OCTAL_MODE ] ceph fs subvolumegroup create cephfs subgroup0 ceph fs subvolumegroup create VOLUME_NAME GROUP_NAME [--size SIZE_IN_BYTES ] [--pool_layout DATA_POOL_NAME ] [--uid UID ] [--gid GID ] [--mode OCTAL_MODE ] ceph fs subvolumegroup create cephfs subvolgroup_2 10737418240 ceph fs subvolumegroup resize VOLUME_NAME GROUP_NAME new_size [--no_shrink] ceph fs subvolumegroup resize cephfs subvolgroup_2 20737418240 [ { "bytes_used": 10768679044 }, { "bytes_quota": 20737418240 }, { "bytes_pcent": "51.93" } ] ceph fs subvolumegroup info VOLUME_NAME GROUP_NAME ceph fs subvolumegroup info cephfs subvolgroup_2 { "atime": "2022-10-05 18:00:39", "bytes_pcent": "51.85", "bytes_quota": 20768679043, "bytes_used": 10768679044, "created_at": "2022-10-05 18:00:39", "ctime": "2022-10-05 18:21:26", "data_pool": "cephfs.cephfs.data", "gid": 0, "mode": 16877, "mon_addrs": [ "60.221.178.236:1221", "205.64.75.112:1221", "20.209.241.242:1221" ], "mtime": "2022-10-05 18:01:25", "uid": 0 } ceph fs subvolumegroup ls VOLUME_NAME ceph fs subvolumegroup ls cephfs ceph fs subvolumegroup getpath VOLUME_NAME GROUP_NAME ceph fs subvolumegroup getpath cephfs subgroup0 ceph fs subvolumegroup snapshot ls VOLUME_NAME GROUP_NAME ceph fs subvolumegroup snapshot ls cephfs subgroup0 ceph fs subvolumegroup snapshot rm VOLUME_NAME GROUP_NAME SNAP_NAME [--force] ceph fs subvolumegroup snapshot rm cephfs subgroup0 snap0 --force ceph fs subvolumegroup rm VOLUME_NAME GROUP_NAME [--force] ceph fs subvolumegroup rm cephfs subgroup0 --force ceph fs subvolume create VOLUME_NAME SUBVOLUME_NAME [--size SIZE_IN_BYTES --group_name SUBVOLUME_GROUP_NAME --pool_layout DATA_POOL_NAME --uid _UID --gid GID --mode OCTAL_MODE ] [--namespace-isolated] ceph fs subvolume create cephfs sub0 --group_name subgroup0 --namespace-isolated ceph fs subvolume ls VOLUME_NAME [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume ls cephfs --group_name subgroup0 ceph fs subvolume resize VOLUME_NAME SUBVOLUME_NAME NEW_SIZE [--group_name SUBVOLUME_GROUP_NAME ] [--no_shrink] ceph fs subvolume resize cephfs sub0 1024000000 --group_name subgroup0 --no_shrink ceph fs subvolume getpath VOLUME_NAME SUBVOLUME_NAME [--group_name _SUBVOLUME_GROUP_NAME ] ceph fs subvolume getpath cephfs sub0 --group_name subgroup0 ceph fs subvolume info VOLUME_NAME SUBVOLUME_NAME [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume info cephfs sub0 --group_name subgroup0 ceph fs subvolume info cephfs sub0 { "atime": "2023-07-14 08:52:46", "bytes_pcent": "0.00", "bytes_quota": 1024000000, "bytes_used": 0, "created_at": "2023-07-14 08:52:46", "ctime": "2023-07-14 08:53:54", "data_pool": "cephfs.cephfs.data", "features": [ "snapshot-clone", "snapshot-autoprotect", "snapshot-retention" ], "flavor": "2", "gid": 0, "mode": 16877, "mon_addrs": [ "10.0.208.172:6789", "10.0.211.197:6789", "10.0.209.212:6789" ], "mtime": "2023-07-14 08:52:46", "path": "/volumes/_nogroup/sub0/834c5cbc-f5db-4481-80a3-aca92ff0e7f3", "pool_namespace": "", "state": "complete", "type": "subvolume", "uid": 0 } ceph auth get CLIENT_NAME ceph auth get client.0 [client.0] key = AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw== caps mds = "allow rw, allow rws path=/bar" 1 caps mon = "allow r" caps osd = "allow rw tag cephfs data=cephfs_a" 2 ceph fs subvolume snapshot create VOLUME_NAME SUBVOLUME_NAME SNAP_NAME [--group_name GROUP_NAME ] ceph fs subvolume snapshot create cephfs sub0 snap0 --group_name subgroup0 CLIENT_NAME key = AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw== caps mds = allow rw, allow rws path= DIRECTORY_PATH caps mon = allow r caps osd = allow rw tag cephfs data= DIRECTORY_NAME [client.0] key = AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw== caps mds = "allow rw, allow rws path=/bar" caps mon = "allow r" caps osd = "allow rw tag cephfs data=cephfs_a" ceph fs volume create VOLUME_NAME ceph fs volume create cephfs ceph fs subvolumegroup create VOLUME_NAME GROUP_NAME [--pool_layout DATA_POOL_NAME --uid UID --gid GID --mode OCTAL_MODE ] ceph fs subvolumegroup create cephfs subgroup0 ceph fs subvolume create VOLUME_NAME SUBVOLUME_NAME [--size SIZE_IN_BYTES --group_name SUBVOLUME_GROUP_NAME --pool_layout DATA_POOL_NAME --uid _UID --gid GID --mode OCTAL_MODE ] ceph fs subvolume create cephfs sub0 --group_name subgroup0 ceph fs subvolume snapshot create VOLUME_NAME _SUBVOLUME_NAME SNAP_NAME [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume snapshot create cephfs sub0 snap0 --group_name subgroup0 ceph fs subvolume snapshot clone VOLUME_NAME SUBVOLUME_NAME SNAP_NAME TARGET_CLONE_NAME ceph fs subvolume snapshot clone cephfs sub0 snap0 clone0 ceph fs subvolume snapshot clone VOLUME_NAME SUBVOLUME_NAME SNAP_NAME TARGET_CLONE_NAME --group_name SUBVOLUME_GROUP_NAME ceph fs subvolume snapshot clone cephfs sub0 snap0 clone0 --group_name subgroup0 ceph fs subvolume snapshot clone VOLUME_NAME SUBVOLUME_NAME SNAP_NAME TARGET_CLONE_NAME --target_group_name SUBVOLUME_GROUP_NAME ceph fs subvolume snapshot clone cephfs sub0 snap0 clone0 --target_group_name subgroup1 ceph fs clone status VOLUME_NAME CLONE_NAME [--group_name TARGET_GROUP_NAME ] ceph fs clone status cephfs clone0 --group_name subgroup1 { "status": { "state": "complete" } } ceph fs subvolume snapshot ls VOLUME_NAME SUBVOLUME_NAME [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume snapshot ls cephfs sub0 --group_name subgroup0 ceph fs subvolume snapshot info VOLUME_NAME SUBVOLUME_NAME SNAP_NAME [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume snapshot info cephfs sub0 snap0 --group_name subgroup0 { "created_at": "2022-05-09 06:18:47.330682", "data_pool": "cephfs_data", "has_pending_clones": "no", "size": 0 } ceph fs subvolume rm VOLUME_NAME SUBVOLUME_NAME [--group_name SUBVOLUME_GROUP_NAME ] [--force] [--retain-snapshots] ceph fs subvolume rm cephfs sub0 --group_name subgroup0 --retain-snapshots ceph fs subvolume snapshot clone VOLUME_NAME DELETED_SUBVOLUME RETAINED_SNAPSHOT NEW_SUBVOLUME --group_name SUBVOLUME_GROUP_NAME --target_group_name SUBVOLUME_TARGET_GROUP_NAME ceph fs subvolume snapshot clone cephfs sub0 snap0 sub1 --group_name subgroup0 --target_group_name subgroup0 ceph fs subvolume snapshot rm VOLUME_NAME SUBVOLUME_NAME SNAP_NAME [--group_name GROUP_NAME --force] ceph fs subvolume snapshot rm cephfs sub0 snap0 --group_name subgroup0 --force ceph fs subvolume metadata set VOLUME_NAME SUBVOLUME_NAME KEY_NAME VALUE [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume metadata set cephfs sub0 test_meta cluster --group_name subgroup0 ceph fs subvolume metadata set cephfs sub0 "test meta" cluster --group_name subgroup0 ceph fs subvolume metadata set cephfs sub0 "test_meta" cluster2 --group_name subgroup0 ceph fs subvolume metadata get VOLUME_NAME SUBVOLUME_NAME KEY_NAME [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume metadata get cephfs sub0 test_meta --group_name subgroup0 cluster ceph fs subvolume metadata ls VOLUME_NAME SUBVOLUME_NAME [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume metadata ls cephfs sub0 { "test_meta": "cluster" } ceph fs subvolume metadata rm VOLUME_NAME SUBVOLUME_NAME KEY_NAME [--group_name SUBVOLUME_GROUP_NAME ] ceph fs subvolume metadata rm cephfs sub0 test_meta --group_name subgroup0 ceph fs subvolume metadata ls cephfs sub0 {} subscription-manager repos --enable=rhceph-8-tools-for-rhel-9-x86_64-rpms dnf install cephfs-top ceph mgr module enable stats ceph auth get-or-create client.fstop mon 'allow r' mds 'allow r' osd 'allow r' mgr 'allow r' > /etc/ceph/ceph.client.fstop.keyring cephfs-top cephfs-top - Wed Nov 30 15:26:05 2022 All Filesystem Info Total Client(s): 4 - 3 FUSE, 1 kclient, 0 libcephfs COMMANDS: m - select a filesystem | s - sort menu | l - limit number of clients | r - reset to default | q - quit client_id mount_root chit(%) dlease(%) ofiles oicaps oinodes rtio(MB) raio(MB) rsp(MB/s) wtio(MB) waio(MB) wsp(MB/s) rlatavg(ms) rlatsd(ms) wlatavg(ms) wlatsd(ms) mlatavg(ms) mlatsd(ms) mount_point@host/addr Filesystem: cephfs1 - 2 client(s) 4500 / 100.0 100.0 0 751 0 0.0 0.0 0.0 578.13 0.03 0.0 N/A N/A N/A N/A N/A N/A N/A@example/192.168.1.4 4501 / 100.0 0.0 0 1 0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.41 0.0 /mnt/cephfs2@example/192.168.1.4 Filesystem: cephfs2 - 2 client(s) 4512 / 100.0 0.0 0 1 0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.4 0.0 /mnt/cephfs3@example/192.168.1.4 4518 / 100.0 0.0 0 1 0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.52 0.0 /mnt/cephfs4@example/192.168.1.4 m Filesystems Press "q" to go back to home (all filesystem info) screen cephfs01 cephfs02 q cephfs-top - Thu Oct 20 07:29:35 2022 Total Client(s): 3 - 2 FUSE, 1 kclient, 0 libcephfs cephfs-top --selftest selftest ok ceph mgr module enable mds_autoscaler umount MOUNT_POINT umount /mnt/cephfs fusermount -u MOUNT_POINT fusermount -u /mnt/cephfs ceph fs authorize FILE_SYSTEM_NAME client. CLIENT_NAME / DIRECTORY CAPABILITY [/ DIRECTORY CAPABILITY ] [user@client ~]USD ceph fs authorize cephfs_a client.1 /temp rwp client.1 key: AQBSdFhcGZFUDRAAcKhG9Cl2HPiDMMRv4DC43A== caps: [mds] allow r, allow rwp path=/temp caps: [mon] allow r caps: [osd] allow rw tag cephfs data=cephfs_a setfattr -n ceph.dir.pin -v RANK DIRECTORY [user@client ~]USD setfattr -n ceph.dir.pin -v 2 /temp setfattr -n ceph.dir.pin -v -1 DIRECTORY [user@client ~]USD setfattr -n ceph.dir.pin -v -1 /home/ceph-user ceph osd pool create POOL_NAME ceph osd pool create cephfs_data_ssd pool 'cephfs_data_ssd' created ceph fs add_data_pool FS_NAME POOL_NAME ceph fs add_data_pool cephfs cephfs_data_ssd added data pool 6 to fsmap ceph fs ls name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data cephfs_data_ssd] ceph fs rm_data_pool FS_NAME POOL_NAME ceph fs rm_data_pool cephfs cephfs_data_ssd removed data pool 6 from fsmap ceph fs ls name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs.cephfs.data] ceph fs set FS_NAME down true ceph fs set cephfs down true ceph fs set FS_NAME down false ceph fs set cephfs down false ceph fs fail FS_NAME ceph fs fail cephfs ceph fs set FS_NAME joinable true ceph fs set cephfs joinable true cephfs marked joinable; MDS may join as newly active. ceph fs set FS_NAME down true ceph fs set cephfs down true cephfs marked down. ceph fs status ceph fs status cephfs - 0 clients ====== +-------------------+----------+-------+-------+ | POOL | TYPE | USED | AVAIL | +-----------------+------------+-------+-------+ |cephfs.cephfs.meta | metadata | 31.5M | 52.6G| |cephfs.cephfs.data | data | 0 | 52.6G| +-----------------+----------+-------+---------+ STANDBY MDS cephfs.ceph-host01 cephfs.ceph-host02 cephfs.ceph-host03 ceph fs rm FS_NAME --yes-i-really-mean-it ceph fs rm cephfs --yes-i-really-mean-it ceph fs ls ceph mds fail MDS_NAME ceph mds fail example01 fs required_client_features FILE_SYSTEM_NAME add FEATURE_NAME fs required_client_features FILE_SYSTEM_NAME rm FEATURE_NAME ceph tell DAEMON_NAME client ls ceph tell mds.0 client ls [ { "id": 4305, "num_leases": 0, "num_caps": 3, "state": "open", "replay_requests": 0, "completed_requests": 0, "reconnecting": false, "inst": "client.4305 172.21.9.34:0/422650892", "client_metadata": { "ceph_sha1": "79f0367338897c8c6d9805eb8c9ad24af0dcd9c7", "ceph_version": "ceph version 16.2.8-65.el8cp (79f0367338897c8c6d9805eb8c9ad24af0dcd9c7)", "entity_id": "0", "hostname": "senta04", "mount_point": "/tmp/tmpcMpF1b/mnt.0", "pid": "29377", "root": "/" } } ] ceph tell DAEMON_NAME client evict id= ID_NUMBER ceph tell mds.0 client evict id=4305 ceph osd blocklist ls listed 1 entries 127.0.0.1:0/3710147553 2022-05-09 11:32:24.716146 ceph osd blocklist rm CLIENT_NAME_OR_IP_ADDR ceph osd blocklist rm 127.0.0.1:0/3710147553 un-blocklisting 127.0.0.1:0/3710147553 recover_session=clean client_reconnect_stale=true getfattr -n ceph.quota.max_bytes DIRECTORY getfattr -n ceph.quota.max_bytes /mnt/cephfs/ getfattr: Removing leading '/' from absolute path names file: mnt/cephfs/ ceph.quota.max_bytes="100000000" getfattr -n ceph.quota.max_files DIRECTORY getfattr -n ceph.quota.max_files /mnt/cephfs/ getfattr: Removing leading '/' from absolute path names file: mnt/cephfs/ ceph.quota.max_files="10000" setfattr -n ceph.quota.max_bytes -v LIMIT_VALUE DIRECTORY setfattr -n ceph.quota.max_bytes -v 2T /cephfs/ setfattr -n ceph.quota.max_files -v LIMIT_VALUE DIRECTORY setfattr -n ceph.quota.max_files -v 10000 /cephfs/ setfattr -n ceph.quota.max_bytes -v 0 DIRECTORY setfattr -n ceph.quota.max_bytes -v 0 /mnt/cephfs/ setfattr -n ceph.quota.max_files -v 0 DIRECTORY setfattr -n ceph.quota.max_files -v 0 /mnt/cephfs/ setfattr -n ceph. TYPE .layout. FIELD -v VALUE PATH setfattr -n ceph.file.layout.stripe_unit -v 1048576 test getfattr -n ceph. TYPE .layout PATH getfattr -n ceph.dir.layout /home/test ceph.dir.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data" getfattr -n ceph. TYPE .layout. FIELD _PATH getfattr -n ceph.file.layout.pool test ceph.file.layout.pool="cephfs_data" setfattr -x ceph.dir.layout DIRECTORY_PATH [user@client ~]USD setfattr -x ceph.dir.layout /home/cephfs setfattr -x ceph.dir.layout.pool_namespace DIRECTORY_PATH [user@client ~]USD setfattr -x ceph.dir.layout.pool_namespace /home/cephfs cephadm shell ceph fs set FILE_SYSTEM_NAME allow_new_snaps true ceph fs set cephfs01 allow_new_snaps true mkdir NEW_DIRECTORY_PATH mkdir /.snap/new-snaps rmdir NEW_DIRECTORY_PATH rmdir /.snap/new-snaps cephadm shell ceph mgr module enable snap_schedule cephadm shell ceph fs snap-schedule add FILE_SYSTEM_VOLUME_PATH REPEAT_INTERVAL [ START_TIME ] --fs CEPH_FILE_SYSTEM_NAME ceph fs snap-schedule add /cephfs_kernelf739cwtus2/pmo9axbwsi 1h 2022-06-27T21:50:00 --fs mycephfs ceph fs snap-schedule retention add FILE_SYSTEM_VOLUME_PATH [ COUNT_TIME_PERIOD_PAIR ] TIME_PERIOD COUNT ceph fs snap-schedule retention add /cephfs h 14 1 ceph fs snap-schedule retention add /cephfs d 4 2 ceph fs snap-schedule retention add /cephfs 14h4w 3 ceph fs snap-schedule list FILE_SYSTEM_VOLUME_PATH [--format=plain|json] [--recursive=true] ceph fs snap-schedule list /cephfs --recursive=true ceph fs snap-schedule status FILE_SYSTEM_VOLUME_PATH [--format=plain|json] ceph fs snap-schedule status /cephfs --format=json ceph fs subvolume getpath VOLUME_NAME SUBVOLUME_NAME SUBVOLUME_GROUP_NAME ceph fs subvolume getpath cephfs subvol_1 subvolgroup_1 ceph fs snap-schedule add SUBVOLUME_DIR_PATH SNAP_SCHEDULE [ START_TIME ] --fs CEPH_FILE_SYSTEM_NAME --subvol SUBVOLUME_NAME ceph fs snap-schedule add /cephfs_kernelf739cwtus2/pmo9axbwsi 1h 2022-06-27T21:50:00 --fs cephfs --subvol subvol_1 Schedule set for path /.. ceph fs snap-schedule add /.. SNAP_SCHEDULE [ START_TIME] --fs CEPH_FILE_SYSTEM_NAME --subvol _SUBVOLUME_NAME ceph fs snap-schedule add - 2M --subvol sv_non_def_1 ceph fs snap-schedule add /.. SNAP_SCHEDULE [ START_TIME] --fs CEPH_FILE_SYSTEM_NAME --subvol _SUBVOLUME_NAME --group NON_DEFAULT_SUBVOLGROUP_NAME ceph fs snap-schedule add - 2M --fs cephfs --subvol sv_non_def_1 --group svg1 ceph fs snap-schedule retention add SUBVOLUME_DIR_PATH [ COUNT_TIME_PERIOD_PAIR ] TIME_PERIOD COUNT ceph fs snap-schedule retention add /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. h 14 1 ceph fs snap-schedule retention add /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. d 4 2 ceph fs snap-schedule retention add /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. 14h4w 3 Retention added to path /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. ceph fs snap-schedule retention add / [ COUNT_TIME_PERIOD_PAIR ] TIME_PERIOD_COUNT --fs CEPH_FILE_SYSTEM_NAME --subvol SUBVOLUME_NAME ceph fs snap-schedule retention add / 5h --fs cephfs --subvol sv_sched Retention added to path /volumes/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. ceph fs snap-schedule retention add / [ COUNT_TIME_PERIOD_PAIR ] TIME_PERIOD_COUNT --fs CEPH_FILE_SYSTEM_NAME --subvol SUBVOLUME_NAME --group NON_DEFAULT_SUBVOLGROUP_NAME ceph fs snap-schedule retention add / 5h --fs cephfs --subvol sv_sched --group subvolgroup_cg Retention added to path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a54j0dda7f16/.. ceph fs snap-schedule list SUBVOLUME_VOLUME_PATH [--format=plain|json] [--recursive=true] ceph fs snap-schedule list / --recursive=true /volumes/_nogroup/subv1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. 4h ceph fs snap-schedule status SUBVOLUME_DIR_PATH [--format=plain|json] ceph fs snap-schedule status /volumes/_nogroup/subv1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. --format=json {"fs": "cephfs", "subvol": "subvol_1", "path": "/volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/..", "rel_path": "/..", "schedule": "4h", "retention": {"h": 14}, "start": "2022-05-16T14:00:00", "created": "2023-03-20T08:47:18", "first": null, "last": null, "last_pruned": null, "created_count": 0, "pruned_count": 0, "active": true} ceph fs snap-schedule status --fs CEPH_FILE_SYSTEM_NAME --subvol SUBVOLUME_NAME ceph fs snap-schedule status --fs cephfs --subvol sv_sched {"fs": "cephfs", "subvol": "sv_sched", "group": "subvolgroup_cg", "path": "/volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/..", "rel_path": "/volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/..", "schedule": "1h", "retention": {"h": 5}, "start": "2024-05-21T00:00:00", "created": "2024-05-21T09:18:58", "first": null, "last": null, "last_pruned": null, "created_count": 0, "pruned_count": 0, "active": true} ceph fs snap-schedule status --fs _CEPH_FILE_SYSTEM_NAME_ --subvol _SUBVOLUME_NAME_ --group _NON-DEFAULT_SUBVOLGROUP_NAME_ ceph fs snap-schedule status --fs cephfs --subvol sv_sched --group subvolgroup_cg {"fs": "cephfs", "subvol": "sv_sched", "group": "subvolgroup_cg", "path": "/volumes/subvolgroup_cg/sv_sched/e564329a-kj87-4763-gh0y-b56c8sev7t23/..", "rel_path": "/volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/..", "schedule": "1h", "retention": {"h": 5}, "start": "2024-05-21T00:00:00", "created": "2024-05-21T09:18:58", "first": null, "last": null, "last_pruned": null, "created_count": 0, "pruned_count": 0, "active": true} ceph fs snap-schedule activate FILE_SYSTEM_VOLUME_PATH [ REPEAT_INTERVAL ] ceph fs snap-schedule activate /cephfs ceph fs snap-schedule activate SUBVOL_DIR_PATH [ REPEAT_INTERVAL ] ceph fs snap-schedule activate /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. ceph fs snap-schedule activate /.. REPEAT_INTERVAL --fs CEPH_FILE_SYSTEM_NAME --subvol SUBVOLUME_NAME ceph fs snap-schedule activate / --fs cephfs --subvol sv_sched Schedule activated for path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. ceph fs snap-schedule activate /.. [ REPEAT_INTERVAL ] --fs CEPH_FILE_SYSTEM_NAME --subvol SUBVOLUME_NAME --group NON-DEFAULT_GROUP_NAME ceph fs snap-schedule activate / --fs cephfs --subvol sv_sched --group subvolgroup_cg Schedule activated for path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. ceph fs snap-schedule deactivate FILE_SYSTEM_VOLUME_PATH [ REPEAT_INTERVAL ] ceph fs snap-schedule deactivate /cephfs 1d ceph fs snap-schedule deactivate SUBVOL_DIR_PATH [ REPEAT_INTERVAL ] ceph fs snap-schedule deactivate /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. 1d ceph fs snap-schedule deactivate / REPEAT_INTERVAL --fs CEPH_FILE_SYSTEM_NAME --subvol SUBVOLUME_NAME ceph fs snap-schedule deactivate / --fs cephfs --subvol sv_sched Schedule deactivated for path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. ceph fs snap-schedule deactivate / REPEAT_INTERVAL --fs CEPH_FILE_SYSTEM_NAME --subvol SUBVOLUME_NAME --group NON-DEFAULT_GROUP_NAME ceph fs snap-schedule deactivate / --fs cephfs --subvol sv_sched --group subvolgroup_cg Schedule deactivated for path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. ceph fs snap-schedule remove FILE_SYSTEM_VOLUME_PATH [ REPEAT_INTERVAL ] [ START_TIME ] ceph fs snap-schedule remove /cephfs 4h 2022-05-16T14:00:00 ceph fs snap-schedule remove FILE_SYSTEM_VOLUME_PATH ceph fs snap-schedule remove /cephfs ceph fs snap-schedule remove SUBVOL_DIR_PATH [ REPEAT_INTERVAL ] [ START_TIME ] ceph fs snap-schedule remove /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. 4h 2022-05-16T14:00:00 ceph fs snap-schedule remove / --fs CEPH_FILESYSTEM_NAME --subvol SUBVOLUME_NAME ceph fs snap-schedule remove / --fs cephfs --subvol sv_sched Schedule removed for path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. ceph fs snap-schedule remove / --fs CEPH_FILESYSTEM_NAME --subvol SUBVOLUME_NAME --group NON-DEFAULT_GROUP_NAME ceph fs snap-schedule remove / --fs cephfs --subvol sv_sched --group subvolgroup_cg Schedule removed for path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. ceph fs snap-schedule retention remove FILE_SYSTEM_VOLUME_PATH [ COUNT_TIME_PERIOD_PAIR ] TIME_PERIOD COUNT ceph fs snap-schedule retention remove /cephfs h 4 1 ceph fs snap-schedule retention remove /cephfs 14d4w 2 ceph fs snap-schedule retention remove SUBVOL_DIR_PATH [ COUNT_TIME_PERIOD_PAIR ] TIME_PERIOD COUNT ceph fs snap-schedule retention remove /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. h 4 1 ceph fs snap-schedule retention remove /volumes/_nogroup/subvol_1/85a615da-e8fa-46c1-afc3-0eb8ae64a954/.. 14d4w 2 ceph fs snap-schedule retention remove / TIME_PERIOD_PAIR TIME_PERIOD COUNT --fs CEPH_FILESYSTEM_NAME --subvol SUBVOLUME_NAME ceph fs snap-schedule retention remove / 5h --fs cephfs --subvol sv_sched --group subvolgroup_cg Retention removed from path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. ceph fs snap-schedule retention remove / TIME_PERIOD_PAIR TIME_PERIOD COUNT --fs CEPH_FILESYSTEM_NAME --subvol SUBVOLUME_NAME --group NON-DEFAULT_GROUP_NAME ceph fs snap-schedule retention remove / 5h --fs cephfs --subvol sv_sched --group subvolgroup_cg Retention removed from path /volumes/subvolgroup_cg/sv_sched/e704342a-ff07-4763-bb0b-a46d9dda6f27/.. cephadm shell ceph orch apply cephfs-mirror [" NODE_NAME "] ceph orch apply cephfs-mirror "node1.example.com" Scheduled cephfs-mirror update ceph orch apply cephfs-mirror --placement=" PLACEMENT_SPECIFICATION " ceph orch apply cephfs-mirror --placement="3 host1 host2 host3" Scheduled cephfs-mirror update Error EINVAL: name component must include only a-z, 0-9, and - ceph fs authorize FILE_SYSTEM_NAME CLIENT_NAME / rwps ceph fs authorize cephfs client.mirror_remote / rwps [client.mirror_remote] key = AQCjZ5Jg739AAxAAxduIKoTZbiFJ0lgose8luQ== ceph mgr module enable mirroring ceph fs snapshot mirror enable FILE_SYSTEM_NAME ceph fs snapshot mirror enable cephfs ceph fs snapshot mirror disable FILE_SYSTEM_NAME ceph fs snapshot mirror disable cephfs ceph mgr module enable mirroring ceph fs snapshot mirror peer_bootstrap create FILE_SYSTEM_NAME CLIENT_NAME SITE_NAME ceph fs snapshot mirror peer_bootstrap create cephfs client.mirror_remote remote-site {"token": "eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ=="} ceph fs snapshot mirror peer_bootstrap import FILE_SYSTEM_NAME TOKEN ceph fs snapshot mirror peer_bootstrap import cephfs eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ== ceph fs snapshot mirror peer_list FILE_SYSTEM_NAME ceph fs snapshot mirror peer_list cephfs {"e5ecb883-097d-492d-b026-a585d1d7da79": {"client_name": "client.mirror_remote", "site_name": "remote-site", "fs_name": "cephfs", "mon_host": "[v2:10.0.211.54:3300/0,v1:10.0.211.54:6789/0] [v2:10.0.210.56:3300/0,v1:10.0.210.56:6789/0] [v2:10.0.210.65:3300/0,v1:10.0.210.65:6789/0]"}} ceph fs snapshot mirror peer_remove FILE_SYSTEM_NAME PEER_UUID ceph fs snapshot mirror peer_remove cephfs e5ecb883-097d-492d-b026-a585d1d7da79 ceph fs snapshot mirror add FILE_SYSTEM_NAME PATH ceph fs snapshot mirror add cephfs /volumes/_nogroup/subvol_1 ceph fs snapshot mirror remove FILE_SYSTEM_NAME PATH ceph fs snapshot mirror remove cephfs /home/user1 cephadm shell ceph fs snapshot mirror daemon status ceph fs snapshot mirror daemon status [ { "daemon_id": 15594, "filesystems": [ { "filesystem_id": 1, "name": "cephfs", "directory_count": 1, "peers": [ { "uuid": "e5ecb883-097d-492d-b026-a585d1d7da79", "remote": { "client_name": "client.mirror_remote", "cluster_name": "remote-site", "fs_name": "cephfs" }, "stats": { "failure_count": 1, "recovery_count": 0 } } ] } ] } ] ceph --admin-daemon PATH_TO_THE_ASOK_FILE help ceph --admin-daemon /var/run/ceph/1011435c-9e30-4db6-b720-5bf482006e0e/ceph-client.cephfs-mirror.node1.bndvox.asok help { "fs mirror peer status cephfs@11 1011435c-9e30-4db6-b720-5bf482006e0e": "get peer mirror status", "fs mirror status cephfs@11": "get filesystem mirror status", } ceph --admin-daemon PATH_TO_THE_ASOK_FILE fs mirror status FILE_SYSTEM_NAME @_FILE_SYSTEM_ID ceph --admin-daemon /var/run/ceph/1011435c-9e30-4db6-b720-5bf482006e0e/ceph-client.cephfs-mirror.node1.bndvox.asok fs mirror status cephfs@11 { "rados_inst": "192.168.0.5:0/1476644347", "peers": { "1011435c-9e30-4db6-b720-5bf482006e0e": { 1 "remote": { "client_name": "client.mirror_remote", "cluster_name": "remote-site", "fs_name": "cephfs" } } }, "snap_dirs": { "dir_count": 1 } } ceph --admin-daemon PATH_TO_ADMIN_SOCKET fs mirror status FILE_SYSTEM_NAME @ FILE_SYSTEM_ID PEER_UUID ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@11 1011435c-9e30-4db6-b720-5bf482006e0e { "/home/user1": { "state": "idle", 1 "last_synced_snap": { "id": 120, "name": "snap1", "sync_duration": 0.079997898999999997, "sync_time_stamp": "274900.558797s" }, "snaps_synced": 2, 2 "snaps_deleted": 0, 3 "snaps_renamed": 0 } } ceph fs snapshot mirror dirmap FILE_SYSTEM_NAME PATH ceph fs snapshot mirror dirmap cephfs /volumes/_nogroup/subvol_1 { "instance_id": "25184", 1 "last_shuffled": 1661162007.012663, "state": "mapped" } ceph fs snapshot mirror dirmap cephfs /volumes/_nogroup/subvol_1 { "reason": "no mirror daemons running", "state": "stalled" 1 } ceph --admin-daemon ASOK_FILE_NAME counter dump ceph --admin-daemon ceph-client.cephfs-mirror.ceph1-hk-n-0mfqao-node7.pnbrlu.2.93909288073464.asok counter dump [ { "key": "cephfs_mirror", "value": [ { "labels": {}, "counters": { "mirrored_filesystems": 1, "mirror_enable_failures": 0 } } ] }, { "key": "cephfs_mirror_mirrored_filesystems", "value": [ { "labels": { "filesystem": "cephfs" }, "counters": { "mirroring_peers": 1, "directory_count": 1 } } ] }, { "key": "cephfs_mirror_peers", "value": [ { "labels": { "peer_cluster_filesystem": "cephfs", "peer_cluster_name": "remote_site", "source_filesystem": "cephfs", "source_fscid": "1" }, "counters": { "snaps_synced": 1, "snaps_deleted": 0, "snaps_renamed": 0, "sync_failures": 0, "avg_sync_time": { "avgcount": 1, "sum": 4.216959457, "avgtime": 4.216959457 }, "sync_bytes": 132 } } ] } ]