Block Device Guide
Block Device Guide Red Hat Ceph Storage 8 Managing, creating, configuring, and using Red Hat Ceph Storage Block Devices Red Hat Ceph Storage Documentation Team
rbd help COMMAND SUBCOMMAND rbd help snap list ceph osd pool create POOL_NAME PG_NUM ceph osd pool application enable POOL_NAME rbd rbd pool init -p POOL_NAME ceph osd pool create pool1 ceph osd pool application enable pool1 rbd rbd pool init -p pool1 rbd create IMAGE_NAME --size MEGABYTES --pool POOL_NAME rbd create image1 --size 1024 --pool pool1 rbd ls rbd ls POOL_NAME rbd ls pool1 rbd --image IMAGE_NAME info rbd --image image1 info rbd --image IMAGE_NAME -p POOL_NAME info rbd --image image1 -p pool1 info rbd resize --image IMAGE_NAME --size SIZE rbd resize --image image1 --size 1024 rbd resize --image POOL_NAME / IMAGE_NAME --size SIZE rbd resize --image pool1/image1 --size 1024 rbd rm IMAGE_NAME rbd rm image1 rbd rm IMAGE_NAME -p POOL_NAME rbd rm image1 -p pool1 rbd trash mv [ POOL_NAME /] IMAGE_NAME rbd trash mv pool1/image1 rbd trash rm [ POOL_NAME /] IMAGE_ID rbd trash rm pool1/d35ed01706a0 rbd trash restore [ POOL_NAME /] IMAGE_ID rbd trash restore pool1/d35ed01706a0 rbd trash purge POOL_NAME rbd trash purge pool1 Removing images: 100% complete...done. rbd trash purge schedule add --pool POOL_NAME INTERVAL rbd trash purge schedule add --pool pool1 10m rbd trash purge schedule ls --pool POOL_NAME rbd trash purge schedule ls --pool pool1 every 10m rbd trash purge schedule status POOL NAMESPACE SCHEDULE TIME pool1 2021-08-02 11:50:00 rbd trash purge schedule remove --pool POOL_NAME INTERVAL rbd trash purge schedule remove --pool pool1 10m rbd --image POOL_NAME / IMAGE_NAME info rbd --image pool1/image1 info rbd feature enable POOL_NAME / IMAGE_NAME FEATURE_NAME rbd feature enable pool1/image1 exclusive-lock rbd object-map rebuild POOL_NAME / IMAGE_NAME rbd feature disable POOL_NAME / IMAGE_NAME FEATURE_NAME rbd feature disable pool1/image1 fast-diff rbd image-meta set POOL_NAME / IMAGE_NAME KEY VALUE rbd image-meta set pool1/image1 last_update 2021-06-06 rbd image-meta get POOL_NAME / IMAGE_NAME KEY rbd image-meta get pool1/image1 last_update rbd image-meta list POOL_NAME / IMAGE_NAME rbd image-meta list pool1/image1 rbd image-meta remove POOL_NAME / IMAGE_NAME KEY rbd image-meta remove pool1/image1 last_update rbd config image set POOL_NAME / IMAGE_NAME PARAMETER VALUE rbd config image set pool1/image1 rbd_cache false rbd migration prepare SOURCE_IMAGE TARGET_IMAGE rbd migration prepare pool1/image1 pool2/image2 rbd status TARGET_IMAGE rbd status pool2/image2 Watchers: none Migration: source: pool1/image1 (5e2cba2f62e) destination: pool2/image2 (5e2ed95ed806) state: prepared rbd migration execute TARGET_IMAGE rbd migration execute pool2/image2 rbd status pool2/image2 Watchers: watcher=1.2.3.4:0/3695551461 client.123 cookie=123 Migration: source: pool1/image1 (5e2cba2f62e) destination: pool2/image2 (5e2ed95ed806) state: executed rbd migration commit TARGET_IMAGE rbd migration commit pool2/image2 rbd migration commit pool2/image2 --force rbd export volumes/ VOLUME_NAME - | rbd import --image-format 2 - volumes_new/ VOLUME_NAME rbd export volumes/volume-3c4c63e3-3208-436f-9585-fee4e2a3de16 - | rbd import --image-format 2 - volumes_new/volume-3c4c63e3-3208-436f-9585-fee4e2a3de16 rbd export volume/ VOLUME_NAME FILE_PATH rbd import --image-format 2 FILE_PATH volumes_new/ VOLUME_NAME rbd export volumes/volume-3c4c63e3-3208-436f-9585-fee4e2a3de16 <path of export file> rbd import --image-format 2 <path> volumes_new/volume-3c4c63e3-3208-436f-9585-fee4e2a3de16 rbd map POOLNAME / IMAGE_NAME -- OPT1 VAL1 -- OPT2 VAL2 foo/bar1 id=admin,keyring=/etc/ceph/ceph.client.admin.keyring foo/bar2 id=admin,keyring=/etc/ceph/ceph.client.admin.keyring,options='lock_on_read,queue_depth=1024' systemctl enable rbdmap.service ceph config set client rbd_persistent_cache_mode CACHE_MODE ceph config set client rbd_plugins pwl_cache ceph config set client rbd_persistent_cache_mode ssd ceph config set client rbd_plugins pwl_cache rbd config pool set POOL_NAME rbd_persistent_cache_mode CACHE_MODE rbd config pool set POOL_NAME rbd_plugins pwl_cache rbd config pool set pool1 rbd_persistent_cache_mode ssd rbd config pool set pool1 rbd_plugins pwl_cache rbd config image set POOL_NAME / IMAGE_NAME rbd_persistent_cache_mode CACHE_MODE rbd config image set POOL_NAME / IMAGE_NAME rbd_plugins pwl_cache rbd config image set pool1/image1 rbd_persistent_cache_mode ssd rbd config image set pool1/image1 rbd_plugins pwl_cache rbd_persistent_cache_mode CACHE_MODE rbd_plugins pwl_cache rbd_persistent_cache_path / PATH_TO_CACHE_DIRECTORY 1 rbd_persistent_cache_size PERSISTENT_CACHE_SIZE 2 rbd_cache false rbd_persistent_cache_mode rwl rbd_plugins pwl_cache rbd_persistent_cache_path /mnt/pmem/cache/ rbd_persistent_cache_size 1073741824 rbd_cache false rbd_persistent_cache_mode ssd rbd_plugins pwl_cache rbd_persistent_cache_path /mnt/nvme/cache rbd_persistent_cache_size 1073741824 rbd status POOL_NAME / IMAGE_NAME rbd status pool1/image1 Watchers: watcher=10.10.0.102:0/1061883624 client.25496 cookie=140338056493088 Persistent cache state: host: host02 path: /mnt/nvme0/rbd-pwl.rbd.101e5824ad9a.pool size: 1 GiB mode: ssd stats_timestamp: Mon Apr 18 13:26:32 2022 present: true empty: false clean: false allocated: 509 MiB cached: 501 MiB dirty: 338 MiB free: 515 MiB hits_full: 1450 / 61% hits_partial: 0 / 0% misses: 924 hit_bytes: 192 MiB / 66% miss_bytes: 97 MiB rbd persistent-cache flush POOL_NAME / IMAGE_NAME rbd persistent-cache flush pool1/image1 rbd persistent-cache invalidate POOL_NAME / IMAGE_NAME rbd persistent-cache invalidate pool1/image1 ceph mgr module ls { "always_on_modules": [ "balancer", "crash", "devicehealth", "orchestrator", "pg_autoscaler", "progress", "rbd_support", <-- "status", "telemetry", "volumes" } [user@mon ~]USD rbd perf image iotop [user@mon ~]USD rbd perf image iostat { "type": "native", ["cluster_name": "<cluster-name>",] (specify if image in another cluster, requires <cluster-name>.conf file) ["client_name": "<client-name>",] (for connecting to another cluster, default is client.admin ) "pool_name": " POOL_NAME ", ["pool_id": " POOL_ID ",] (optional, alternative to " POOL_NAME " key) ["pool_namespace": " POOL_NAMESPACE ",] (optional) "image_name": " IMAGE_NAME >", ["image_id": " IMAGE_ID ",] (specify if image is in trash) "snap_name": " SNAP_NAME ", ["snap_id": " SNAP_ID ",] (optional, alternative to " SNAP_NAME " key) } { "type": "native", "pool_name": "rbd", "pool_namespace": "ns1", "image_name": "image1", "snap_name": "snap1" } { "type": "qcow", "stream": { "type": "file", "file_path": "/mnt/image.qcow" } } { "type": "raw", "stream": { "type": "file", "file_path": "/mnt/image-head.raw" }, "snapshots": [ { "type": "raw", "name": "snap1", "stream": { "type": "file", "file_path": "/mnt/image-snap1.raw" } }, ] (optional oldest to newest ordering of snapshots) } { <format unique parameters> "stream": { "type": "file", "file_path": " FILE_PATH " } } { "type": "raw", "stream": { "type": "file", "file_path": "/mnt/image.raw" } } { <format unique parameters> "stream": { "type": "http", "url": " URL_PATH " } } { "type": "raw", "stream": { "type": "http", "url": "http://download.ceph.com/image.raw" } } { <format unique parameters> "stream": { "type": "s3", "url": " URL_PATH ", "access_key": " ACCESS_KEY ", "secret_key": " SECRET_KEY " } } { "type": "raw", "stream": { "type": "s3", "url": "http://s3.ceph.com/bucket/image.raw", "access_key": "NX5QOQKC6BH2IDN8HC7A", "secret_key": "LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn" } } { <format unique parameters> "stream": { "type": "nbd", "uri": "<nbd-uri>", } } { "type": "raw", "stream": { "type": "nbd", "uri": "nbd://nbd.ceph.com/image.raw", } } scp /etc/ceph/ceph.conf root@10.0.67.67:/etc/ceph/c1.conf root@10.0.67.67's password: ceph.conf 100% 263 1.2MB/s 00:00 scp /etc/ceph/ceph.client.admin.keyring root@10.0.67.67:/etc/ceph/c1.keyring root@10.0.67.67's password: ceph.client.admin.keyring scp /etc/ceph/ceph.conf root@10.0.67.67:/etc/ceph/c2.conf ceph.conf 100% 261 864.5KB/s 00:00 scp /etc/ceph/ceph.client.admin.keyring root@10.0.67.67:/etc/ceph/c2.keyring root@10.0.67.67's password: ceph.client.admin.keyring rbd migration prepare SOURCE_POOL_NAME / SOURCE_IMAGE_NAME TARGET_POOL_NAME / SOURCE_IMAGE_NAME rbd migration prepare sourcepool1/sourceimage1 targetpool1/sourceimage1 rbd migration prepare SOURCE_POOL_NAME / SOURCE_IMAGE_NAME TARGET_POOL_NAME / NEW_SOURCE_IMAGE_NAME rbd migration prepare sourcepool1/sourceimage1 targetpool1/newsourceimage1 rbd status TARGET_POOL_NAME / SOURCE_IMAGE_NAME rbd status targetpool1/sourceimage1 Watchers: none Migration: source: sourcepool1/sourceimage1 (adb429cb769a) destination: targetpool2/testimage1 (add299966c63) state: prepared rbd info sourceimage1 rbd: error opening image sourceimage1: (2) No such file or directory rbd trash ls --all sourcepool1 adb429cb769a sourceimage1 cat testspec.json { "type": "raw", "stream": { "type": "s3", "url": "http:10.74.253.18:80/testbucket1/image.raw", "access_key": "RLJOCP6345BGB38YQXI5", "secret_key": "oahWRB2ote2rnLy4dojYjDrsvaBADriDDgtSfk6o" } rbd migration prepare --import-only --source-spec-path " JSON_FILE " TARGET_POOL_NAME rbd migration prepare --import-only --source-spec-path "testspec.json" targetpool1 rbd status targetpool1/sourceimage1 Watchers: none Migration: source: {"stream":{"access_key":"RLJOCP6345BGB38YQXI5","secret_key":"oahWRB2ote2rnLy4dojYjDrsvaBADriDDgtSfk6o","type":"s3","url":"http://10.74.253.18:80/testbucket1/image.raw"},"type":"raw"} destination: targetpool1/sourceimage1 (b13865345e66) state: prepared cat /tmp/native_spec { "cluster_name": "c1", "type": "native", "pool_name": "pool1", "image_name": "image1", "snap_name": "snap1" } ceph: root@rbd-client /]# rbd migration prepare --import-only --source-spec-path /tmp/native_spec c2pool1/c2image1 --cluster c2 ceph: root@rbd-client /]# rbd migration execute c2pool1/c2image1 --cluster c2 Image migration: 100% complete...done. ceph: root@rbd-client /]# rbd migration commit c2pool1/c2image1 --cluster c2 Commit image migration: 100% complete...done. rbd migration execute TARGET_POOL_NAME / SOURCE_IMAGE_NAME rbd migration execute targetpool1/sourceimage1 Image migration: 100% complete...done. rbd status TARGET_POOL_NAME / SOURCE_IMAGE_NAME rbd status targetpool1/sourceimage1 Watchers: none Migration: source: sourcepool1/testimage1 (adb429cb769a) destination: targetpool1/testimage1 (add299966c63) state: executed rbd migration commit TARGET_POOL_NAME / SOURCE_IMAGE_NAME rbd migration commit targetpool1/sourceimage1 Commit image migration: 100% complete...done. rbd trash list --all sourcepool1 rbd migration abort TARGET_POOL_NAME / SOURCE_IMAGE_NAME rbd migration abort targetpool1/sourceimage1 Abort image migration: 100% complete...done. rbd ls sourcepool1 sourceimage1 rbd encryption format POOL_NAME / LUKS_IMAGE luks1|luks2 PASSPHRASE_FILE rbd encryption format pool1/luksimage1 luks1 passphrase.bin rbd device map -t nbd -o encryption-format=luks1|luks2,encryption-passphrase-file=passphrase.txt POOL_NAME / LUKS_IMAGE rbd device map -t nbd -o encryption-format=luks1,encryption-passphrase-file=passphrase.txt pool1/luksimage1 rbd create --size SIZE POOL_NAME / LUKS_IMAGE rbd encryption format POOL_NAME / LUKS_IMAGE luks1|luks2 PASSPHRASE_FILE rbd resize --size 50G --encryption-passphrase-file PASSPHRASE_FILE POOL_NAME / LUKS_IMAGE rbd create --size 50G mypool/myimage rbd encryption format mypool/myimage luks2 passphrase.txt rbd resize --size 50G --encryption-passphrase-file passphrase.txt mypool/myimage rbd snap create POOL_NAME / IMAGE_NAME @ SNAP_NAME rbd snap protect POOL_NAME / IMAGE_NAME @ SNAP_NAME rbd clone POOL_NAME / IMAGE_NAME @ SNAP_NAME POOL_NAME / CLONE_NAME rbd encryption format POOL_NAME / CLONE_NAME luks1 CLONE_PASSPHRASE_FILE rbd snap create mypool/myimage@snap rbd snap protect mypool/myimage@snap rbd clone mypool/myimage@snap mypool/myclone rbd encryption format mypool/myclone luks1 clone-passphrase.bin rbd snap create POOL_NAME / IMAGE_NAME @ SNAP_NAME rbd snap protect POOL_NAME / IMAGE_NAME @ SNAP_NAME rbd clone POOL_NAME / IMAGE_NAME @ SNAP_NAME POOL_NAME / CLONE_NAME rbd encryption format POOL_NAME / CLONE_NAME luks1 CLONE_PASSPHRASE_FILE rbd resize --size SIZE --allow-shrink --encryption-passphrase-file CLONE_PASSPHRASE_FILE --encryption-passphrase-file PASSPHRASE_FILE POOL_NAME / CLONE_NAME rbd snap create mypool/myimage@snap rbd snap protect mypool/myimage@snap rbd clone mypool/myimage@snap mypool/myclone rbd encryption format mypool/myclone luks1 clone-passphrase.bin rbd resize --size 50G --allow-shrink --encryption-passphrase-file clone-passphrase.bin --encryption-passphrase-file passphrase.bin mypool/myclone rbd resize --size SIZE POOL_NAME / LUKS_IMAGE rbd snap create POOL_NAME / IMAGE_NAME @ SNAP_NAME rbd snap protect POOL_NAME / IMAGE_NAME @ SNAP_NAME rbd clone POOL_NAME / IMAGE_NAME @ SNAP_NAME POOL_NAME / CLONE_NAME rbd encryption format POOL_NAME / CLONE_NAME luks2 CLONE_PASSPHRASE_FILE rbd resize --size SIZE --allow-shrink --encryption-passphrase-file PASSPHRASE_FILE POOL_NAME / LUKS_IMAGE rbd resize --size SIZE --allow-shrink --encryption-passphrase-file CLONE_PASSPHRASE_FILE --encryption-passphrase-file PASSPHRASE_FILE POOL_NAME_/ CLONE_NAME rbd resize --size 51G mypool/myimage rbd snap create mypool/myimage@snap rbd snap protect mypool/myimage@snap rbd clone mypool/my-image@snap mypool/myclone rbd encryption format mypool/myclone luks2 clone-passphrase.bin rbd resize --size 50G --allow-shrink --encryption-passphrase-file passphrase.bin mypool/myimage rbd resize --size 50G --allow-shrink --encryption-passphrase-file clone-passphrase.bin --encryption-passphrase-file passphrase.bin mypool/myclone rbd --id USER_ID --keyring=/path/to/secret [commands] rbd --name USERNAME --keyring=/path/to/secret [commands] rbd --id admin --keyring=/etc/ceph/ceph.keyring [commands] rbd --name client.admin --keyring=/etc/ceph/ceph.keyring [commands] rbd --pool POOL_NAME snap create --snap SNAP_NAME IMAGE_NAME rbd --pool pool1 snap create --snap snap1 image1 rbd snap create POOL_NAME / IMAGE_NAME @ SNAP_NAME rbd snap create pool1/image1@snap1 rbd --pool POOL_NAME --image IMAGE_NAME snap ls rbd snap ls POOL_NAME / IMAGE_NAME rbd --pool pool1 --image image1 snap ls rbd snap ls pool1/image1 rbd --pool POOL_NAME snap rollback --snap SNAP_NAME IMAGE_NAME rbd snap rollback POOL_NAME / IMAGE_NAME @ SNAP_NAME rbd --pool pool1 snap rollback --snap snap1 image1 rbd snap rollback pool1/image1@snap1 rbd --pool POOL_NAME snap rm --snap SNAP_NAME IMAGE_NAME rbd snap rm POOL_NAME -/ IMAGE_NAME @ SNAP_NAME rbd --pool pool1 snap rm --snap snap2 image1 rbd snap rm pool1/image1@snap1 rbd --pool POOL_NAME snap purge IMAGE_NAME rbd snap purge POOL_NAME / IMAGE_NAME rbd --pool pool1 snap purge image1 rbd snap purge pool1/image1 rbd snap rename POOL_NAME / IMAGE_NAME @ ORIGINAL_SNAPSHOT_NAME POOL_NAME / IMAGE_NAME @ NEW_SNAPSHOT_NAME rbd snap rename data/dataset@snap1 data/dataset@snap2 ceph osd set-require-min-compat-client mimic rbd --pool POOL_NAME snap protect --image IMAGE_NAME --snap SNAPSHOT_NAME rbd snap protect POOL_NAME / IMAGE_NAME @ SNAPSHOT_NAME rbd --pool pool1 snap protect --image image1 --snap snap1 rbd snap protect pool1/image1@snap1 rbd clone --pool POOL_NAME --image PARENT_IMAGE --snap SNAP_NAME --dest-pool POOL_NAME --dest CHILD_IMAGE_NAME rbd clone POOL_NAME / PARENT_IMAGE @ SNAP_NAME POOL_NAME / CHILD_IMAGE_NAME rbd clone --pool pool1 --image image1 --snap snap1 --dest-pool pool1 --dest childimage1 rbd clone pool1/image1@snap1 pool1/childimage1 ] rbd clone --snap-id 4 pool1/image1 pool1/i1clone1 --rbd-default-clone-format 2 rbd snap ls --all POOL_NAME / PARENT_IMAGE_NAME rbd snap ls --all pool1/image1 SNAPID NAME SIZE PROTECTED TIMESTAMP NAMESPACE 3 snap1 10 GiB yes Thu Jul 25 06:21:33 2024 user 4 .group.2_39d 10 GiB Wed Jul 31 02:28:49 2024 group (pool1/group1@p1g1snap1) rbd clone --snap-id SNAP_ID POOL_NAME / IMAGE_NAME POOL_NAME / CLONE_IMAGE_NAME --rbd-default-clone-format 2 rbd clone --snap-id 4 pool1/image1 pool2/clone2 --rbd-default-clone-format 2 rbd ls -p pool2 clone2 rbd --pool POOL_NAME snap unprotect --image IMAGE_NAME --snap SNAPSHOT_NAME rbd snap unprotect POOL_NAME / IMAGE_NAME @ SNAPSHOT_NAME rbd --pool pool1 snap unprotect --image image1 --snap snap1 rbd snap unprotect pool1/image1@snap1 rbd --pool POOL_NAME children --image IMAGE_NAME --snap SNAP_NAME rbd children POOL_NAME / IMAGE_NAME @ SNAPSHOT_NAME rbd --pool pool1 children --image image1 --snap snap1 rbd children pool1/image1@snap1 rbd --pool POOL_NAME flatten --image IMAGE_NAME rbd flatten POOL_NAME / IMAGE_NAME rbd --pool pool1 flatten --image childimage1 rbd flatten pool1/childimage1 cephadm shell cephadm shell ceph orch apply rbd-mirror --placement= NODENAME ceph orch apply rbd-mirror --placement=host04 rbd create IMAGE_NAME --size MEGABYTES --pool POOL_NAME --image-feature FEATURE FEATURE rbd create image1 --size 1024 --pool data --image-feature exclusive-lock,journaling one or more requested features are already enabled (22) Invalid argument rbd feature enable POOL_NAME / IMAGE_NAME FEATURE , FEATURE rbd feature enable data/image1 exclusive-lock, journaling ceph config set global rbd_default_features 125 ceph config show mon.host01 rbd_default_features rbd mirror pool enable POOL_NAME MODE rbd mirror pool enable data pool rbd mirror pool enable data pool rbd mirror pool enable POOL_NAME MODE rbd mirror pool enable data image rbd mirror pool enable data image rbd mirror pool info POOL_NAME rbd mirror pool info data Mode: pool Site Name: c13d8065-b33d-4cb5-b35f-127a02768e7f Peer Sites: none rbd mirror pool info data Mode: pool Site Name: a4c667e2-b635-47ad-b462-6faeeee78df7 Peer Sites: none rbd mirror pool peer bootstrap create --site-name PRIMARY_LOCAL_SITE_NAME POOL_NAME > PATH_TO_BOOTSTRAP_TOKEN rbd mirror pool peer bootstrap create --site-name site-a data > /root/bootstrap_token_site-a rbd mirror pool peer bootstrap import --site-name SECONDARY_LOCAL_SITE_NAME --direction rx-only POOL_NAME PATH_TO_BOOTSTRAP_TOKEN rbd mirror pool peer bootstrap import --site-name site-b --direction rx-only data /root/bootstrap_token_site-a rbd mirror image status POOL_NAME / IMAGE_NAME rbd mirror image status data/image1 image1: global_id: c13d8065-b33d-4cb5-b35f-127a02768e7f state: up+stopped description: remote image is non-primary service: host03.yuoosv on host03 last_update: 2021-10-06 09:13:58 rbd mirror image status data/image1 image1: global_id: c13d8065-b33d-4cb5-b35f-127a02768e7f cephadm shell cephadm shell ceph orch apply rbd-mirror --placement=host01 ceph orch apply rbd-mirror --placement= NODENAME ceph orch apply rbd-mirror --placement=host04 rbd create IMAGE_NAME --size MEGABYTES --pool POOL_NAME --image-feature FEATURE FEATURE rbd create image1 --size 1024 --pool data --image-feature exclusive-lock,journaling one or more requested features are already enabled (22) Invalid argument rbd feature enable POOL_NAME / IMAGE_NAME FEATURE , FEATURE rbd feature enable data/image1 exclusive-lock, journaling ceph config set global rbd_default_features 125 ceph config show mon.host01 rbd_default_features rbd mirror pool enable POOL_NAME MODE rbd mirror pool enable data pool rbd mirror pool enable data pool rbd mirror pool enable POOL_NAME MODE rbd mirror pool enable data image rbd mirror pool enable data image rbd mirror pool info POOL_NAME rbd mirror pool info data Mode: pool Site Name: c13d8065-b33d-4cb5-b35f-127a02768e7f Peer Sites: none rbd mirror pool info data Mode: pool Site Name: a4c667e2-b635-47ad-b462-6faeeee78df7 Peer Sites: none rbd mirror pool peer bootstrap create --site-name PRIMARY_LOCAL_SITE_NAME POOL_NAME > PATH_TO_BOOTSTRAP_TOKEN rbd mirror pool peer bootstrap create --site-name site-a data > /root/bootstrap_token_site-a rbd mirror pool peer bootstrap import --site-name SECONDARY_LOCAL_SITE_NAME --direction rx-tx POOL_NAME PATH_TO_BOOTSTRAP_TOKEN rbd mirror pool peer bootstrap import --site-name site-b --direction rx-tx data /root/bootstrap_token_site-a rbd mirror image status POOL_NAME / IMAGE_NAME rbd mirror image status data/image1 image1: global_id: a4c667e2-b635-47ad-b462-6faeeee78df7 state: up+stopped description: local image is primary service: host03.glsdbv on host03.ceph.redhat.com last_update: 2021-09-16 10:55:58 peer_sites: name: a state: up+stopped description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} last_update: 2021-09-16 10:55:50 rbd mirror image status data/image1 image1: global_id: a4c667e2-b635-47ad-b462-6faeeee78df7 state: up+replaying description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} service: host05.dtisty on host05 last_update: 2021-09-16 10:57:20 peer_sites: name: b state: up+stopped description: local image is primary last_update: 2021-09-16 10:57:28 rbd mirror pool info POOL_NAME rbd mirror pool info data Mode: pool Site Name: a Peer Sites: UUID: 950ddadf-f995-47b7-9416-b9bb233f66e3 Name: b Mirror UUID: 4696cd9d-1466-4f98-a97a-3748b6b722b3 Direction: rx-tx Client: client.rbd-mirror-peer rbd mirror pool enable POOL_NAME MODE rbd mirror pool enable data pool rbd mirror pool enable data image rbd mirror pool disable POOL_NAME rbd mirror pool disable data rbd mirror pool enable POOL_NAME / LOCAL_NAMESPACE_NAME _MODE --remote-namespace REMOTE_NAMESPACE_NAME rbd mirror pool enable image-pool/namespace-a image --remote-namespace namespace-b Remote cluster: rbd mirror pool enable image-pool/namespace-b image --remote-namespace namespace-a rbd mirror pool disable POOL_NAME / NAMESPACE_NAME rbd mirror pool disable image-pool/namespace-a rbd mirror pool disable image-pool/namespace-b rbd mirror image enable POOL_NAME / IMAGE_NAME rbd mirror image enable data/image2 rbd mirror image disable POOL_NAME / IMAGE_NAME rbd mirror image disable data/image2 rbd mirror image demote POOL_NAME / IMAGE_NAME rbd mirror image demote data/image2 rbd mirror image promote POOL_NAME / IMAGE_NAME rbd mirror image promote data/image2 rbd mirror image promote --force POOL_NAME / IMAGE_NAME rbd mirror image promote --force data/image2 rbd mirror image resync POOL_NAME / IMAGE_NAME rbd mirror image resync data/image2 rbd --cluster CLUSTER_NAME mirror pool peer add POOL_NAME PEER_CLIENT_NAME @ PEER_CLUSTER_NAME -n CLIENT_NAME rbd --cluster site-b mirror pool peer add data client.site-a@site-a -n client.site-b rbd mirror pool peer remove POOL_NAME PEER_UUID rbd mirror pool peer remove data 7e90b4ce-e36d-4f07-8cbc-42050896825d rbd mirror pool status POOL_NAME rbd mirror pool status data health: OK daemon health: OK image health: OK images: 1 total 1 replaying rbd mirror image status POOL_NAME / IMAGE_NAME rbd mirror image status data/image2 image2: global_id: 1e3422a2-433e-4316-9e43-1827f8dbe0ef state: up+unknown description: remote image is non-primary service: pluto008.yuoosv on pluto008 last_update: 2021-10-06 09:37:58 rbd image-meta set POOL_NAME / IMAGE_NAME conf_rbd_mirroring_replay_delay MINIMUM_DELAY_IN_SECONDS rbd image-meta set vms/vm-1 conf_rbd_mirroring_replay_delay 600 cephadm shell rbd mirror image disable POOL_NAME / IMAGE_NAME rbd mirror image disable mirror_pool/mirror_image Mirroring disabled rbd mirror image enable POOL_NAME / IMAGE_NAME snapshot rbd mirror image enable mirror_pool/mirror_image snapshot Mirroring enabled rbd --cluster CLUSTER_NAME mirror image snapshot POOL_NAME / IMAGE_NAME rbd mirror image snapshot data/image1 rbd --cluster CLUSTER_NAME mirror snapshot schedule add --pool POOL_NAME --image IMAGE_NAME INTERVAL [ START_TIME ] rbd mirror snapshot schedule add --pool data --image image1 6h rbd mirror snapshot schedule add --pool data --image image1 24h 14:00:00-05:00 rbd --cluster site-a mirror snapshot schedule ls --pool POOL_NAME --recursive rbd mirror snapshot schedule ls --pool data --recursive POOL NAMESPACE IMAGE SCHEDULE data - - every 1d starting at 14:00:00-05:00 data - image1 every 6h rbd --cluster CLUSTER_NAME mirror snapshot schedule remove --pool POOL_NAME --image IMAGE_NAME INTERVAL START_TIME rbd mirror snapshot schedule remove --pool data --image image1 6h rbd mirror snapshot schedule remove --pool data --image image1 24h 14:00:00-05:00 rbd --cluster site-a mirror snapshot schedule status [--pool POOL_NAME ] [--image IMAGE_NAME ] rbd mirror snapshot schedule status SCHEDULE TIME IMAGE 2021-09-21 18:00:00 data/image1 rbd mirror image demote POOL_NAME / IMAGE_NAME rbd mirror image demote data/image1 rbd mirror image demote data/image2 rbd mirror image promote POOL_NAME / IMAGE_NAME rbd mirror image promote data/image1 rbd mirror image promote data/image2 rbd mirror image status data/image1 image1: global_id: 08027096-d267-47f8-b52e-59de1353a034 state: up+stopped description: local image is primary last_update: 2019-04-17 16:04:37 rbd mirror image status data/image2 image2: global_id: 596f41bc-874b-4cd4-aefe-4929578cc834 state: up+stopped description: local image is primary last_update: 2019-04-17 16:04:37 rbd mirror image promote --force POOL_NAME / IMAGE_NAME rbd mirror image promote --force data/image1 rbd mirror image promote --force data/image2 rbd mirror image status data/image1 image1: global_id: 08027096-d267-47f8-b52e-59de1353a034 state: up+stopping_replay description: force promoted last_update: 2023-04-17 13:25:06 rbd mirror image status data/image1 image1: global_id: 08027096-d267-47f8-b52e-59de1353a034 state: up+stopped description: force promoted last_update: 2023-04-17 13:25:06 cephadm shell ceph orch apply rbd-mirror --placement=host01 rbd mirror pool info POOL_NAME rbd mirror pool info pool_failback rbd mirror pool peer remove POOL_NAME PEER_UUID rbd mirror pool peer remove pool_failback f055bb88-6253-4041-923d-08c4ecbe799a ceph osd pool create POOL_NAME PG_NUM ceph osd pool application enable POOL_NAME rbd rbd pool init -p POOL_NAME ceph osd pool create pool1 ceph osd pool application enable pool1 rbd rbd pool init -p pool1 rbd mirror pool peer bootstrap create --site-name LOCAL_SITE_NAME POOL_NAME > PATH_TO_BOOTSTRAP_TOKEN rbd mirror pool peer bootstrap create --site-name site-a data > /root/bootstrap_token_site-a rbd mirror pool peer bootstrap import --site-name LOCAL_SITE_NAME --direction rx-only POOL_NAME PATH_TO_BOOTSTRAP_TOKEN rbd mirror pool peer bootstrap import --site-name site-b --direction rx-only data /root/bootstrap_token_site-a rbd mirror pool info -p data Mode: image Peers: UUID NAME CLIENT d2ae0594-a43b-4c67-a167-a36c646e8643 site-b client.site-b rbd mirror image status data/image1 image1: global_id: 08027096-d267-47f8-b52e-59de1353a034 state: up+stopped description: local image is primary last_update: 2019-04-22 17:37:48 rbd mirror image status data/image2 image2: global_id: 08027096-d267-47f8-b52e-59de1353a034 state: up+stopped description: local image is primary last_update: 2019-04-22 17:38:18 rbd mirror pool info POOL_NAME / IMAGE_NAME rbd info data/image1 rbd info data/image2 rbd mirror image demote POOL_NAME / IMAGE_NAME rbd mirror image demote data/image1 rbd mirror image resync POOL_NAME / IMAGE_NAME rbd mirror image resync data/image1 Flagged image for resync from primary rbd mirror image resync data/image2 Flagged image for resync from primary rbd mirror image status POOL_NAME / IMAGE_NAME rbd mirror image status data/image1 rbd mirror image status data/image2 rbd mirror image demote POOL_NAME / IMAGE_NAME rbd mirror image demote data/image1 rbd mirror image demote data/image2 rbd mirror image promote POOL_NAME / IMAGE_NAME rbd mirror image promote data/image1 rbd mirror image promote data/image2 rbd mirror image status POOL_NAME / IMAGE_NAME rbd mirror image status data/image1 image1: global_id: 08027096-d267-47f8-b52e-59de1353a034 state: up+stopped description: local image is primary last_update: 2019-04-22 11:14:51 rbd mirror image status data/image2 image2: global_id: 596f41bc-874b-4cd4-aefe-4929578cc834 state: up+stopped description: local image is primary last_update: 2019-04-22 11:14:51 rbd mirror pool peer remove data client.remote@remote --cluster local rbd --cluster site-a mirror pool peer remove data client.site-b@site-b -n client.site-a systemctl stop ceph-rbd-mirror@ CLIENT_ID systemctl disable ceph-rbd-mirror@ CLIENT_ID systemctl disable ceph-rbd-mirror.target systemctl stop ceph-rbd-mirror@site-a systemctl disable ceph-rbd-mirror@site-a systemctl disable ceph-rbd-mirror.target vi /etc/ceph/ceph.conf [client] rbd parent cache enabled = true rbd plugins = parent_cache dnf install ceph-immutable-object-cache ceph auth get-or-create client.ceph-immutable-object-cache. USER_NAME mon 'profile rbd' osd 'profile rbd-read-only' ceph auth get-or-create client.ceph-immutable-object-cache.user mon 'profile rbd' osd 'profile rbd-read-only' [client.ceph-immutable-object-cache.user] key = AQCVPH1gFgHRAhAAp8ExRIsoxQK4QSYSRoVJLw== vi /etc/ceph/ceph.client.ceph-immutable-object-cache.user.keyring [client.ceph-immutable-object-cache.user] key = AQCVPH1gFgHRAhAAp8ExRIsoxQK4QSYSRoVJLw systemctl enable ceph-immutable-object-cache@ceph-immutable-object-cache. USER_NAME systemctl enable ceph-immutable-object-cache@ceph-immutable-object-cache.user Created symlink /etc/systemd/system/ceph-immutable-object-cache.target.wants/ceph-immutable-object-cache@ceph-immutable-object-cache.user.service -> /usr/lib/systemd/system/ceph-immutable-object-cache@.service. systemctl start ceph-immutable-object-cache@ceph-immutable-object-cache. USER_NAME systemctl start ceph-immutable-object-cache@ceph-immutable-object-cache.user systemctl status ceph-immutable-object-cache@ceph-immutable-object-cache. USER_NAME systemctl status ceph-immutable-object-cache@ceph-immutable-object-cache.user ● ceph-immutable-object-cache@ceph-immutable-object-cache.user> Loaded: loaded (/usr/lib/systemd/system/ceph-immutable-objec> Active: active (running) since Mon 2021-04-19 13:49:06 IST; > Main PID: 85020 (ceph-immutable-) Tasks: 15 (limit: 49451) Memory: 8.3M CGroup: /system.slice/system-ceph\x2dimmutable\x2dobject\x2d> └─85020 /usr/bin/ceph-immutable-object-cache -f --cl> subscription-manager repos --enable=rhceph-8-tools-for-rhel-9-x86_64-rpms dnf install ceph-common scp root@ MONITOR_NODE :/etc/ceph/ceph.conf /etc/ceph/ceph.conf scp root@cluster1-node2:/etc/ceph/ceph.conf /etc/ceph/ceph.conf root@192.168.0.32's password: ceph.conf 100% 497 724.9KB/s 00:00 scp root@ MONITOR_NODE :/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring scp root@cluster1-node2:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring root@192.168.0.32's password: ceph.client.admin.keyring 100% 151 265.0KB/s 00:00 rbd map --pool POOL_NAME IMAGE_NAME --id admin rbd map --pool block-device-pool image1 --id admin /dev/rbd0 parted /dev/ MAPPED_BLOCK_DEVICE mklabel msdos parted /dev/rbd0 mklabel msdos Information: You may need to update /etc/fstab. parted /dev/ MAPPED_BLOCK_DEVICE mkpart primary xfs 0% 100% parted /dev/rbd0 mkpart primary xfs 0% 100% Information: You may need to update /etc/fstab. mkfs.xfs /dev/ MAPPED_BLOCK_DEVICE_WITH_PARTITION_NUMBER mkfs.xfs /dev/rbd0p1 meta-data=/dev/rbd0p1 isize=512 agcount=16, agsize=163824 blks = sectsz=512 attr=2, projid32bit=1 = crc=1 finobt=1, sparse=1, rmapbt=0 = reflink=1 data = bsize=4096 blocks=2621184, imaxpct=25 = sunit=16 swidth=16 blks naming =version 2 bsize=4096 ascii-ci=0, ftype=1 log =internal log bsize=4096 blocks=2560, version=2 = sectsz=512 sunit=16 blks, lazy-count=1 realtime =none extsz=4096 blocks=0, rtextents=0 mkdir PATH_TO_DIRECTORY mkdir /mnt/ceph mount /dev/ MAPPED_BLOCK_DEVICE_WITH_PARTITION_NUMBER PATH_TO_DIRECTORY mount /dev/rbd0p1 /mnt/ceph/ df -h PATH_TO_DIRECTORY df -h /mnt/ceph/ Filesystem Size Used Avail Use% Mounted on /dev/rbd0p1 10G 105M 9.9G 2% /mnt/ceph rbd list rbd device map POOL_NAME / IMAGE_NAME --id USER_NAME rbd device map rbd/myimage --id admin rbd device map POOL_NAME / IMAGE_NAME --id USER_NAME --keyring PATH_TO_KEYRING rbd device map POOL_NAME / IMAGE_NAME --id USER_NAME --keyfile PATH_TO_FILE rbd device list rbd device list rbd device unmap /dev/rbd/ POOL_NAME / IMAGE_NAME rbd device unmap /dev/rbd/pool1/image1 ceph osd pool create POOL_NAME PG_NUM ceph osd pool create mypool 100 pool 'mypool' created ceph osd pool application enable POOL_NAME rbd ceph osd pool application enable mypool rbd enabled application 'rbd' on pool 'mypool' rbd pool init -p POOL_NAME rbd pool init -p mypool rbd namespace create --namespace NAMESPACE rbd namespace create --namespace namespace1 rbd namespace create --namespace namespace2 rbd namespace ls --format=json [{"name":"namespace2"},{"name":"namespace1"}] ceph auth get-or-create client. USER_NAME mon 'profile rbd' osd 'profile rbd pool=rbd namespace= NAMESPACE ' -o /etc/ceph/client. USER_NAME .keyring ceph auth get-or-create client.testuser mon 'profile rbd' osd 'profile rbd pool=rbd namespace=namespace1' -o /etc/ceph/client.testuser.keyring ceph auth get-or-create client.newuser mon 'profile rbd' osd 'profile rbd pool=rbd namespace=namespace2' -o /etc/ceph/client.newuser.keyring ceph auth get client. USER_NAME ceph auth get client.testuser [client.testuser] key = AQDMp61hBf5UKRAAgjQ2In0Z3uwAase7mrlKnQ== caps mon = "profile rbd" caps osd = "profile rbd pool=rbd namespace=namespace1" exported keyring for client.testuser ceph auth get client.newuser [client.newuser] key = AQDfp61hVfLFHRAA7D80ogmZl80ROY+AUG4A+Q== caps mon = "profile rbd" caps osd = "profile rbd pool=rbd namespace=namespace2" exported keyring for client.newuser rbd create --namespace NAMESPACE IMAGE_NAME --size SIZE_IN_GB rbd create --namespace namespace1 image01 --size 1G rbd create --namespace namespace2 image02 --size 1G rbd --namespace NAMESPACE ls --long rbd --namespace namespace1 ls --long NAME SIZE PARENT FMT PROT LOCK image01 1 GiB 2 rbd --namespace namespace2 ls --long NAME SIZE PARENT FMT PROT LOCK image02 1 GiB 2 scp /etc/ceph/ceph.conf root@ CLIENT_NODE :/etc/ceph/ scp /etc/ceph/ceph.conf root@host02:/etc/ceph/ root@host02's password: ceph.conf 100% 497 724.9KB/s 00:00 scp /etc/ceph/ceph.client.admin.keyring root@ CLIENT_NODE :/etc/ceph scp /etc/ceph/ceph.client.admin.keyring root@host02:/etc/ceph/ root@host02's password: ceph.client.admin.keyring 100% 151 265.0KB/s 00:00 scp /etc/ceph/ceph.client. USER_NAME .keyring root@ CLIENT_NODE :/etc/ceph/ scp /etc/ceph/client.newuser.keyring root@host02:/etc/ceph/ scp /etc/ceph/client.testuser.keyring root@host02:/etc/ceph/ rbd map --name NAMESPACE IMAGE_NAME -n client. USER_NAME --keyring /etc/ceph/client. USER_NAME .keyring rbd map --namespace namespace1 image01 -n client.testuser --keyring=/etc/ceph/client.testuser.keyring /dev/rbd0 rbd map --namespace namespace2 image02 -n client.newuser --keyring=/etc/ceph/client.newuser.keyring /dev/rbd1 rbd map --namespace namespace2 image02 -n client.testuser --keyring=/etc/ceph/client.testuser.keyring rbd: warning: image already mapped as /dev/rbd1 rbd: sysfs write failed rbd: error asserting namespace: (1) Operation not permitted In some cases useful info is found in syslog - try "dmesg | tail". 2021-12-06 02:49:08.106 7f8d4fde2500 -1 librbd::api::Namespace: exists: error asserting namespace: (1) Operation not permitted rbd: map failed: (1) Operation not permitted rbd map --namespace namespace1 image01 -n client.newuser --keyring=/etc/ceph/client.newuser.keyring rbd: warning: image already mapped as /dev/rbd0 rbd: sysfs write failed rbd: error asserting namespace: (1) Operation not permitted In some cases useful info is found in syslog - try "dmesg | tail". 2021-12-03 12:16:24.011 7fcad776a040 -1 librbd::api::Namespace: exists: error asserting namespace: (1) Operation not permitted rbd: map failed: (1) Operation not permitted rbd showmapped id pool namespace image snap device 0 rbd namespace1 image01 - /dev/rbd0 1 rbd namespace2 image02 - /dev/rbd1 cluster = rados.Rados(conffile='my_ceph.conf') cluster.connect() ioctx = cluster.open_ioctx('mypool') rbd_inst = rbd.RBD() size = 4 * 1024**3 # 4 GiB rbd_inst.create(ioctx, 'myimage', size) image = rbd.Image(ioctx, 'myimage') data = 'foo' * 200 image.write(data, 0) image.close() ioctx.close() cluster.shutdown() import rados import rbd cluster = rados.Rados(conffile='my_ceph_conf') try: ioctx = cluster.open_ioctx('my_pool') try: rbd_inst = rbd.RBD() size = 4 * 1024**3 # 4 GiB rbd_inst.create(ioctx, 'myimage', size) image = rbd.Image(ioctx, 'myimage') try: data = 'foo' * 200 image.write(data, 0) finally: image.close() finally: ioctx.close() finally: cluster.shutdown() with rados.Rados(conffile='my_ceph.conf') as cluster: with cluster.open_ioctx('mypool') as ioctx: rbd_inst = rbd.RBD() size = 4 * 1024**3 # 4 GiB rbd_inst.create(ioctx, 'myimage', size) with rbd.Image(ioctx, 'myimage') as image: data = 'foo' * 200 image.write(data, 0)