Administration Guide
Administration Guide Red Hat Ceph Storage 7 Administration of Red Hat Ceph Storage Red Hat Ceph Storage Documentation Team
systemctl --type=service ceph-499829b4-832f-11eb-8d6d-001a4a000635@mon.host01.service systemctl start SERVICE_ID systemctl start ceph-499829b4-832f-11eb-8d6d-001a4a000635@mon.host01.service systemctl stop SERVICE_ID systemctl stop ceph-499829b4-832f-11eb-8d6d-001a4a000635@mon.host01.service systemctl restart SERVICE_ID systemctl restart ceph-499829b4-832f-11eb-8d6d-001a4a000635@mon.host01.service systemctl list-units "ceph*" cephadm shell ceph orch ls NAME RUNNING REFRESHED AGE PLACEMENT IMAGE NAME IMAGE ID alertmanager 1/1 4m ago 4M count:1 registry.redhat.io/openshift4/ose-prometheus-alertmanager:v4.5 b7bae610cd46 crash 3/3 4m ago 4M * registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 grafana 1/1 4m ago 4M count:1 registry.redhat.io/rhceph-alpha/rhceph-6-dashboard-rhel9:latest bd3d7748747b mgr 2/2 4m ago 4M count:2 registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 mon 2/2 4m ago 10w count:2 registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 nfs.foo 0/1 - - count:1 <unknown> <unknown> node-exporter 1/3 4m ago 4M * registry.redhat.io/openshift4/ose-prometheus-node-exporter:v4.5 mix osd.all-available-devices 5/5 4m ago 3M * registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 prometheus 1/1 4m ago 4M count:1 registry.redhat.io/openshift4/ose-prometheus:v4.6 bebb0ddef7f0 rgw.test_realm.test_zone 2/2 4m ago 3M count:2 registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 ceph orch start SERVICE_ID ceph orch start node-exporter ceph orch stop SERVICE_ID ceph orch stop node-exporter ceph orch restart SERVICE_ID ceph orch restart node-exporter journalctl -u ceph SERVICE_ID journalctl -u ceph-499829b4-832f-11eb-8d6d-001a4a000635@osd.8.service journalctl -fu SERVICE_ID journalctl -fu ceph-499829b4-832f-11eb-8d6d-001a4a000635@osd.8.service cephadm shell ceph -s ceph fs set FS_NAME max_mds 1 ceph fs fail FS_NAME ceph status ceph fs set FS_NAME joinable false ceph fs set cephfs max_mds 1 ceph fs fail cephfs ceph status ceph fs set cephfs joinable false ceph osd set noout ceph osd set norecover ceph osd set norebalance ceph osd set nobackfill ceph osd set nodown ceph osd set pause systemctl list-units --type target | grep ceph ceph-0b007564-ec48-11ee-b736-525400fd02f8.target loaded active active Ceph cluster 0b007564-ec48-11ee-b736-525400fd02f8 ceph.target loaded active active All Ceph clusters and services systemctl disable ceph-0b007564-ec48-11ee-b736-525400fd02f8.target Removed "/etc/systemd/system/multi-user.target.wants/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target". Removed "/etc/systemd/system/ceph.target.wants/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target". systemctl stop ceph-0b007564-ec48-11ee-b736-525400fd02f8.target shutdown Shutdown scheduled for Wed 2024-03-27 11:47:19 EDT, use 'shutdown -c' to cancel. systemctl enable ceph-0b007564-ec48-11ee-b736-525400fd02f8.target Created symlink /etc/systemd/system/multi-user.target.wants/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target -> /etc/systemd/system/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target. Created symlink /etc/systemd/system/ceph.target.wants/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target -> /etc/systemd/system/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target. systemctl start ceph-0b007564-ec48-11ee-b736-525400fd02f8.target ceph osd unset noout ceph osd unset norecover ceph osd unset norebalance ceph osd unset nobackfill ceph osd unset nodown ceph osd unset pause ceph fs set FS_NAME joinable true ceph fs set cephfs joinable true ceph -s cephadm shell ceph -s ceph fs set FS_NAME max_mds 1 ceph fs fail FS_NAME ceph status ceph fs set FS_NAME joinable false ceph mds fail FS_NAME : N ceph fs set cephfs max_mds 1 ceph fs fail cephfs ceph status ceph fs set cephfs joinable false ceph mds fail cephfs:1 ceph osd set noout ceph osd set norecover ceph osd set norebalance ceph osd set nobackfill ceph osd set nodown ceph osd set pause ceph orch ls --service-type mds ceph orch stop SERVICE-NAME ceph orch ls --service-type rgw ceph orch stop SERVICE-NAME ceph orch stop alertmanager ceph orch stop node-exporter ceph orch stop prometheus ceph orch stop grafana ceph orch stop crash ceph orch ps --daemon-type=osd ceph orch daemon stop osd.1 Scheduled to stop osd.1 on host 'host02' ceph orch ps --daemon-type mon systemctl list-units ceph-* | grep mon systemct stop SERVICE-NAME cephadm shell ceph orch ls ceph -s ceph osd unset noout ceph osd unset norecover ceph osd unset norebalance ceph osd unset nobackfill ceph osd unset nodown ceph osd unset pause ceph fs set FS_NAME joinable true ceph fs set cephfs joinable true ceph -s root@host01 ~]# cephadm shell ceph health HEALTH_OK ceph status root@host01 ~]# cephadm shell ceph -w cluster: id: 8c9b0072-67ca-11eb-af06-001a4a0002a0 health: HEALTH_OK services: mon: 2 daemons, quorum Ceph5-2,Ceph5-adm (age 3d) mgr: Ceph5-1.nqikfh(active, since 3w), standbys: Ceph5-adm.meckej osd: 5 osds: 5 up (since 2d), 5 in (since 8w) rgw: 2 daemons active (test_realm.test_zone.Ceph5-2.bfdwcn, test_realm.test_zone.Ceph5-adm.acndrh) data: pools: 11 pools, 273 pgs objects: 459 objects, 32 KiB usage: 2.6 GiB used, 72 GiB / 75 GiB avail pgs: 273 active+clean io: client: 170 B/s rd, 730 KiB/s wr, 0 op/s rd, 729 op/s wr 2021-06-02 15:45:21.655871 osd.0 [INF] 17.71 deep-scrub ok 2021-06-02 15:45:47.880608 osd.1 [INF] 1.0 scrub ok 2021-06-02 15:45:48.865375 osd.1 [INF] 1.3 scrub ok 2021-06-02 15:45:50.866479 osd.1 [INF] 1.4 scrub ok 2021-06-02 15:45:01.345821 mon.0 [INF] pgmap v41339: 952 pgs: 952 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail 2021-06-02 15:45:05.718640 mon.0 [INF] pgmap v41340: 952 pgs: 1 active+clean+scrubbing+deep, 951 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail 2021-06-02 15:45:53.997726 osd.1 [INF] 1.5 scrub ok 2021-06-02 15:45:06.734270 mon.0 [INF] pgmap v41341: 952 pgs: 1 active+clean+scrubbing+deep, 951 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail 2021-06-02 15:45:15.722456 mon.0 [INF] pgmap v41342: 952 pgs: 952 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail 2021-06-02 15:46:06.836430 osd.0 [INF] 17.75 deep-scrub ok 2021-06-02 15:45:55.720929 mon.0 [INF] pgmap v41343: 952 pgs: 1 active+clean+scrubbing+deep, 951 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 5 TiB 2.9 TiB 2.1 TiB 2.1 TiB 42.98 TOTAL 5 TiB 2.9 TiB 2.1 TiB 2.1 TiB 42.98 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL .mgr 1 1 5.3 MiB 3 16 MiB 0 629 GiB .rgw.root 2 32 1.3 KiB 4 48 KiB 0 629 GiB default.rgw.log 3 32 3.6 KiB 209 408 KiB 0 629 GiB default.rgw.control 4 32 0 B 8 0 B 0 629 GiB default.rgw.meta 5 32 1.7 KiB 10 96 KiB 0 629 GiB default.rgw.buckets.index 7 32 5.5 MiB 22 17 MiB 0 629 GiB default.rgw.buckets.data 8 32 807 KiB 3 2.4 MiB 0 629 GiB default.rgw.buckets.non-ec 9 32 1.0 MiB 1 3.1 MiB 0 629 GiB source-ecpool-86 11 32 1.2 TiB 391.13k 2.1 TiB 53.49 1.1 TiB ceph osd df ID CLASS WEIGHT REWEIGHT SIZE USE DATA OMAP META AVAIL %USE VAR PGS 3 hdd 0.90959 1.00000 931GiB 70.1GiB 69.1GiB 0B 1GiB 861GiB 7.53 2.93 66 4 hdd 0.90959 1.00000 931GiB 1.30GiB 308MiB 0B 1GiB 930GiB 0.14 0.05 59 0 hdd 0.90959 1.00000 931GiB 18.1GiB 17.1GiB 0B 1GiB 913GiB 1.94 0.76 57 MIN/MAX VAR: 0.02/2.98 STDDEV: 2.91 cephadm shell ceph status ceph -s ceph ceph> status cluster: id: 499829b4-832f-11eb-8d6d-001a4a000635 health: HEALTH_WARN 1 stray daemon(s) not managed by cephadm 1/3 mons down, quorum host03,host02 too many PGs per OSD (261 > max 250) services: mon: 3 daemons, quorum host03,host02 (age 3d), out of quorum: host01 mgr: host01.hdhzwn(active, since 9d), standbys: host05.eobuuv, host06.wquwpj osd: 12 osds: 11 up (since 2w), 11 in (since 5w) rgw: 2 daemons active (test_realm.test_zone.host04.hgbvnq, test_realm.test_zone.host05.yqqilm) rgw-nfs: 1 daemon active (nfs.foo.host06-rgw) data: pools: 8 pools, 960 pgs objects: 414 objects, 1.0 MiB usage: 5.7 GiB used, 214 GiB / 220 GiB avail pgs: 960 active+clean io: client: 41 KiB/s rd, 0 B/s wr, 41 op/s rd, 27 op/s wr ceph> health HEALTH_WARN 1 stray daemon(s) not managed by cephadm; 1/3 mons down, quorum host03,host02; too many PGs per OSD (261 > max 250) ceph> mon stat e3: 3 mons at {host01=[v2:10.74.255.0:3300/0,v1:10.74.255.0:6789/0],host02=[v2:10.74.249.253:3300/0,v1:10.74.249.253:6789/0],host03=[v2:10.74.251.164:3300/0,v1:10.74.251.164:6789/0]}, election epoch 6688, leader 1 host03, quorum 1,2 host03,host02 cephadm shell ceph mon stat ceph mon dump ceph quorum_status -f json-pretty { "election_epoch": 6686, "quorum": [ 0, 1, 2 ], "quorum_names": [ "host01", "host03", "host02" ], "quorum_leader_name": "host01", "quorum_age": 424884, "features": { "quorum_con": "4540138297136906239", "quorum_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ] }, "monmap": { "epoch": 3, "fsid": "499829b4-832f-11eb-8d6d-001a4a000635", "modified": "2021-03-15T04:51:38.621737Z", "created": "2021-03-12T12:35:16.911339Z", "min_mon_release": 16, "min_mon_release_name": "pacific", "election_strategy": 1, "disallowed_leaders: ": "", "stretch_mode": false, "features": { "persistent": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "optional": [] }, "mons": [ { "rank": 0, "name": "host01", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.255.0:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.255.0:6789", "nonce": 0 } ] }, "addr": "10.74.255.0:6789/0", "public_addr": "10.74.255.0:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 1, "name": "host03", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.251.164:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.251.164:6789", "nonce": 0 } ] }, "addr": "10.74.251.164:6789/0", "public_addr": "10.74.251.164:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 2, "name": "host02", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.249.253:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.249.253:6789", "nonce": 0 } ] }, "addr": "10.74.249.253:6789/0", "public_addr": "10.74.249.253:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" } ] } } Error 111: Connection Refused cephadm shell ceph daemon MONITOR_ID COMMAND ceph daemon mon.host01 help { "add_bootstrap_peer_hint": "add peer address as potential bootstrap peer for cluster bringup", "add_bootstrap_peer_hintv": "add peer address vector as potential bootstrap peer for cluster bringup", "compact": "cause compaction of monitor's leveldb/rocksdb storage", "config diff": "dump diff of current config and default config", "config diff get": "dump diff get <field>: dump diff of current and default config setting <field>", "config get": "config get <field>: get the config value", "config help": "get config setting schema and descriptions", "config set": "config set <field> <val> [<val> ...]: set a config variable", "config show": "dump current config settings", "config unset": "config unset <field>: unset a config variable", "connection scores dump": "show the scores used in connectivity-based elections", "connection scores reset": "reset the scores used in connectivity-based elections", "counter dump": "dump all labeled and non-labeled counters and their values", "counter schema": "dump all labeled and non-labeled counters schemas", "dump_historic_ops": "show recent ops", "dump_historic_slow_ops": "show recent slow ops", "dump_mempools": "get mempool stats", "get_command_descriptions": "list available commands", "git_version": "get git sha1", "heap": "show heap usage info (available only if compiled with tcmalloc)", "help": "list available commands", "injectargs": "inject configuration arguments into running daemon", "log dump": "dump recent log entries to log file", "log flush": "flush log entries to log file", "log reopen": "reopen log file", "mon_status": "report status of monitors", "ops": "show the ops currently in flight", "perf dump": "dump non-labeled counters and their values", "perf histogram dump": "dump perf histogram values", "perf histogram schema": "dump perf histogram schema", "perf reset": "perf reset <name>: perf reset all or one perfcounter name", "perf schema": "dump non-labeled counters schemas", "quorum enter": "force monitor back into quorum", "quorum exit": "force monitor out of the quorum", "sessions": "list existing sessions", "smart": "Query health metrics for underlying device", "sync_force": "force sync of and clear monitor store", "version": "get ceph version" } ceph daemon mon.host01 mon_status { "name": "host01", "rank": 0, "state": "leader", "election_epoch": 120, "quorum": [ 0, 1, 2 ], "quorum_age": 206358, "features": { "required_con": "2449958747317026820", "required_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "quorum_con": "4540138297136906239", "quorum_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ] }, "outside_quorum": [], "extra_probe_peers": [], "sync_provider": [], "monmap": { "epoch": 3, "fsid": "81a4597a-b711-11eb-8cb8-001a4a000740", "modified": "2021-05-18T05:50:17.782128Z", "created": "2021-05-17T13:13:13.383313Z", "min_mon_release": 16, "min_mon_release_name": "pacific", "election_strategy": 1, "disallowed_leaders: ": "", "stretch_mode": false, "features": { "persistent": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "optional": [] }, "mons": [ { "rank": 0, "name": "host01", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.249.41:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.249.41:6789", "nonce": 0 } ] }, "addr": "10.74.249.41:6789/0", "public_addr": "10.74.249.41:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 1, "name": "host02", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.249.55:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.249.55:6789", "nonce": 0 } ] }, "addr": "10.74.249.55:6789/0", "public_addr": "10.74.249.55:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 2, "name": "host03", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.249.49:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.249.49:6789", "nonce": 0 } ] }, "addr": "10.74.249.49:6789/0", "public_addr": "10.74.249.49:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" } ] }, "feature_map": { "mon": [ { "features": "0x3f01cfb9fffdffff", "release": "luminous", "num": 1 } ], "osd": [ { "features": "0x3f01cfb9fffdffff", "release": "luminous", "num": 3 } ] }, "stretch_mode": false } ceph daemon /var/run/ceph/ SOCKET_FILE COMMAND ceph daemon /var/run/ceph/ceph-osd.0.asok status { "cluster_fsid": "9029b252-1668-11ee-9399-001a4a000429", "osd_fsid": "1de9b064-b7a5-4c54-9395-02ccda637d21", "whoami": 0, "state": "active", "oldest_map": 1, "newest_map": 58, "num_pgs": 33 } ls /var/run/ceph ceph osd stat ceph osd dump eNNNN: x osds: y up, z in ceph osd tree id weight type name up/down reweight -1 3 pool default -3 3 rack mainrack -2 3 host osd-host 0 1 osd.0 up 1 1 1 osd.1 up 1 2 1 osd.2 up 1 systemctl start CEPH_OSD_SERVICE_ID systemctl start ceph-499829b4-832f-11eb-8d6d-001a4a000635@osd.6.service cephadm shell ceph pg dump ceph pg map PG_NUM ceph pg map 128 ceph pg stat vNNNNNN: x pgs: y active+clean; z bytes data, aa MB used, bb GB / cc GB avail 244 active+clean+snaptrim_wait 32 active+clean+snaptrim POOL_NUM . PG_ID 0.1f ceph pg dump ceph pg dump -o FILE_NAME --format=json ceph pg dump -o test --format=json ceph pg POOL_NUM . PG_ID query ceph pg 5.fe query { "snap_trimq": "[]", "snap_trimq_len": 0, "state": "active+clean", "epoch": 2449, "up": [ 3, 8, 10 ], "acting": [ 3, 8, 10 ], "acting_recovery_backfill": [ "3", "8", "10" ], "info": { "pgid": "5.ff", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 114, "epoch_pool_created": 82, "last_epoch_started": 2402, "last_interval_started": 2401, "last_epoch_clean": 2402, "last_interval_clean": 2401, "last_epoch_split": 114, "last_epoch_marked_full": 0, "same_up_since": 2401, "same_interval_since": 2401, "same_primary_since": 2086, "last_scrub": "0'0", "last_scrub_stamp": "2021-06-17T01:32:03.763988+0000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2021-06-17T01:32:03.763988+0000", "last_clean_scrub_stamp": "2021-06-17T01:32:03.763988+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": "2989", "reported_epoch": "2449", "state": "active+clean", "last_fresh": "2021-06-18T05:16:59.401080+0000", "last_change": "2021-06-17T01:32:03.764162+0000", "last_active": "2021-06-18T05:16:59.401080+0000", . pg 1.5: up=acting: [0,1,2] ADD_OSD_3 pg 1.5: up: [0,3,1] acting: [0,1,2] pg 1.5: up=acting: [0,3,1] ceph pg dump_stuck {inactive|unclean|stale|undersized|degraded [inactive|unclean|stale|undersized|degraded...]} {<int>} ceph pg dump_stuck stale OK ceph osd map POOL_NAME OBJECT_NAME ceph osd map mypool myobject ceph mon enable_stretch_mode host05 stretch_rule datacenter Error EINVAL: the 2 datacenter instances in the cluster have differing weights 25947 and 15728 but stretch mode currently requires they be the same! service_type: host addr: host01 hostname: host01 location: root: default datacenter: DC1 labels: - osd - mon - mgr --- service_type: host addr: host02 hostname: host02 location: datacenter: DC1 labels: - osd - mon --- service_type: host addr: host03 hostname: host03 location: datacenter: DC1 labels: - osd - mds - rgw --- service_type: host addr: host04 hostname: host04 location: root: default datacenter: DC2 labels: - osd - mon - mgr --- service_type: host addr: host05 hostname: host05 location: datacenter: DC2 labels: - osd - mon --- service_type: host addr: host06 hostname: host06 location: datacenter: DC2 labels: - osd - mds - rgw --- service_type: host addr: host07 hostname: host07 labels: - mon --- service_type: mon placement: label: "mon" --- service_id: cephfs placement: label: "mds" --- service_type: mgr service_name: mgr placement: label: "mgr" --- service_type: osd service_id: all-available-devices service_name: osd.all-available-devices placement: label: "osd" spec: data_devices: all: true --- service_type: rgw service_id: objectgw service_name: rgw.objectgw placement: count: 2 label: "rgw" spec: rgw_frontend_port: 8080 cephadm bootstrap --apply-spec CONFIGURATION_FILE_NAME --mon-ip MONITOR_IP_ADDRESS --ssh-private-key PRIVATE_KEY --ssh-public-key PUBLIC_KEY --registry-url REGISTRY_URL --registry-username USER_NAME --registry-password PASSWORD cephadm bootstrap --apply-spec initial-config.yaml --mon-ip 10.10.128.68 --ssh-private-key /home/ceph/.ssh/id_rsa --ssh-public-key /home/ceph/.ssh/id_rsa.pub --registry-url registry.redhat.io --registry-username myuser1 --registry-password mypassword1 ceph osd crush add-bucket BUCKET_NAME BUCKET_TYPE ceph osd crush add-bucket DC1 datacenter ceph osd crush add-bucket DC2 datacenter ceph osd crush move BUCKET_NAME root=default ceph osd crush move DC1 root=default ceph osd crush move DC2 root=default ceph osd crush move HOST datacenter= DATACENTER ceph osd crush move host01 datacenter=DC1 ceph mon set_location HOST datacenter= DATACENTER ceph mon set_location host01 datacenter=DC1 ceph mon set_location host02 datacenter=DC1 ceph mon set_location host04 datacenter=DC2 ceph mon set_location host05 datacenter=DC2 ceph mon set_location host07 datacenter=DC3 ceph osd getcrushmap > COMPILED_CRUSHMAP_FILENAME crushtool -d COMPILED_CRUSHMAP_FILENAME -o DECOMPILED_CRUSHMAP_FILENAME ceph osd getcrushmap > crush.map.bin crushtool -d crush.map.bin -o crush.map.txt rule stretch_rule { id 1 1 type replicated min_size 1 max_size 10 step take DC1 2 step chooseleaf firstn 2 type host step emit step take DC2 3 step chooseleaf firstn 2 type host step emit } rule stretch_rule { id 1 type replicated min_size 1 max_size 10 step take default step choose firstn 0 type datacenter step chooseleaf firstn 2 type host step emit } crushtool -c DECOMPILED_CRUSHMAP_FILENAME -o COMPILED_CRUSHMAP_FILENAME ceph osd setcrushmap -i COMPILED_CRUSHMAP_FILENAME crushtool -c crush.map.txt -o crush2.map.bin ceph osd setcrushmap -i crush2.map.bin ceph mon set election_strategy connectivity ceph mon set_location HOST datacenter= DATACENTER ceph mon enable_stretch_mode HOST stretch_rule datacenter ceph mon set_location host07 datacenter=DC3 ceph mon enable_stretch_mode host07 stretch_rule datacenter Error EINVAL: there are 3 datacenters in the cluster but stretch mode currently only works with 2! ceph osd dump epoch 361 fsid 1234ab78-1234-11ed-b1b1-de456ef0a89d created 2023-01-16T05:47:28.482717+0000 modified 2023-01-17T17:36:50.066183+0000 flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit crush_version 31 full_ratio 0.95 backfillfull_ratio 0.92 nearfull_ratio 0.85 require_min_compat_client luminous min_compat_client luminous require_osd_release quincy stretch_mode_enabled true stretch_bucket_count 2 degraded_stretch_mode 0 recovering_stretch_mode 0 stretch_mode_bucket 8 ceph mon dump epoch 19 fsid 1234ab78-1234-11ed-b1b1-de456ef0a89d last_changed 2023-01-17T04:12:05.709475+0000 created 2023-01-16T05:47:25.631684+0000 min_mon_release 16 (pacific) election_strategy: 3 stretch_mode_enabled 1 tiebreaker_mon host07 disallowed_leaders host07 0: [v2:132.224.169.63:3300/0,v1:132.224.169.63:6789/0] mon.host07; crush_location {datacenter=DC3} 1: [v2:220.141.179.34:3300/0,v1:220.141.179.34:6789/0] mon.host04; crush_location {datacenter=DC2} 2: [v2:40.90.220.224:3300/0,v1:40.90.220.224:6789/0] mon.host01; crush_location {datacenter=DC1} 3: [v2:60.140.141.144:3300/0,v1:60.140.141.144:6789/0] mon.host02; crush_location {datacenter=DC1} 4: [v2:186.184.61.92:3300/0,v1:186.184.61.92:6789/0] mon.host05; crush_location {datacenter=DC2} dumped monmap epoch 19 ceph orch device ls [--hostname= HOST_1 HOST_2 ] [--wide] [--refresh] ceph orch device ls ceph orch daemon add osd HOST : DEVICE_PATH ceph orch daemon add osd host03:/dev/sdb ceph orch apply osd --all-available-devices ceph osd crush move HOST datacenter= DATACENTER ceph osd crush move host03 datacenter=DC1 ceph osd crush move host06 datacenter=DC2 ceph osd set FLAG ceph osd set noout ceph osd unset FLAG ceph osd unset noout DAEMON_TYPE 'allow CAPABILITY ' [ DAEMON_TYPE 'allow CAPABILITY '] mon 'allow rwx` mon 'allow profile osd' osd 'allow CAPABILITY ' [pool= POOL_NAME ] [namespace= NAMESPACE_NAME ] ceph auth list installed auth entries: osd.10 key: AQBW7U5gqOsEExAAg/CxSwZ/gSh8iOsDV3iQOA== caps: [mgr] allow profile osd caps: [mon] allow profile osd caps: [osd] allow * osd.11 key: AQBX7U5gtj/JIhAAPsLBNG+SfC2eMVEFkl3vfA== caps: [mgr] allow profile osd caps: [mon] allow profile osd caps: [osd] allow * osd.9 key: AQBV7U5g1XDULhAAKo2tw6ZhH1jki5aVui2v7g== caps: [mgr] allow profile osd caps: [mon] allow profile osd caps: [osd] allow * client.admin key: AQADYEtgFfD3ExAAwH+C1qO7MSLE4TWRfD2g6g== caps: [mds] allow * caps: [mgr] allow * caps: [mon] allow * caps: [osd] allow * client.bootstrap-mds key: AQAHYEtgpbkANBAANqoFlvzEXFwD8oB0w3TF4Q== caps: [mon] allow profile bootstrap-mds client.bootstrap-mgr key: AQAHYEtg3dcANBAAVQf6brq3sxTSrCrPe0pKVQ== caps: [mon] allow profile bootstrap-mgr client.bootstrap-osd key: AQAHYEtgD/QANBAATS9DuP3DbxEl86MTyKEmdw== caps: [mon] allow profile bootstrap-osd client.bootstrap-rbd key: AQAHYEtgjxEBNBAANho25V9tWNNvIKnHknW59A== caps: [mon] allow profile bootstrap-rbd client.bootstrap-rbd-mirror key: AQAHYEtgdE8BNBAAr6rLYxZci0b2hoIgH9GXYw== caps: [mon] allow profile bootstrap-rbd-mirror client.bootstrap-rgw key: AQAHYEtgwGkBNBAAuRzI4WSrnowBhZxr2XtTFg== caps: [mon] allow profile bootstrap-rgw client.crash.host04 key: AQCQYEtgz8lGGhAAy5bJS8VH9fMdxuAZ3CqX5Q== caps: [mgr] profile crash caps: [mon] profile crash client.crash.host02 key: AQDuYUtgqgfdOhAAsyX+Mo35M+HFpURGad7nJA== caps: [mgr] profile crash caps: [mon] profile crash client.crash.host03 key: AQB98E5g5jHZAxAAklWSvmDsh2JaL5G7FvMrrA== caps: [mgr] profile crash caps: [mon] profile crash client.nfs.foo.host03 key: AQCgTk9gm+HvMxAAHbjG+XpdwL6prM/uMcdPdQ== caps: [mon] allow r caps: [osd] allow rw pool=nfs-ganesha namespace=foo client.nfs.foo.host03-rgw key: AQCgTk9g8sJQNhAAPykcoYUuPc7IjubaFx09HQ== caps: [mon] allow r caps: [osd] allow rwx tag rgw *=* client.rgw.test_realm.test_zone.host01.hgbvnq key: AQD5RE9gAQKdCRAAJzxDwD/dJObbInp9J95sXw== caps: [mgr] allow rw caps: [mon] allow * caps: [osd] allow rwx tag rgw *=* client.rgw.test_realm.test_zone.host02.yqqilm key: AQD0RE9gkxA4ExAAFXp3pLJWdIhsyTe2ZR6Ilw== caps: [mgr] allow rw caps: [mon] allow * caps: [osd] allow rwx tag rgw *=* mgr.host01.hdhzwn key: AQAEYEtg3lhIBxAAmHodoIpdvnxK0llWF80ltQ== caps: [mds] allow * caps: [mon] profile mgr caps: [osd] allow * mgr.host02.eobuuv key: AQAn6U5gzUuiABAA2Fed+jPM1xwb4XDYtrQxaQ== caps: [mds] allow * caps: [mon] profile mgr caps: [osd] allow * mgr.host03.wquwpj key: AQAd6U5gIzWsLBAAbOKUKZlUcAVe9kBLfajMKw== caps: [mds] allow * caps: [mon] profile mgr caps: [osd] allow * ceph auth export TYPE . ID ceph auth export mgr.host02.eobuuv ceph auth export TYPE . ID -o FILE_NAME ceph auth export osd.9 -o filename export auth(key=AQBV7U5g1XDULhAAKo2tw6ZhH1jki5aVui2v7g==) ceph auth add client.john mon 'allow r' osd 'allow rw pool=mypool' ceph auth get-or-create client.paul mon 'allow r' osd 'allow rw pool=mypool' ceph auth get-or-create client.george mon 'allow r' osd 'allow rw pool=mypool' -o george.keyring ceph auth get-or-create-key client.ringo mon 'allow r' osd 'allow rw pool=mypool' -o ringo.key ceph auth caps USERTYPE . USERID DAEMON 'allow [r|w|x|*|...] [pool= POOL_NAME ] [namespace= NAMESPACE_NAME ]' ceph auth caps client.john mon 'allow r' osd 'allow rw pool=mypool' ceph auth caps client.paul mon 'allow rw' osd 'allow rwx pool=mypool' ceph auth caps client.brian-manager mon 'allow *' osd 'allow *' ceph auth caps client.ringo mon ' ' osd ' ' ceph auth del TYPE . ID ceph auth del osd.6 ceph auth print-key TYPE . ID ceph auth print-key osd.6 AQBQ7U5gAry3JRAA3NoPrqBBThpFMcRL6Sr+5w==[ceph: root@host01 /]# ceph-volume lvm ceph auth get client. ID -o ceph.client. ID .keyring ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring ceph-volume lvm prepare --bluestore --data VOLUME_GROUP / LOGICAL_VOLUME ceph-volume lvm prepare --bluestore --data example_vg/data_lv ceph-volume lvm prepare --bluestore --block.db BLOCK_DB_DEVICE --block.wal BLOCK_WAL_DEVICE --data DATA_DEVICE ceph-volume lvm prepare --bluestore --block.db /dev/sda --block.wal /dev/sdb --data /dev/sdc ceph-volume lvm prepare --bluestore --dmcrypt --data VOLUME_GROUP / LOGICAL_VOLUME ceph-volume lvm prepare --bluestore --dmcrypt --data example_vg/data_lv 1 devices have fault light turned on ceph-volume lvm list ====== osd.6 ======= [block] /dev/ceph-83909f70-95e9-4273-880e-5851612cbe53/osd-block-7ce687d9-07e7-4f8f-a34e-d1b0efb89920 block device /dev/ceph-83909f70-95e9-4273-880e-5851612cbe53/osd-block-7ce687d9-07e7-4f8f-a34e-d1b0efb89920 block uuid 4d7gzX-Nzxp-UUG0-bNxQ-Jacr-l0mP-IPD8cX cephx lockbox secret cluster fsid 1ca9f6a8-d036-11ec-8263-fa163ee967ad cluster name ceph crush device class None encrypted 0 osd fsid 7ce687d9-07e7-4f8f-a34e-d1b0efb89920 osd id 6 osdspec affinity all-available-devices type block vdo 0 devices /dev/vdc ceph device ls-lights { "fault": [ "SEAGATE_ST12000NM002G_ZL2KTGCK0000C149" ], "ident": [] } ceph device light off DEVICE_NAME FAULT/INDENT --force ceph device light off SEAGATE_ST12000NM002G_ZL2KTGCK0000C149 fault --force ceph-volume lvm list ceph-volume lvm activate --bluestore OSD_ID OSD_FSID ceph-volume lvm activate --bluestore 10 7ce687d9-07e7-4f8f-a34e-d1b0efb89920 ceph-volume lvm activate --all ceph-volume lvm trigger SYSTEMD_DATA ceph-volume lvm trigger 10 7ce687d9-07e7-4f8f-a34e-d1b0efb89920 ceph-volume lvm list ceph-volume lvm deactivate OSD_ID ceph-volume lvm deactivate 16 ceph-volume lvm create --bluestore --data VOLUME_GROUP / LOGICAL_VOLUME ceph-volume lvm create --bluestore --data example_vg/data_lv cephadm shell ceph orch daemon stop osd.1 cephadm shell --mount /var/lib/ceph/72436d46-ca06-11ec-9809-ac1f6b5635ee/osd.1:/var/lib/ceph/osd/ceph-1 ceph-volume lvm new-db --osd-id OSD_ID --osd-fsid OSD_FSID --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME ceph-volume lvm new-db --osd-id 1 --osd-fsid 7ce687d9-07e7-4f8f-a34e-d1b0efb89921 --target vgname/new_db ceph-volume lvm new-wal --osd-id 1 --osd-fsid 7ce687d9-07e7-4f8f-a34e-d1b0efb89921 --target vgname/new_wal ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from data --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from data --target vgname/db ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from data --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from data --target vgname/new_db ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from db --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from db --target vgname/new_db ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from data db --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from data db --target vgname/new_db ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from data db wal --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from data db --target vgname/new_db ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from db wal --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from db wal --target vgname/data ceph-volume lvm list ====== osd.3 ======= [db] /dev/db-test/db1 block device /dev/test/lv1 block uuid N5zoix-FePe-uExe-UngY-D9YG-BMs0-1tTDyB cephx lockbox secret cluster fsid 1a6112da-ed05-11ee-bacd-525400565cda cluster name ceph crush device class db device /dev/db-test/db1 db uuid 1TUaDY-3mEt-fReP-cyB2-JyZ1-oUPa-hKPfo6 encrypted 0 osd fsid 94ff742c-7bfd-4fb5-8dc4-843d10ac6731 osd id 3 osdspec affinity None type db vdo 0 devices /dev/vdh [block] /dev/test/lv1 block device /dev/test/lv1 block uuid N5zoix-FePe-uExe-UngY-D9YG-BMs0-1tTDyB cephx lockbox secret cluster fsid 1a6112da-ed05-11ee-bacd-525400565cda cluster name ceph crush device class db device /dev/db-test/db1 db uuid 1TUaDY-3mEt-fReP-cyB2-JyZ1-oUPa-hKPfo6 encrypted 0 osd fsid 94ff742c-7bfd-4fb5-8dc4-843d10ac6731 osd id 3 osdspec affinity None type block vdo 0 devices /dev/vdg vgs VG #PV #LV #SN Attr VSize VFree db-test 1 1 0 wz--n- <200.00g <160.00g test 1 1 0 wz--n- <200.00g <170.00g systemctl stop host01a6112da-ed05-11ee-bacd-525400565cda@osd.3.service lvresize -l 100%FREE /dev/db-test/db1 Size of logical volume db-test/db1 changed from 40.00 GiB (10240 extents) to <160.00 GiB (40959 extents). Logical volume db-test/db1 successfully resized. cephadm shell -m /var/lib/ceph/ CLUSTER_FSID /osd. OSD_ID :/var/lib/ceph/osd/ceph- OSD_ID :z cephadm shell -m /var/lib/ceph/1a6112da-ed05-11ee-bacd-525400565cda/osd.3:/var/lib/ceph/osd/ceph-3:z ceph-bluestore-tool show-label --path OSD_DIRECTORY_PATH ceph-bluestore-tool show-label --path /var/lib/ceph/osd/ceph-3/ inferring bluefs devices from bluestore path { "/var/lib/ceph/osd/ceph-3/block": { "osd_uuid": "94ff742c-7bfd-4fb5-8dc4-843d10ac6731", "size": 32212254720, "btime": "2024-04-03T08:34:12.742848+0000", "description": "main", "bfm_blocks": "7864320", "bfm_blocks_per_key": "128", "bfm_bytes_per_block": "4096", "bfm_size": "32212254720", "bluefs": "1", "ceph_fsid": "1a6112da-ed05-11ee-bacd-525400565cda", "ceph_version_when_created": "ceph version 19.0.0-2493-gd82c9aa1 (d82c9aa17f09785fe698d262f9601d87bb79f962) squid (dev)", "created_at": "2024-04-03T08:34:15.637253Z", "elastic_shared_blobs": "1", "kv_backend": "rocksdb", "magic": "ceph osd volume v026", "mkfs_done": "yes", "osd_key": "AQCEFA1m9xuwABAAwKEHkASVbgB1GVt5jYC2Sg==", "osdspec_affinity": "None", "ready": "ready", "require_osd_release": "19", "whoami": "3" }, "/var/lib/ceph/osd/ceph-3/block.db": { "osd_uuid": "94ff742c-7bfd-4fb5-8dc4-843d10ac6731", "size": 40794497536, "btime": "2024-04-03T08:34:12.748816+0000", "description": "bluefs db" } } ceph-bluestore-tool bluefs-bdev-expand --path OSD_DIRECTORY_PATH ceph-bluestore-tool bluefs-bdev-expand --path /var/lib/ceph/osd/ceph-3/ inferring bluefs devices from bluestore path 1 : device size 0x27ffbfe000 : using 0x2300000(35 MiB) 2 : device size 0x780000000 : using 0x52000(328 KiB) Expanding DB/WAL 1 : expanding to 0x171794497536 1 : size label updated to 171794497536 ceph-bluestore-tool show-label --path OSD_DIRECTORY_PATH ceph-bluestore-tool show-label --path /var/lib/ceph/osd/ceph-3/ inferring bluefs devices from bluestore path { "/var/lib/ceph/osd/ceph-3/block": { "osd_uuid": "94ff742c-7bfd-4fb5-8dc4-843d10ac6731", "size": 32212254720, "btime": "2024-04-03T08:34:12.742848+0000", "description": "main", "bfm_blocks": "7864320", "bfm_blocks_per_key": "128", "bfm_bytes_per_block": "4096", "bfm_size": "32212254720", "bluefs": "1", "ceph_fsid": "1a6112da-ed05-11ee-bacd-525400565cda", "ceph_version_when_created": "ceph version 19.0.0-2493-gd82c9aa1 (d82c9aa17f09785fe698d262f9601d87bb79f962) squid (dev)", "created_at": "2024-04-03T08:34:15.637253Z", "elastic_shared_blobs": "1", "kv_backend": "rocksdb", "magic": "ceph osd volume v026", "mkfs_done": "yes", "osd_key": "AQCEFA1m9xuwABAAwKEHkASVbgB1GVt5jYC2Sg==", "osdspec_affinity": "None", "ready": "ready", "require_osd_release": "19", "whoami": "3" }, "/var/lib/ceph/osd/ceph-3/block.db": { "osd_uuid": "94ff742c-7bfd-4fb5-8dc4-843d10ac6731", "size": 171794497536, "btime": "2024-04-03T08:34:12.748816+0000", "description": "bluefs db" } } systemctl start host01a6112da-ed05-11ee-bacd-525400565cda@osd.3.service osd.3 host01 running (15s) 0s ago 13m 46.9M 4096M 19.0.0-2493-gd82c9aa1 3714003597ec 02150b3b6877 ceph-volume lvm batch --bluestore PATH_TO_DEVICE [ PATH_TO_DEVICE ] ceph-volume lvm batch --bluestore /dev/sda /dev/sdb /dev/nvme0n1 ceph-volume lvm zap VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME [--destroy] ceph-volume lvm zap osd-vg/data-lv ceph-volume lvm zap DEVICE_PATH_PARTITION [--destroy] ceph-volume lvm zap /dev/sdc1 ceph-volume lvm zap DEVICE_PATH --destroy ceph-volume lvm zap /dev/sdc --destroy ceph-volume lvm zap --destroy --osd-id OSD_ID ceph-volume lvm zap --destroy --osd-id 16 ceph-volume lvm zap --destroy --osd-fsid OSD_FSID ceph-volume lvm zap --destroy --osd-fsid 65d7b6b1-e41a-4a3c-b363-83ade63cb32b echo 3 | sudo tee /proc/sys/vm/drop_caches && sudo sync ceph osd pool create testbench 100 100 rados bench -p testbench 10 write --no-cleanup Maintaining 16 concurrent writes of 4194304 bytes for up to 10 seconds or 0 objects Object prefix: benchmark_data_cephn1.home.network_10510 sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 0 0 0 0 0 0 - 0 1 16 16 0 0 0 - 0 2 16 16 0 0 0 - 0 3 16 16 0 0 0 - 0 4 16 17 1 0.998879 1 3.19824 3.19824 5 16 18 2 1.59849 4 4.56163 3.87993 6 16 18 2 1.33222 0 - 3.87993 7 16 19 3 1.71239 2 6.90712 4.889 8 16 25 9 4.49551 24 7.75362 6.71216 9 16 25 9 3.99636 0 - 6.71216 10 16 27 11 4.39632 4 9.65085 7.18999 11 16 27 11 3.99685 0 - 7.18999 12 16 27 11 3.66397 0 - 7.18999 13 16 28 12 3.68975 1.33333 12.8124 7.65853 14 16 28 12 3.42617 0 - 7.65853 15 16 28 12 3.19785 0 - 7.65853 16 11 28 17 4.24726 6.66667 12.5302 9.27548 17 11 28 17 3.99751 0 - 9.27548 18 11 28 17 3.77546 0 - 9.27548 19 11 28 17 3.57683 0 - 9.27548 Total time run: 19.505620 Total writes made: 28 Write size: 4194304 Bandwidth (MB/sec): 5.742 Stddev Bandwidth: 5.4617 Max bandwidth (MB/sec): 24 Min bandwidth (MB/sec): 0 Average Latency: 10.4064 Stddev Latency: 3.80038 Max latency: 19.503 Min latency: 3.19824 rados bench -p testbench 10 seq sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 0 0 0 0 0 0 - 0 Total time run: 0.804869 Total reads made: 28 Read size: 4194304 Bandwidth (MB/sec): 139.153 Average Latency: 0.420841 Max latency: 0.706133 Min latency: 0.0816332 rados bench -p testbench 10 rand sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 0 0 0 0 0 0 - 0 1 16 46 30 119.801 120 0.440184 0.388125 2 16 81 65 129.408 140 0.577359 0.417461 3 16 120 104 138.175 156 0.597435 0.409318 4 15 157 142 141.485 152 0.683111 0.419964 5 16 206 190 151.553 192 0.310578 0.408343 6 16 253 237 157.608 188 0.0745175 0.387207 7 16 287 271 154.412 136 0.792774 0.39043 8 16 325 309 154.044 152 0.314254 0.39876 9 16 362 346 153.245 148 0.355576 0.406032 10 16 405 389 155.092 172 0.64734 0.398372 Total time run: 10.302229 Total reads made: 405 Read size: 4194304 Bandwidth (MB/sec): 157.248 Average Latency: 0.405976 Max latency: 1.00869 Min latency: 0.0378431 rados bench -p testbench 10 write -t 4 --run-name client1 Maintaining 4 concurrent writes of 4194304 bytes for up to 10 seconds or 0 objects Object prefix: benchmark_data_node1_12631 sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 0 0 0 0 0 0 - 0 1 4 4 0 0 0 - 0 2 4 6 2 3.99099 4 1.94755 1.93361 3 4 8 4 5.32498 8 2.978 2.44034 4 4 8 4 3.99504 0 - 2.44034 5 4 10 6 4.79504 4 2.92419 2.4629 6 3 10 7 4.64471 4 3.02498 2.5432 7 4 12 8 4.55287 4 3.12204 2.61555 8 4 14 10 4.9821 8 2.55901 2.68396 9 4 16 12 5.31621 8 2.68769 2.68081 10 4 17 13 5.18488 4 2.11937 2.63763 11 4 17 13 4.71431 0 - 2.63763 12 4 18 14 4.65486 2 2.4836 2.62662 13 4 18 14 4.29757 0 - 2.62662 Total time run: 13.123548 Total writes made: 18 Write size: 4194304 Bandwidth (MB/sec): 5.486 Stddev Bandwidth: 3.0991 Max bandwidth (MB/sec): 8 Min bandwidth (MB/sec): 0 Average Latency: 2.91578 Stddev Latency: 0.956993 Max latency: 5.72685 Min latency: 1.91967 rados -p testbench cleanup rbd bench --io-type write image01 --pool=testbench bench-write io_size 4096 io_threads 16 bytes 1073741824 pattern seq SEC OPS OPS/SEC BYTES/SEC 2 11127 5479.59 22444382.79 3 11692 3901.91 15982220.33 4 12372 2953.34 12096895.42 5 12580 2300.05 9421008.60 6 13141 2101.80 8608975.15 7 13195 356.07 1458459.94 8 13820 390.35 1598876.60 9 14124 325.46 1333066.62 .. cd /mnt/ceph-block-device cd /mnt/ceph-file-system fio --name=randwrite --rw=randwrite --direct=1 --ioengine=libaio --bs=4k --iodepth=32 --size=5G --runtime=60 --group_reporting=1 fio --name=randwrite --rw=randwrite --direct=1 --ioengine=libaio --bs=8k --iodepth=32 --size=5G --runtime=60 --group_reporting=1 time s3cmd put PATH_OF_SOURCE_FILE PATH_OF_DESTINATION_FILE time s3cmd put /path-to-local-file s3://bucket-name/remote/file time s3cmd get PATH_OF_DESTINATION_FILE DESTINATION_PATH time s3cmd get s3://bucket-name/remote/file /path-to-local-destination time s3cmd ls s3:// BUCKET_NAME time s3cmd ls s3://bucket-name ceph daemon DAEMON_NAME perf schema ceph daemon mon.host01 perf schema ceph daemon osd.11 perf schema ceph config set client.rgw.rgw.1.host05 rgw_user_counters_cache true ceph config set client.rgw.rgw.1.host05 rgw_bucket_counters_cache true ceph orch restart rgw.rgw.1 s3cmd --configure s3cmd mb s3:// NAME_OF_THE_BUCKET_FOR_S3 s3cmd mb s3://bucket Bucket 's3://bucket/' created s3cmd put FILE_NAME s3:// NAME_OF_THE_BUCKET_ON_S3 s3cmd put test.txt s3://bucket upload: 'test.txt' -> 's3://bucket/test.txt' [1 of 1] 21 of 21 100% in 1s 16.75 B/s done s3cmd ls s3://bucket config dump ceph daemon DAEMON_ID counter dump http:// RGW_IP_ADDRESS : CEPH-EXPORTER_PORT / HELP ceph_rgw_op_per_bucket_put_obj_ops Puts TYPE ceph_rgw_op_per_bucket_put_obj_ops counter ceph_rgw_op_per_bucket_put_obj_ops{bucket="test-bkt1",instance_id="ceph-ck-perf-ej61qj-node5"} 10 HELP ceph_rgw_op_per_user_put_obj_ops Puts TYPE ceph_rgw_op_per_user_put_obj_ops counter ceph_rgw_op_per_user_put_obj_ops{instance_id="ceph-ck-perf-ej61qj-node5",user="ckulal"} 10 http:// RGW_IP_ADDRESS : PROMETHEUS_PORT / https://10.0.210.100:9283/ ceph daemon DAEMON_NAME perf dump ceph daemon mon.host01 perf dump ceph daemon osd.11 perf dump cephadm shell ceph config set osd. OSD_ID osd_mclock_profile VALUE ceph config set osd.0 osd_mclock_profile high_recovery_ops ceph config set osd osd_mclock_profile VALUE cephadm shell ceph config set osd. OSD_ID osd_mclock_profile custom ceph config set osd.0 osd_mclock_profile custom ceph config set osd osd_mclock_profile custom ceph config set osd. OSD_ID MCLOCK_CONFIGURATION_OPTION VALUE ceph config set osd.0 osd_mclock_scheduler_client_res 0.5 cephadm shell ceph config set osd osd_mclock_profile MCLOCK_PROFILE ceph config set osd osd_mclock_profile high_client_ops ceph config dump ceph config rm osd MCLOCK_CONFIGURATION_OPTION ceph config rm osd osd_mclock_scheduler_client_res ceph config show osd. OSD_ID ceph config show osd.0 cephadm shell ceph tell osd. OSD_ID injectargs '-- MCLOCK_CONFIGURATION_OPTION = VALUE ' ceph tell osd.0 injectargs '--osd_mclock_profile=high_recovery_ops' ceph daemon osd. OSD_ID config set MCLOCK_CONFIGURATION_OPTION VALUE ceph daemon osd.0 config set osd_mclock_profile high_recovery_ops cephadm shell ceph config set osd osd_mclock_override_recovery_settings true ceph config set osd OPTION VALUE ceph config set osd osd_max_backfills_ 5 ceph config show osd. OSD_ID_ | grep OPTION ceph config show osd.0 | grep osd_max_backfills ceph config set osd osd_mclock_override_recovery_settings false ceph config show osd.0 osd_mclock_iops_capacity_threshold_hdd 500.000000 ceph config show osd.0 osd_mclock_iops_capacity_threshold_ssd 80000.000000 ceph config show osd. N osd_mclock_max_capacity_iops_[hdd, ssd] ceph config show osd.0 osd_mclock_max_capacity_iops_ssd 3403 Sep 11 11:52:50 dell-r640-039.dsal.lab.eng.rdu2.redhat.com ceph-osd[70342]: log_channel(cluster) log [WRN] : OSD bench result of 49691.213005 IOPS exceeded the threshold limit of 500.000000 IOPS for osd.27. IOPS capacity is unchanged at 315.000000 IOPS. The recommendation is to establish the osd's IOPS capacity using other benchmark tools (e.g. Fio) and then override osd_mclock_max_capacity_iops_[hdd|ssd]. cephadm shell ceph config show osd. OSD_ID osd_mclock_max_capacity_iops_[hdd, ssd] ceph config show osd.0 osd_mclock_max_capacity_iops_ssd 21500.000000 cephadm shell ceph tell osd. OSD_ID bench [ TOTAL_BYTES ] [ BYTES_PER_WRITE ] [ OBJ_SIZE ] [ NUM_OBJS ] ceph tell osd.0 bench 12288000 4096 4194304 100 { "bytes_written": 12288000, "blocksize": 4096, "elapsed_sec": 1.3718913019999999, "bytes_per_sec": 8956977.8466311768, "iops": 2186.7621695876896 } ceph tell osd. OSD_ID cache drop ceph tell osd.0 cache drop cephadm shell ceph tell osd. OSD_ID bench 12288000 4096 4194304 100 ceph tell osd.0 bench 12288000 4096 4194304 100 { "bytes_written": 12288000, "blocksize": 4096, "elapsed_sec": 1.3718913019999999, "bytes_per_sec": 8956977.8466311768, "iops": 2186.7621695876896 1 } ceph config set osd. OSD_ID bluestore_throttle_bytes 32768 ceph config set osd. OSD_ID bluestore_throttle_deferred_bytes 32768 ceph config set osd.0 bluestore_throttle_bytes 32768 ceph config set osd.0 bluestore_throttle_deferred_bytes 32768 ceph tell osd.0 bench 12288000 4096 4194304 100 cephadm shell ceph config set osd. OSD_ID osd_mclock_max_capacity_iops_[hdd,ssd] VALUE ceph config set osd.0 osd_mclock_max_capacity_iops_hdd 350 ceph config set osd. OSD_ID bluestore_min_alloc_size_DEVICE_NAME_ VALUE ceph config set osd.4 bluestore_min_alloc_size_hdd 8192 systemctl restart SERVICE_ID systemctl restart ceph-499829b4-832f-11eb-8d6d-001a4a000635@osd.4.service ceph daemon osd. OSD_ID config get bluestore_min_alloc_size__DEVICE_ ceph daemon osd.4 config get bluestore_min_alloc_size_hdd ceph daemon osd.4 config get bluestore_min_alloc_size { "bluestore_min_alloc_size": "8192" } cd /usr/share/cephadm-ansible ansible-playbook -i hosts rocksdb-resharding.yml -e osd_id= OSD_ID -e admin_node= HOST_NAME ansible-playbook -i hosts rocksdb-resharding.yml -e osd_id=7 -e admin_node=host03 ............ TASK [stop the osd] *********************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:18 +0000 (0:00:00.037) 0:00:03.864 **** changed: [localhost -> host03] TASK [set_fact ceph_cmd] ****************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:32 +0000 (0:00:14.128) 0:00:17.992 **** ok: [localhost -> host03] TASK [check fs consistency with fsck before resharding] *********************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:32 +0000 (0:00:00.041) 0:00:18.034 **** ok: [localhost -> host03] TASK [show current sharding] ************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:43 +0000 (0:00:11.053) 0:00:29.088 **** ok: [localhost -> host03] TASK [reshard] **************************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:45 +0000 (0:00:01.446) 0:00:30.534 **** ok: [localhost -> host03] TASK [check fs consistency with fsck after resharding] ************************************************************************************************************************************************************ Wednesday 29 November 2023 11:25:46 +0000 (0:00:01.479) 0:00:32.014 **** ok: [localhost -> host03] TASK [restart the osd] ******************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:57 +0000 (0:00:10.699) 0:00:42.714 **** changed: [localhost -> host03] ceph orch daemon stop osd.7 cephadm shell --name osd.7 ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-7/ show-sharding m(3) p(3,0-12) O(3,0-13) L P ceph orch daemon start osd.7 cephadm shell ceph orch ps cephadm unit --name OSD_ID stop cephadm unit --name osd.0 stop cephadm shell --name OSD_ID cephadm shell --name osd.0 ceph-bluestore-tool --path/var/lib/ceph/osd/ceph- OSD_ID / fsck ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-0/ fsck fsck success ceph-bluestore-tool --path /var/lib/ceph/osd/ceph- OSD_ID / show-sharding ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-6/ show-sharding m(3) p(3,0-12) O(3,0-13) L P ceph-bluestore-tool --log-level 10 -l log.txt --path /var/lib/ceph/osd/ceph- OSD_ID / --sharding="m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P" reshard ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-6/ --sharding="m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P" reshard reshard success ceph-bluestore-tool --path /var/lib/ceph/osd/ceph- OSD_ID / show-sharding ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-6/ show-sharding m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P exit cephadm unit --name OSD_ID start cephadm unit --name osd.0 start ceph daemon OSD_ID bluestore allocator score block ceph daemon osd.123 bluestore allocator score block ceph daemon OSD_ID bluestore allocator dump block ceph daemon osd.123 bluestore allocator dump block cephadm shell --name osd. ID cephadm shell --name osd.2 Inferring fsid 110bad0a-bc57-11ee-8138-fa163eb9ffc2 Inferring config /var/lib/ceph/110bad0a-bc57-11ee-8138-fa163eb9ffc2/osd.2/config Using ceph image with id `17334f841482` and tag `ceph-7-rhel-9-containers-candidate-59483-20240301201929` created on 2024-03-01 20:22:41 +0000 UTC registry-proxy.engineering.redhat.com/rh-osbs/rhceph@sha256:09fc3e5baf198614d70669a106eb87dbebee16d4e91484375778d4adbccadacd ceph-bluestore-tool --path PATH_TO_OSD_DATA_DIRECTORY --allocator block free-score ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-123 --allocator block free-score ceph-bluestore-tool --path PATH_TO_OSD_DATA_DIRECTORY --allocator block free-dump block: { "fragmentation_rating": 0.018290238194701977 } ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-123 --allocator block free-dump block: { "capacity": 21470642176, "alloc_unit": 4096, "alloc_type": "hybrid", "alloc_name": "block", "extents": [ { "offset": "0x370000", "length": "0x20000" }, { "offset": "0x3a0000", "length": "0x10000" }, { "offset": "0x3f0000", "length": "0x20000" }, { "offset": "0x460000", "length": "0x10000" }, cephadm shell ceph config get osd bluefs_buffered_io ceph config get OSD_ID bluefs_buffered_io ceph config get osd.2 bluefs_buffered_io ceph config show OSD_ID bluefs_buffered_io ceph config show osd.3 bluefs_buffered_io cephadm shell ceph daemon osd. OSD_ID bluefs stats ceph daemon osd.1 bluefs stats 1 : device size 0x3bfc00000 : using 0x1a428000(420 MiB) wal_total:0, db_total:15296836403, slow_total:0 ceph daemon osd.1 bluefs stats 0 : 1 : device size 0x1dfbfe000 : using 0x1100000(17 MiB) 2 : device size 0x27fc00000 : using 0x248000(2.3 MiB) RocksDBBlueFSVolumeSelector: wal_total:0, db_total:7646425907, slow_total:10196562739, db_avail:935539507 Usage matrix: DEV/LEV WAL DB SLOW * * REAL FILES LOG 0 B 4 MiB 0 B 0 B 0 B 756 KiB 1 WAL 0 B 4 MiB 0 B 0 B 0 B 3.3 MiB 1 DB 0 B 9 MiB 0 B 0 B 0 B 76 KiB 10 SLOW 0 B 0 B 0 B 0 B 0 B 0 B 0 TOTALS 0 B 17 MiB 0 B 0 B 0 B 0 B 12 MAXIMUMS: LOG 0 B 4 MiB 0 B 0 B 0 B 756 KiB WAL 0 B 4 MiB 0 B 0 B 0 B 3.3 MiB DB 0 B 11 MiB 0 B 0 B 0 B 112 KiB SLOW 0 B 0 B 0 B 0 B 0 B 0 B TOTALS 0 B 17 MiB 0 B 0 B 0 B 0 B ceph-bluestore-tool COMMAND [ --dev DEVICE ... ] [ -i OSD_ID ] [ --path OSD_PATH ] [ --out-dir DIR ] [ --log-file | -l filename ] [ --deep ] ceph-bluestore-tool fsck|repair --path OSD_PATH [ --deep ] ceph-bluestore-tool qfsck --path OSD_PATH ceph-bluestore-tool allocmap --path OSD_PATH ceph-bluestore-tool restore_cfb --path OSD_PATH ceph-bluestore-tool show-label --dev DEVICE ... ceph-bluestore-tool prime-osd-dir --dev DEVICE --path OSD_PATH ceph-bluestore-tool bluefs-export --path OSD_PATH --out-dir DIR ceph-bluestore-tool bluefs-bdev-new-wal --path OSD_PATH --dev-target NEW_DEVICE ceph-bluestore-tool bluefs-bdev-new-db --path OSD_PATH --dev-target NEW_DEVICE ceph-bluestore-tool bluefs-bdev-migrate --path OSD_PATH --dev-target NEW_DEVICE --devs-source DEVICE1 [--devs-source DEVICE2 ] ceph-bluestore-tool free-dump|free-score --path OSD_PATH [ --allocator block/bluefs-wal/bluefs-db/bluefs-slow ] ceph-bluestore-tool reshard --path OSD_PATH --sharding NEW_SHARDING [ --sharding-ctrl CONTROL_STRING ] ceph-bluestore-tool show-sharding --path OSD_PATH ceph orch daemon stop osd. ID ceph orch daemon stop osd.2 cephadm shell --name osd. ID ceph shell --name osd.2 ceph-bluestore-tool bluefs-bdev-new-wal --dev-target /dev/test/newdb --path /var/lib/ceph/osd/ceph-0 ceph orch daemon start osd. ID ceph orch daemon start osd.2 ceph tell OSD_ID dump_metrics ceph tell OSD_ID dump_metrics reactor_utilization ceph tell osd.0 dump_metrics ceph tell osd.0 dump_metrics reactor_utilization cephadm --image quay.ceph.io/ceph-ci/ceph:b682861f8690608d831f58603303388dd7915aa7-crimson bootstrap --mon-ip 10.1.240.54 --allow-fqdn-hostname --initial-dashboard-password Ceph_Crims cephadm shell ceph config set global 'enable_experimental_unrecoverable_data_corrupting_features' crimson ceph osd set-allow-crimson --yes-i-really-mean-it ceph config set mon osd_pool_default_crimson true dnf install libnbd git clone git://git.kernel.dk/fio.git cd fio ./configure --enable-libnbd make cd build ninja crimson-store-nbd export disk_img=/tmp/disk.img export unix_socket=/tmp/store_nbd_socket.sock rm -f USDdisk_img USDunix_socket truncate -s 512M USDdisk_img ./bin/crimson-store-nbd --device-path USDdisk_img --smp 1 --mkfs true --type transaction_manager --uds-path USD{unix_socket} & --smp is the CPU cores. --mkfs initializes the device first. --type is the backend. [global] ioengine=nbd uri=nbd+unix:///?socket=USD{unix_socket} rw=randrw time_based runtime=120 group_reporting iodepth=1 size=512M [job0] offset=0 ./fio nbd.fio git checkout master make crimson-osd ../src/script/run-cbt.sh --cbt ~/dev/cbt -a /tmp/baseline ../src/test/crimson/cbt/radosbench_4K_read.yaml git checkout topic make crimson-osd ../src/script/run-cbt.sh --cbt ~/dev/cbt -a /tmp/yap ../src/test/crimson/cbt/radosbench_4K_read.yaml ~/dev/cbt/compare.py -b /tmp/baseline -a /tmp/yap -v ceph orch pause ceph orch set backend '' ceph mgr module disable cephadm ceph mgr module enable cephadm ceph orch set backend cephadm ceph orch ls --service_name SERVICE_NAME --format yaml ceph orch ls --service_name alertmanager --format yaml service_type: alertmanager service_name: alertmanager placement: hosts: - unknown_host status: running: 1 size: 1 events: - 2021-02-01T08:58:02.741162 service:alertmanager [INFO] "service was created" - '2021-02-01T12:09:25.264584 service:alertmanager [ERROR] "Failed to apply: Cannot place <AlertManagerSpec for service_name=alertmanager> on unknown_host: Unknown hosts"' ceph orch ps --service-name SERVICE_NAME --daemon-id DAEMON_ID --format yaml ceph orch ps --service-name mds --daemon-id cephfs.hostname.ppdhsz --format yaml daemon_type: mds daemon_id: cephfs.hostname.ppdhsz hostname: hostname status_desc: running events: - 2021-02-01T08:59:43.845866 daemon:mds.cephfs.hostname.ppdhsz [INFO] "Reconfigured mds.cephfs.hostname.ppdhsz on host 'hostname'" ceph -W cephadm ceph log last cephadm cephadm logs --name DAEMON_NAME cephadm logs --name cephfs.hostname.ppdhsz cephadm logs --fsid FSID --name DAEMON_NAME cephadm logs --fsid 2d2fd136-6df1-11ea-ae74-002590e526e8 --name cephfs.hostname.ppdhsz for name in USD(cephadm ls | python3 -c "import sys, json; [print(i['name']) for i in json.load(sys.stdin)]") ; do cephadm logs --fsid FSID_OF_CLUSTER --name "USDname" > USDname; done for name in USD(cephadm ls | python3 -c "import sys, json; [print(i['name']) for i in json.load(sys.stdin)]") ; do cephadm logs --fsid 57bddb48-ee04-11eb-9962-001a4a000672 --name "USDname" > USDname; done [root@host01 ~]USD systemctl status ceph-a538d494-fb2a-48e4-82c8-b91c37bb0684@mon.host01.service podman ps -a --format json | jq '.[].Image' "docker.io/library/rhel9" "registry.redhat.io/rhceph-alpha/rhceph-6-rhel9@sha256:9aaea414e2c263216f3cdcb7a096f57c3adf6125ec9f4b0f5f65fa8c43987155" execnet.gateway_bootstrap.HostNotFound: -F /tmp/cephadm-conf-73z09u6g -i /tmp/cephadm-identity-ky7ahp_5 root@10.10.1.2 raise OrchestratorError(msg) from e orchestrator._interface.OrchestratorError: Failed to connect to 10.10.1.2 (10.10.1.2). Please make sure that the host is reachable and accepts connections using the cephadm SSH key ceph config-key get mgr/cephadm/ssh_identity_key > ~/cephadm_private_key INFO:cephadm:Inferring fsid f8edc08a-7f17-11ea-8707-000c2915dd98 INFO:cephadm:Using recent ceph image docker.io/ceph/ceph:v15 obtained 'mgr/cephadm/ssh_identity_key' chmod 0600 ~/cephadm_private_key chmod 0600 ~/cephadm_private_key cat ~/cephadm_private_key | ceph cephadm set-ssk-key -i- ceph cephadm get-ssh-config ssh -F config -i ~/cephadm_private_key root@host01 ceph cephadm get-pub-key grep "`cat ~/ceph.pub`" /root/.ssh/authorized_keys ceph config set host public_network hostnetwork cephadm enter --name cephfs.hostname.ppdhsz ceph --admin-daemon /var/run/ceph/ceph-cephfs.hostname.ppdhsz.asok config show cephadm shell ceph config-key set mgr/cephadm/pause true ceph auth get-or-create mgr.host01.smfvfd1 mon "profile mgr" osd "allow *" mds "allow *" [mgr.host01.smfvfd1] key = AQDhcORgW8toCRAAlMzlqWXnh3cGRjqYEa9ikw== ceph config generate-minimal-conf minimal ceph.conf for 8c9b0072-67ca-11eb-af06-001a4a0002a0 [global] fsid = 8c9b0072-67ca-11eb-af06-001a4a0002a0 mon_host = [v2:10.10.200.10:3300/0,v1:10.10.200.10:6789/0] [v2:10.10.10.100:3300/0,v1:10.10.200.100:6789/0] ceph config get "mgr.host01.smfvfd1" container_image { { "config": "# minimal ceph.conf for 8c9b0072-67ca-11eb-af06-001a4a0002a0\n[global]\n\tfsid = 8c9b0072-67ca-11eb-af06-001a4a0002a0\n\tmon_host = [v2:10.10.200.10:3300/0,v1:10.10.200.10:6789/0] [v2:10.10.10.100:3300/0,v1:10.10.200.100:6789/0]\n", "keyring": "[mgr.Ceph5-2.smfvfd1]\n\tkey = AQDhcORgW8toCRAAlMzlqWXnh3cGRjqYEa9ikw==\n" } } exit cephadm --image registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest deploy --fsid 8c9b0072-67ca-11eb-af06-001a4a0002a0 --name mgr.host01.smfvfd1 --config-json config-json.json ceph -s ceph -W cephadm 2022-06-10T17:51:36.335728+0000 mgr.Ceph5-1.nqikfh [INF] refreshing Ceph5-adm facts 2022-06-10T17:51:37.170982+0000 mgr.Ceph5-1.nqikfh [INF] deploying 1 monitor(s) instead of 2 so monitors may achieve consensus 2022-06-10T17:51:37.173487+0000 mgr.Ceph5-1.nqikfh [ERR] It is NOT safe to stop ['mon.Ceph5-adm']: not enough monitors would be available (Ceph5-2) after stopping mons [Ceph5-adm] 2022-06-10T17:51:37.174415+0000 mgr.Ceph5-1.nqikfh [INF] Checking pool "nfs-ganesha" exists for service nfs.foo 2022-06-10T17:51:37.176389+0000 mgr.Ceph5-1.nqikfh [ERR] Failed to apply nfs.foo spec NFSServiceSpec({'placement': PlacementSpec(count=1), 'service_type': 'nfs', 'service_id': 'foo', 'unmanaged': False, 'preview_only': False, 'pool': 'nfs-ganesha', 'namespace': 'nfs-ns'}): Cannot find pool "nfs-ganesha" for service nfs.foo Traceback (most recent call last): File "/usr/share/ceph/mgr/cephadm/serve.py", line 408, in _apply_all_services if self._apply_service(spec): File "/usr/share/ceph/mgr/cephadm/serve.py", line 509, in _apply_service config_func(spec) File "/usr/share/ceph/mgr/cephadm/services/nfs.py", line 23, in config self.mgr._check_pool_exists(spec.pool, spec.service_name()) File "/usr/share/ceph/mgr/cephadm/module.py", line 1840, in _check_pool_exists raise OrchestratorError(f'Cannot find pool "{pool}" for ' orchestrator._interface.OrchestratorError: Cannot find pool "nfs-ganesha" for service nfs.foo 2022-06-10T17:51:37.179658+0000 mgr.Ceph5-1.nqikfh [INF] Found osd claims -> {} 2022-06-10T17:51:37.180116+0000 mgr.Ceph5-1.nqikfh [INF] Found osd claims for drivegroup all-available-devices -> {} 2022-06-10T17:51:37.182138+0000 mgr.Ceph5-1.nqikfh [INF] Applying all-available-devices on host Ceph5-adm 2022-06-10T17:51:37.182987+0000 mgr.Ceph5-1.nqikfh [INF] Applying all-available-devices on host Ceph5-1 2022-06-10T17:51:37.183395+0000 mgr.Ceph5-1.nqikfh [INF] Applying all-available-devices on host Ceph5-2 2022-06-10T17:51:43.373570+0000 mgr.Ceph5-1.nqikfh [INF] Reconfiguring node-exporter.Ceph5-1 (unknown last config time) 2022-06-10T17:51:43.373840+0000 mgr.Ceph5-1.nqikfh [INF] Reconfiguring daemon node-exporter.Ceph5-1 on Ceph5-1 ceph config set mgr mgr/cephadm/log_to_cluster_level debug ceph -W cephadm --watch-debug ceph -W cephadm --verbose ceph config set mgr mgr/cephadm/log_to_cluster_level info ceph log last cephadm journalctl -u ceph-5c5a50ae-272a-455d-99e9-32c6a013e694@host01 ceph config set global log_to_stderr false ceph config set global mon_cluster_log_to_stderr false ceph config set global log_to_file true ceph config set global mon_cluster_log_to_file true service_type: grafana service_name: grafana custom_configs: - mount_path: /etc/example.conf content: | setting1 = value1 setting2 = value2 - mount_path: /usr/share/grafana/example.cert content: | -----BEGIN PRIVATE KEY----- V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8= -----END CERTIFICATE----- ceph orch redeploy SERVICE_NAME ceph orch redeploy grafana ceph config set mgr mgr/cephadm/config_checks_enabled true ALL cephadm checks are disabled, use 'ceph config set mgr mgr/cephadm/config_checks_enabled true' to enable CEPHADM 8/8 checks enabled and executed (0 bypassed, 0 disabled). No issues detected ceph cephadm config-check status ceph cephadm config-check ls NAME HEALTHCHECK STATUS DESCRIPTION kernel_security CEPHADM_CHECK_KERNEL_LSM enabled checks SELINUX/Apparmor profiles are consistent across cluster hosts os_subscription CEPHADM_CHECK_SUBSCRIPTION enabled checks subscription states are consistent for all cluster hosts public_network CEPHADM_CHECK_PUBLIC_MEMBERSHIP enabled check that all hosts have a NIC on the Ceph public_netork osd_mtu_size CEPHADM_CHECK_MTU enabled check that OSD hosts share a common MTU setting osd_linkspeed CEPHADM_CHECK_LINKSPEED enabled check that OSD hosts share a common linkspeed network_missing CEPHADM_CHECK_NETWORK_MISSING enabled checks that the cluster/public networks defined exist on the Ceph hosts ceph_release CEPHADM_CHECK_CEPH_RELEASE enabled check for Ceph version consistency - ceph daemons should be on the same release (unless upgrade is active) kernel_version CEPHADM_CHECK_KERNEL_VERSION enabled checks that the MAJ.MIN of the kernel on Ceph hosts is consistent [ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible sudo vi INVENTORY_FILE HOST1 labels="[' LABEL1 ', ' LABEL2 ']" HOST2 labels="[' LABEL1 ', ' LABEL2 ']" HOST3 labels="[' LABEL1 ']" [admin] ADMIN_HOST monitor_address= MONITOR_IP_ADDRESS labels="[' ADMIN_LABEL ', ' LABEL1 ', ' LABEL2 ']" [ceph-admin@admin cephadm-ansible]USD sudo vi hosts host02 labels="['mon', 'mgr']" host03 labels="['mon', 'mgr']" host04 labels="['osd']" host05 labels="['osd']" host06 labels="['osd']" [admin] host01 monitor_address=10.10.128.68 labels="['_admin', 'mon', 'mgr']" ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" sudo vi PLAYBOOK_FILENAME .yml --- - name: NAME_OF_PLAY hosts: BOOTSTRAP_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: -name: NAME_OF_TASK cephadm_registry_login: state: STATE registry_url: REGISTRY_URL registry_username: REGISTRY_USER_NAME registry_password: REGISTRY_PASSWORD - name: NAME_OF_TASK cephadm_bootstrap: mon_ip: "{{ monitor_address }}" dashboard_user: DASHBOARD_USER dashboard_password: DASHBOARD_PASSWORD allow_fqdn_hostname: ALLOW_FQDN_HOSTNAME cluster_network: NETWORK_CIDR [ceph-admin@admin cephadm-ansible]USD sudo vi bootstrap.yml --- - name: bootstrap the cluster hosts: host01 become: true gather_facts: false tasks: - name: login to registry cephadm_registry_login: state: login registry_url: registry.redhat.io registry_username: user1 registry_password: mypassword1 - name: bootstrap initial cluster cephadm_bootstrap: mon_ip: "{{ monitor_address }}" dashboard_user: mydashboarduser dashboard_password: mydashboardpassword allow_fqdn_hostname: true cluster_network: 10.10.128.0/28 ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml -vvv [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts bootstrap.yml -vvv [ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible sudo vi INVENTORY_FILE NEW_HOST1 labels="[' LABEL1 ', ' LABEL2 ']" NEW_HOST2 labels="[' LABEL1 ', ' LABEL2 ']" NEW_HOST3 labels="[' LABEL1 ']" [admin] ADMIN_HOST monitor_address= MONITOR_IP_ADDRESS labels="[' ADMIN_LABEL ', ' LABEL1 ', ' LABEL2 ']" [ceph-admin@admin cephadm-ansible]USD sudo vi hosts host02 labels="['mon', 'mgr']" host03 labels="['mon', 'mgr']" host04 labels="['osd']" host05 labels="['osd']" host06 labels="['osd']" [admin] host01 monitor_address= 10.10.128.68 labels="['_admin', 'mon', 'mgr']" ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit NEWHOST [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit host02 sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: HOSTS_OR_HOST_GROUPS become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_host: name: "{{ ansible_facts['hostname'] }}" address: "{{ ansible_facts['default_ipv4']['address'] }}" labels: "{{ labels }}" delegate_to: HOST_TO_DELEGATE_TASK_TO - name: NAME_OF_TASK when: inventory_hostname in groups['admin'] ansible.builtin.shell: cmd: CEPH_COMMAND_TO_RUN register: REGISTER_NAME - name: NAME_OF_TASK when: inventory_hostname in groups['admin'] debug: msg: "{{ REGISTER_NAME .stdout }}" [ceph-admin@admin cephadm-ansible]USD sudo vi add-hosts.yml --- - name: add additional hosts to the cluster hosts: all become: true gather_facts: true tasks: - name: add hosts to the cluster ceph_orch_host: name: "{{ ansible_facts['hostname'] }}" address: "{{ ansible_facts['default_ipv4']['address'] }}" labels: "{{ labels }}" delegate_to: host01 - name: list hosts in the cluster when: inventory_hostname in groups['admin'] ansible.builtin.shell: cmd: ceph orch host ls register: host_list - name: print current list of hosts when: inventory_hostname in groups['admin'] debug: msg: "{{ host_list.stdout }}" ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts add-hosts.yml [ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible sudo vi PLAYBOOK_FILENAME .yml --- - name: NAME_OF_PLAY hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_host: name: HOST_TO_REMOVE state: STATE - name: NAME_OF_TASK ceph_orch_host: name: HOST_TO_REMOVE state: STATE retries: NUMBER_OF_RETRIES delay: DELAY until: CONTINUE_UNTIL register: REGISTER_NAME - name: NAME_OF_TASK ansible.builtin.shell: cmd: ceph orch host ls register: REGISTER_NAME - name: NAME_OF_TASK debug: msg: "{{ REGISTER_NAME .stdout }}" [ceph-admin@admin cephadm-ansible]USD sudo vi remove-hosts.yml --- - name: remove host hosts: host01 become: true gather_facts: true tasks: - name: drain host07 ceph_orch_host: name: host07 state: drain - name: remove host from the cluster ceph_orch_host: name: host07 state: absent retries: 20 delay: 1 until: result is succeeded register: result - name: list hosts in the cluster ansible.builtin.shell: cmd: ceph orch host ls register: host_list - name: print current list of hosts debug: msg: "{{ host_list.stdout }}" ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts remove-hosts.yml TASK [print current hosts] ****************************************************************************************************** Friday 24 June 2022 14:52:40 -0400 (0:00:03.365) 0:02:31.702 *********** ok: [host01] => msg: |- HOST ADDR LABELS STATUS host01 10.10.128.68 _admin mon mgr host02 10.10.128.69 mon mgr host03 10.10.128.70 mon mgr host04 10.10.128.71 osd host05 10.10.128.72 osd host06 10.10.128.73 osd [ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_config: action: GET_OR_SET who: DAEMON_TO_SET_CONFIGURATION_TO option: CEPH_CONFIGURATION_OPTION value: VALUE_OF_PARAMETER_TO_SET - name: NAME_OF_TASK ceph_config: action: GET_OR_SET who: DAEMON_TO_SET_CONFIGURATION_TO option: CEPH_CONFIGURATION_OPTION register: REGISTER_NAME - name: NAME_OF_TASK debug: msg: " MESSAGE_TO_DISPLAY {{ REGISTER_NAME .stdout }}" [ceph-admin@admin cephadm-ansible]USD sudo vi change_configuration.yml --- - name: set pool delete hosts: host01 become: true gather_facts: false tasks: - name: set the allow pool delete option ceph_config: action: set who: mon option: mon_allow_pool_delete value: true - name: get the allow pool delete setting ceph_config: action: get who: mon option: mon_allow_pool_delete register: verify_mon_allow_pool_delete - name: print current mon_allow_pool_delete setting debug: msg: "the value of 'mon_allow_pool_delete' is {{ verify_mon_allow_pool_delete.stdout }}" ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts change_configuration.yml TASK [print current mon_allow_pool_delete setting] ************************************************************* Wednesday 29 June 2022 13:51:41 -0400 (0:00:05.523) 0:00:17.953 ******** ok: [host01] => msg: the value of 'mon_allow_pool_delete' is true [ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: HOSTS_OR_HOST_GROUPS become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_apply: spec: | service_type: SERVICE_TYPE service_id: UNIQUE_NAME_OF_SERVICE placement: host_pattern: ' HOST_PATTERN_TO_SELECT_HOSTS ' label: LABEL spec: SPECIFICATION_OPTIONS : [ceph-admin@admin cephadm-ansible]USD sudo vi deploy_osd_service.yml --- - name: deploy osd service hosts: host01 become: true gather_facts: true tasks: - name: apply osd spec ceph_orch_apply: spec: | service_type: osd service_id: osd placement: host_pattern: '*' label: osd spec: data_devices: all: true ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts deploy_osd_service.yml [ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_daemon: state: STATE_OF_SERVICE daemon_id: DAEMON_ID daemon_type: TYPE_OF_SERVICE [ceph-admin@admin cephadm-ansible]USD sudo vi restart_services.yml --- - name: start and stop services hosts: host01 become: true gather_facts: false tasks: - name: start osd.0 ceph_orch_daemon: state: started daemon_id: 0 daemon_type: osd - name: stop mon.host02 ceph_orch_daemon: state: stopped daemon_id: host02 daemon_type: mon ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts restart_services.yml