Object Gateway Guide
Object Gateway Guide Red Hat Ceph Storage 8 Deploying, configuring, and administering a Ceph Object Gateway Red Hat Ceph Storage Documentation Team
ceph orch apply mon --placement="host1 host2 host3" service_type: mon placement: hosts: - host01 - host02 - host03 ceph orch apply -i mon.yml ceph orch apply rgw example --placement="6 host1 host2 host3" service_type: rgw service_id: example placement: count: 6 hosts: - host01 - host02 - host03 ceph orch apply -i rgw.yml mon_pg_warn_max_per_osd = n ceph osd pool create .us-west.rgw.buckets.non-ec 64 64 replicated rgw-service ## SAS-SSD ROOT DECLARATION ## root sas-ssd { id -1 # do not change unnecessarily # weight 0.000 alg straw hash 0 # rjenkins1 item data2-sas-ssd weight 4.000 item data1-sas-ssd weight 4.000 item data0-sas-ssd weight 4.000 } ## INDEX ROOT DECLARATION ## root index { id -2 # do not change unnecessarily # weight 0.000 alg straw hash 0 # rjenkins1 item data2-index weight 1.000 item data1-index weight 1.000 item data0-index weight 1.000 } host data2-sas-ssd { id -11 # do not change unnecessarily # weight 0.000 alg straw hash 0 # rjenkins1 item osd.0 weight 1.000 item osd.1 weight 1.000 item osd.2 weight 1.000 item osd.3 weight 1.000 } host data2-index { id -21 # do not change unnecessarily # weight 0.000 alg straw hash 0 # rjenkins1 item osd.4 weight 1.000 } osd_crush_update_on_start = false [osd.0] osd crush location = "host=data2-sas-ssd" [osd.1] osd crush location = "host=data2-sas-ssd" [osd.2] osd crush location = "host=data2-sas-ssd" [osd.3] osd crush location = "host=data2-sas-ssd" [osd.4] osd crush location = "host=data2-index" ## SERVICE RULE DECLARATION ## rule rgw-service { type replicated min_size 1 max_size 10 step take sas-ssd step chooseleaf firstn 0 type rack step emit } ## THROUGHPUT RULE DECLARATION ## rule rgw-throughput { type replicated min_size 1 max_size 10 step take sas-ssd step chooseleaf firstn 0 type host step emit } ## INDEX RULE DECLARATION ## rule rgw-index { type replicated min_size 1 max_size 10 step take index step chooseleaf firstn 0 type rack step emit } rule ecpool-86 { step take default class hdd step choose indep 4 type host step choose indep 4 type osd step emit } rule ecpool-86 { type msr_indep step take default class hdd step choosemsr 4 type host step choosemsr 4 type osd step emit } rule ecpool-86 { step take default class hdd step choose indep 4 type host step choose indep 4 type osd step emit } rule ecpool-86 { type msr_indep step take default class hdd step choosemsr 4 type host step choosemsr 4 type osd step emit } [osd] osd_max_backfills = 1 osd_recovery_max_active = 1 osd_recovery_op_priority = 1 ceph config set global osd_map_message_max 10 ceph config set osd osd_map_cache_size 20 ceph config set osd osd_map_share_max_epochs 10 ceph config set osd osd_pg_epoch_persisted_max_stale 10 [osd] osd_scrub_begin_hour = 23 #23:01H, or 10:01PM. osd_scrub_end_hour = 6 #06:01H or 6:01AM. [osd] osd_scrub_load_threshold = 0.25 objecter_inflight_ops = 24576 rgw_thread_pool_size = 512 ceph soft nofile unlimited USER_NAME soft nproc unlimited cephadm shell radosgw-admin realm create --rgw-realm= REALM_NAME --default radosgw-admin realm create --rgw-realm=test_realm --default radosgw-admin zonegroup create --rgw-zonegroup= ZONE_GROUP_NAME --master --default radosgw-admin zonegroup create --rgw-zonegroup=default --master --default radosgw-admin zone create --rgw-zonegroup= ZONE_GROUP_NAME --rgw-zone= ZONE_NAME --master --default radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=test_zone --master --default radosgw-admin period update --rgw-realm= REALM_NAME --commit radosgw-admin period update --rgw-realm=test_realm --commit ceph orch apply rgw NAME [--realm= REALM_NAME ] [--zone= ZONE_NAME ] [--zonegroup= ZONE_GROUP_NAME ] --placement=" NUMBER_OF_DAEMONS [ HOST_NAME_1 HOST_NAME_2 ]" ceph orch apply rgw test --realm=test_realm --zone=test_zone --zonegroup=default --placement="2 host01 host02" ceph orch apply rgw SERVICE_NAME ceph orch apply rgw foo ceph orch host label add HOST_NAME_1 LABEL_NAME ceph orch host label add HOSTNAME_2 LABEL_NAME ceph orch apply rgw SERVICE_NAME --placement="label: LABEL_NAME count-per-host: NUMBER_OF_DAEMONS " --port=8000 ceph orch host label add host01 rgw # the 'rgw' label can be anything ceph orch host label add host02 rgw ceph orch apply rgw foo --placement="label:rgw count-per-host:2" --port=8000 ceph orch ls ceph orch ps --daemon_type= DAEMON_NAME ceph orch ps --daemon_type=rgw cephadm shell cat nfs-conf.yml service_type: nfs service_id: nfs-rgw-service placement: hosts: ['host1'] spec: port: 2049 ceph orch apply -i nfs-conf.yml ceph orch ls --service_name nfs.nfs-rgw-service --service_type nfs touch radosgw.yml service_type: rgw service_id: REALM_NAME . ZONE_NAME placement: hosts: - HOST_NAME_1 - HOST_NAME_2 count_per_host: NUMBER_OF_DAEMONS spec: rgw_realm: REALM_NAME rgw_zone: ZONE_NAME rgw_zonegroup: ZONE_GROUP_NAME rgw_frontend_port: FRONT_END_PORT networks: - NETWORK_CIDR # Ceph Object Gateway service binds to a specific network service_type: rgw service_id: default placement: hosts: - host01 - host02 - host03 count_per_host: 2 spec: rgw_realm: default rgw_zone: default rgw_zonegroup: default rgw_frontend_port: 1234 networks: - 192.169.142.0/24 radosgw-admin realm create --rgw-realm=test_realm --default radosgw-admin zonegroup create --rgw-zonegroup=test_zonegroup --default radosgw-admin zone create --rgw-zonegroup=test_zonegroup --rgw-zone=test_zone --default radosgw-admin period update --rgw-realm=test_realm --commit service_type: rgw service_id: test_realm.test_zone placement: hosts: - host01 - host02 - host03 count_per_host: 2 spec: rgw_realm: test_realm rgw_zone: test_zone rgw_zonegroup: test_zonegroup rgw_frontend_port: 1234 networks: - 192.169.142.0/24 cephadm shell --mount radosgw.yml:/var/lib/ceph/radosgw/radosgw.yml ceph orch apply -i FILE_NAME .yml ceph orch apply -i /var/lib/ceph/radosgw/radosgw.yml ceph orch ls ceph orch ps --daemon_type= DAEMON_NAME ceph orch ps --daemon_type=rgw radosgw-admin realm create --rgw-realm= REALM_NAME --default radosgw-admin realm create --rgw-realm=test_realm --default radosgw-admin zonegroup create --rgw-zonegroup= ZONE_GROUP_NAME --endpoints=http:// RGW_PRIMARY_HOSTNAME : RGW_PRIMARY_PORT_NUMBER_1 --master --default radosgw-admin zonegroup create --rgw-zonegroup=us --endpoints=http://rgw1:80 --master --default radosgw-admin zone create --rgw-zonegroup= PRIMARY_ZONE_GROUP_NAME --rgw-zone= PRIMARY_ZONE_NAME --endpoints=http:// RGW_PRIMARY_HOSTNAME : RGW_PRIMARY_PORT_NUMBER_1 --access-key= SYSTEM_ACCESS_KEY --secret= SYSTEM_SECRET_KEY radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east-1 --endpoints=http://rgw1:80 --access-key=LIPEYZJLTWXRKXS9LPJC --secret-key=IsAje0AVDNXNw48LjMAimpCpI7VaxJYSnfD0FFKQ radosgw-admin zonegroup delete --rgw-zonegroup=default ceph osd pool rm default.rgw.log default.rgw.log --yes-i-really-really-mean-it ceph osd pool rm default.rgw.meta default.rgw.meta --yes-i-really-really-mean-it ceph osd pool rm default.rgw.control default.rgw.control --yes-i-really-really-mean-it ceph osd pool rm default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it ceph osd pool rm default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it radosgw-admin user create --uid= USER_NAME --display-name=" USER_NAME " --access-key= SYSTEM_ACCESS_KEY --secret= SYSTEM_SECRET_KEY --system radosgw-admin user create --uid=zone.user --display-name="Zone user" --system radosgw-admin zone modify --rgw-zone= PRIMARY_ZONE_NAME --access-key= ACCESS_KEY --secret= SECRET_KEY radosgw-admin zone modify --rgw-zone=us-east-1 --access-key=NE48APYCAODEPLKBCZVQ--secret=u24GHQWRE3yxxNBnFBzjM4jn14mFIckQ4EKL6LoW radosgw-admin period update --commit radosgw-admin period update --commit systemctl list-units | grep ceph systemctl start ceph- FSID @ DAEMON_NAME systemctl enable ceph- FSID @ DAEMON_NAME systemctl start ceph-62a081a6-88aa-11eb-a367-001a4a000672@rgw.test_realm.us-east-1.host01.ahdtsw.service systemctl enable ceph-62a081a6-88aa-11eb-a367-001a4a000672@rgw.test_realm.us-east-1.host01.ahdtsw.service radosgw-admin realm pull --rgw-realm= PRIMARY_REALM --url= URL_TO_PRIMARY_ZONE_GATEWAY --access-key= ACCESS_KEY --secret-key= SECRET_KEY --default radosgw-admin realm pull --rgw-realm=test_realm --url=http://10.74.249.26:80 --access-key=LIPEYZJLTWXRKXS9LPJC --secret-key=IsAje0AVDNXNw48LjMAimpCpI7VaxJYSnfD0FFKQ --default radosgw-admin period pull --url= URL_TO_PRIMARY_ZONE_GATEWAY --access-key= ACCESS_KEY --secret-key= SECRET_KEY radosgw-admin period pull --url=http://10.74.249.26:80 --access-key=LIPEYZJLTWXRKXS9LPJC --secret-key=IsAje0AVDNXNw48LjMAimpCpI7VaxJYSnfD0FFKQ radosgw-admin zone create --rgw-zonegroup= ZONE_GROUP_NAME --rgw-zone= SECONDARY_ZONE_NAME --endpoints=http:// RGW_SECONDARY_HOSTNAME : RGW_PRIMARY_PORT_NUMBER_1 --access-key= SYSTEM_ACCESS_KEY --secret= SYSTEM_SECRET_KEY --endpoints=http:// FQDN :80 [--read-only] radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east-2 --endpoints=http://rgw2:80 --access-key=LIPEYZJLTWXRKXS9LPJC --secret-key=IsAje0AVDNXNw48LjMAimpCpI7VaxJYSnfD0FFKQ radosgw-admin zone rm --rgw-zone=default ceph osd pool rm default.rgw.log default.rgw.log --yes-i-really-really-mean-it ceph osd pool rm default.rgw.meta default.rgw.meta --yes-i-really-really-mean-it ceph osd pool rm default.rgw.control default.rgw.control --yes-i-really-really-mean-it ceph osd pool rm default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it ceph osd pool rm default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it ceph config set SERVICE_NAME rgw_zone SECONDARY_ZONE_NAME ceph config set rgw rgw_zone us-east-2 radosgw-admin period update --commit radosgw-admin period update --commit systemctl list-units | grep ceph systemctl start ceph- FSID @ DAEMON_NAME systemctl enable ceph- FSID @ DAEMON_NAME systemctl start ceph-62a081a6-88aa-11eb-a367-001a4a000672@rgw.test_realm.us-east-2.host04.ahdtsw.service systemctl enable ceph-62a081a6-88aa-11eb-a367-001a4a000672@rgw.test_realm.us-east-2.host04.ahdtsw.service ceph orch apply rgw NAME --realm= REALM_NAME --zone= PRIMARY_ZONE_NAME --placement=" NUMBER_OF_DAEMONS HOST_NAME_1 HOST_NAME_2 " ceph orch apply rgw east --realm=test_realm --zone=us-east-1 --placement="2 host01 host02" radosgw-admin sync status cephadm shell ceph orch ls ceph orch rm SERVICE_NAME ceph orch rm rgw.test_realm.test_zone_bb ceph orch ps ceph orch ps cephadm shell ceph mgr module enable rgw ceph rgw realm bootstrap [--realm name REALM_NAME ] [--zonegroup-name ZONEGROUP_NAME ] [--zone-name ZONE_NAME ] [--port PORT_NUMBER ] [--placement HOSTNAME ] [--start-radosgw] ceph rgw realm bootstrap --realm-name myrealm --zonegroup-name myzonegroup --zone-name myzone --port 5500 --placement="host01 host02" --start-radosgw Realm(s) created correctly. Please, use 'ceph rgw realm tokens' to get the token. rgw_realm: REALM_NAME rgw_zonegroup: ZONEGROUP_NAME rgw_zone: ZONE_NAME placement: hosts: - _HOSTNAME_1_ - _HOSTNAME_2_ cat rgw.yaml rgw_realm: myrealm rgw_zonegroup: myzonegroup rgw_zone: myzone placement: hosts: - host01 - host02 service_type: rgw placement: hosts: - _host1_ - _host2_ spec: rgw_realm: my_realm rgw_zonegroup: my_zonegroup rgw_zone: my_zone zonegroup_hostnames: - _hostname1_ - _hostname2_ service_type: rgw placement: hosts: - _host1_ - _host2_ spec: rgw_realm: my_realm rgw_zonegroup: my_zonegroup rgw_zone: my_zone zonegroup_hostnames: - foo - bar cephadm shell --mount rgw.yaml:/var/lib/ceph/rgw/rgw.yaml ceph rgw realm bootstrap -i /var/lib/ceph/rgw/rgw.yaml ceph rgw realm tokens | jq [ { "realm": "myrealm", "token": "ewogICAgInJlYWxtX25hbWUiOiAibXlyZWFsbSIsCiAgICAicmVhbG1faWQiOiAiZDA3YzAwZWYtOTA0MS00ZjZlLTg4MDQtN2Q0MDI0MDU1NmFlIiwKICAgICJlbmRwb2ludCI6ICJodHRwOi8vdm0tMDA6NDMyMSIsCiAgICAiYWNjZXNzX2tleSI6ICI5NTY1VFZSMVFWTExFRzdVNFIxRCIsCiAgICAic2VjcmV0IjogImQ3b0FJQXZrNEdYeXpyd3Q2QVZ6bEZNQmNnRG53RVdMMHFDenE3cjUiCn1=" } ] ceph orch list --daemon-type=rgw NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID rgw.myrealm.myzonegroup.ceph-saya-6-osd-host01.eburst ceph-saya-6-osd-host01 *:80 running (111m) 9m ago 111m 82.3M - 17.2.6-22.el9cp 2d5b080de0b0 2f3eaca7e88e radosgw-admin zonegroup get --rgw-zonegroup _zone_group_name_ radosgw-admin zonegroup get --rgw-zonegroup my_zonegroup { "id": "02a175e2-7f23-4882-8651-6fbb15d25046", "name": "my_zonegroup_ck", "api_name": "my_zonegroup_ck", "is_master": true, "endpoints": [ "http://vm-00:80" ], "hostnames": [ "foo" "bar" ], "hostnames_s3website": [], "master_zone": "f42fea84-a89e-4995-996e-61b7223fb0b0", "zones": [ { "id": "f42fea84-a89e-4995-996e-61b7223fb0b0", "name": "my_zone_ck", "endpoints": [ "http://vm-00:80" ], "log_meta": false, "log_data": false, "bucket_index_max_shards": 11, "read_only": false, "tier_type": "", "sync_from_all": true, "sync_from": [], "redirect_zone": "", "supported_features": [ "compress-encrypted", "resharding" ] } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD" ] } ], "default_placement": "default-placement", "realm_id": "439e9c37-4ddc-43a3-99e9-ea1f3825bb51", "sync_policy": { "groups": [] }, "enabled_features": [ "resharding" ] } cephadm shell ceph mgr module enable rgw ceph rgw realm bootstrap [--realm name REALM_NAME ] [--zonegroup-name ZONEGROUP_NAME ] [--zone-name ZONE_NAME ] [--port PORT_NUMBER ] [--placement HOSTNAME ] [--start-radosgw] ceph rgw realm bootstrap --realm-name myrealm --zonegroup-name myzonegroup --zone-name myzone --port 5500 --placement="host01 host02" --start-radosgw Realm(s) created correctly. Please, use 'ceph rgw realm tokens' to get the token. rgw_realm: REALM_NAME rgw_zonegroup: ZONEGROUP_NAME rgw_zone: ZONE_NAME placement: hosts: - HOSTNAME_1 - HOSTNAME_2 spec: rgw_frontend_port: PORT_NUMBER zone_endpoints: http:// RGW_HOSTNAME_1 : RGW_PORT_NUMBER_1 , http:// RGW_HOSTNAME_2 : RGW_PORT_NUMBER_2 cat rgw.yaml rgw_realm: myrealm rgw_zonegroup: myzonegroup rgw_zone: myzone placement: hosts: - host01 - host02 spec: rgw_frontend_port: 5500 zone_endpoints: http://<rgw_host1>:<rgw_port1>, http://<rgw_host2>:<rgw_port2> cephadm shell --mount rgw.yaml:/var/lib/ceph/rgw/rgw.yaml ceph rgw realm bootstrap -i /var/lib/ceph/rgw/rgw.yaml ceph rgw realm tokens | jq [ { "realm": "myrealm", "token": "ewogICAgInJlYWxtX25hbWUiOiAibXlyZWFsbSIsCiAgICAicmVhbG1faWQiOiAiZDA3YzAwZWYtOTA0MS00ZjZlLTg4MDQtN2Q0MDI0MDU1NmFlIiwKICAgICJlbmRwb2ludCI6ICJodHRwOi8vdm0tMDA6NDMyMSIsCiAgICAiYWNjZXNzX2tleSI6ICI5NTY1VFZSMVFWTExFRzdVNFIxRCIsCiAgICAic2VjcmV0IjogImQ3b0FJQXZrNEdYeXpyd3Q2QVZ6bEZNQmNnRG53RVdMMHFDenE3cjUiCn1=" } ] cat zone-spec.yaml rgw_zone: my-secondary-zone rgw_realm_token: <token> placement: hosts: - ceph-node-1 - ceph-node-2 spec: rgw_frontend_port: 5500 cephadm shell --mount zone-spec.yaml:/var/lib/ceph/radosgw/zone-spec.yaml ceph mgr module enable rgw ceph rgw zone create -i /var/lib/ceph/radosgw/zone-spec.yaml radosgw-admin realm list { "default_info": "d07c00ef-9041-4f6e-8804-7d40240556ae", "realms": [ "myrealm" ] } bucket-name.domain-name.com address=/. HOSTNAME_OR_FQDN / HOST_IP_ADDRESS address=/.gateway-host01/192.168.122.75 USDTTL 604800 @ IN SOA gateway-host01. root.gateway-host01. ( 2 ; Serial 604800 ; Refresh 86400 ; Retry 2419200 ; Expire 604800 ) ; Negative Cache TTL ; @ IN NS gateway-host01. @ IN A 192.168.122.113 * IN CNAME @ ping mybucket. HOSTNAME ping mybucket.gateway-host01 radosgw-admin zonegroup get --rgw-zonegroup= ZONEGROUP_NAME > zonegroup.json radosgw-admin zonegroup get --rgw-zonegroup=us > zonegroup.json cp zonegroup.json zonegroup.backup.json cat zonegroup.json { "id": "d523b624-2fa5-4412-92d5-a739245f0451", "name": "asia", "api_name": "asia", "is_master": "true", "endpoints": [], "hostnames": [], "hostnames_s3website": [], "master_zone": "d2a3b90f-f4f3-4d38-ac1f-6463a2b93c32", "zones": [ { "id": "d2a3b90f-f4f3-4d38-ac1f-6463a2b93c32", "name": "india", "endpoints": [], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 11, "read_only": "false", "tier_type": "", "sync_from_all": "true", "sync_from": [], "redirect_zone": "" } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD" ] } ], "default_placement": "default-placement", "realm_id": "d7e2ad25-1630-4aee-9627-84f24e13017f", "sync_policy": { "groups": [] } } "hostnames": ["host01", "host02","host03"], radosgw-admin zonegroup set --rgw-zonegroup= ZONEGROUP_NAME --infile=zonegroup.json radosgw-admin zonegroup set --rgw-zonegroup=us --infile=zonegroup.json radosgw-admin period update --commit [client.rgw.node1] rgw frontends = beast ssl_endpoint=192.168.0.100:443 ssl_certificate=<path to SSL certificate> touch rgw.yml service_type: rgw service_id: SERVICE_ID service_name: SERVICE_NAME placement: hosts: - HOST_NAME spec: ssl: true rgw_frontend_ssl_certificate: CERT_HASH service_type: rgw service_id: foo service_name: rgw.foo placement: hosts: - host01 spec: ssl: true rgw_frontend_ssl_certificate: | -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEA+Cf4l9OagD6x67HhdCy4Asqw89Zz9ZuGbH50/7ltIMQpJJU0 gu9ObNtIoC0zabJ7n1jujueYgIpOqGnhRSvsGJiEkgN81NLQ9rqAVaGpadjrNLcM bpgqJCZj0vzzmtFBCtenpb5l/EccMFcAydGtGeLP33SaWiZ4Rne56GBInk6SATI/ JSKweGD1y5GiAWipBR4C74HiAW9q6hCOuSdp/2WQxWT3T1j2sjlqxkHdtInUtwOm j5Ism276IndeQ9hR3reFR8PJnKIPx73oTBQ7p9CMR1J4ucq9Ny0J12wQYT00fmJp -----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIEBTCCAu2gAwIBAgIUGfYFsj8HyA9Zv2l600hxzT8+gG4wDQYJKoZIhvcNAQEL BQAwgYkxCzAJBgNVBAYTAklOMQwwCgYDVQQIDANLQVIxDDAKBgNVBAcMA0JMUjEM MAoGA1UECgwDUkhUMQswCQYDVQQLDAJCVTEkMCIGA1UEAwwbY2VwaC1zc2wtcmhj czUtOGRjeHY2LW5vZGU1MR0wGwYJKoZIhvcNAQkBFg5hYmNAcmVkaGF0LmNvbTAe -----END CERTIFICATE----- ceph orch apply -i rgw.yml mkfs.ext4 nvme-drive-path mkfs.ext4 /dev/nvme0n1 mount /dev/nvme0n1 /mnt/nvme0n1/ mkdir <nvme-mount-path>/cache-directory-name mkdir /mnt/nvme0n1/rgw_datacache chmod a+rwx nvme-mount-path ; chmod a+rwx rgw_d3n_l1_datacache_persistent_path chmod a+rwx /mnt/nvme0n1 ; chmod a+rwx /mnt/nvme0n1/rgw_datacache/ "extra_container_args: "-v" "rgw_d3n_l1_datacache_persistent_path:rgw_d3n_l1_datacache_persistent_path" " cat rgw-spec.yml service_type: rgw service_id: rgw.test placement: hosts: host1 host2 extra_container_args: "-v" "/mnt/nvme0n1/rgw_datacache/:/mnt/nvme0n1/rgw_datacache/" "extra_container_args: "-v" "/mnt/nvme0n1/rgw_datacache/rgw1/:/mnt/nvme0n1/rgw_datacache/rgw1/" "-v" "/mnt/nvme0n1/rgw_datacache/rgw2/:/mnt/nvme0n1/rgw_datacache/rgw2/" " cat rgw-spec.yml service_type: rgw service_id: rgw.test placement: hosts: host1 host2 count_per_host: 2 extra_container_args: "-v" "/mnt/nvme0n1/rgw_datacache/rgw1/:/mnt/nvme0n1/rgw_datacache/rgw1/" "-v" "/mnt/nvme0n1/rgw_datacache/rgw2/:/mnt/nvme0n1/rgw_datacache/rgw2/" ceph orch apply -i rgw-spec.yml ceph config set <client.rgw> <CONF-OPTION> <VALUE> rgw_d3n_l1_datacache_persistent_path=/mnt/nvme/rgw_datacache/ rgw_d3n_l1_datacache_size=10737418240 fallocate -l 1G ./1G.dat s3cmd mb s3://bkt s3cmd put ./1G.dat s3://bkt s3cmd get s3://bkt/1G.dat /dev/shm/1G_get.dat download: 's3://bkt/1G.dat' -> './1G_get.dat' [1 of 1] 1073741824 of 1073741824 100% in 13s 73.94 MB/s done ls -lh /mnt/nvme/rgw_datacache rw-rr. 1 ceph ceph 1.0M Jun 2 06:18 cc7f967c-0021-43b2-9fdf-23858e868663.615391.1_shadow.ZCiCtMWeu_19wb100JIEZ-o4tv2IyA_1 s3cmd get s3://bkt/1G.dat /dev/shm/1G_get.dat download: 's3://bkt/1G.dat' -> './1G_get.dat' [1 of 1] 1073741824 of 1073741824 100% in 6s 155.07 MB/s done ceph config set client.rgw debug_rgw VALUE ceph config set client.rgw debug_rgw 20 ceph --admin-daemon /var/run/ceph/ceph-client.rgw. NAME .asok config set debug_rgw VALUE ceph --admin-daemon /var/run/ceph/ceph-client.rgw.rgw.asok config set debug_rgw 20 ceph config set global log_to_file true ceph config set global mon_cluster_log_to_file true ceph config set client.rgw OPTION VALUE ceph config set client.rgw rgw_enable_static_website true ceph config set client.rgw rgw_enable_apis s3,s3website ceph config set client.rgw rgw_dns_name objects-zonegroup.example.com ceph config set client.rgw rgw_dns_s3website_name objects-website-zonegroup.example.com ceph config set client.rgw rgw_resolve_cname true objects-zonegroup.domain.com. IN A 192.0.2.10 objects-zonegroup.domain.com. IN AAAA 2001:DB8::192:0:2:10 *.objects-zonegroup.domain.com. IN CNAME objects-zonegroup.domain.com. objects-website-zonegroup.domain.com. IN A 192.0.2.20 objects-website-zonegroup.domain.com. IN AAAA 2001:DB8::192:0:2:20 *.objects-website-zonegroup.domain.com. IN CNAME objects-website-zonegroup.domain.com. http://bucket1.objects-website-zonegroup.domain.com www.example.com. IN CNAME bucket2.objects-website-zonegroup.domain.com. http://www.example.com www.example.com. IN CNAME www.example.com.objects-website-zonegroup.domain.com. http://www.example.com www.example.com. IN A 192.0.2.20 www.example.com. IN AAAA 2001:DB8::192:0:2:20 http://www.example.com [root@host01 ~] touch ingress.yaml service_type: ingress 1 service_id: SERVICE_ID 2 placement: 3 hosts: - HOST1 - HOST2 - HOST3 spec: backend_service: SERVICE_ID virtual_ip: IP_ADDRESS / CIDR 4 frontend_port: INTEGER 5 monitor_port: INTEGER 6 virtual_interface_networks: 7 - IP_ADDRESS / CIDR ssl_cert: | 8 service_type: ingress service_id: rgw.foo placement: hosts: - host01.example.com - host02.example.com - host03.example.com spec: backend_service: rgw.foo virtual_ip: 192.168.1.2/24 frontend_port: 8080 monitor_port: 1967 virtual_interface_networks: - 10.10.0.0/16 ssl_cert: | -----BEGIN CERTIFICATE----- MIIEpAIBAAKCAQEA+Cf4l9OagD6x67HhdCy4Asqw89Zz9ZuGbH50/7ltIMQpJJU0 gu9ObNtIoC0zabJ7n1jujueYgIpOqGnhRSvsGJiEkgN81NLQ9rqAVaGpadjrNLcM bpgqJCZj0vzzmtFBCtenpb5l/EccMFcAydGtGeLP33SaWiZ4Rne56GBInk6SATI/ JSKweGD1y5GiAWipBR4C74HiAW9q6hCOuSdp/2WQxWT3T1j2sjlqxkHdtInUtwOm j5Ism276IndeQ9hR3reFR8PJnKIPx73oTBQ7p9CMR1J4ucq9Ny0J12wQYT00fmJp -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- MIIEBTCCAu2gAwIBAgIUGfYFsj8HyA9Zv2l600hxzT8+gG4wDQYJKoZIhvcNAQEL BQAwgYkxCzAJBgNVBAYTAklOMQwwCgYDVQQIDANLQVIxDDAKBgNVBAcMA0JMUjEM MAoGA1UECgwDUkhUMQswCQYDVQQLDAJCVTEkMCIGA1UEAwwbY2VwaC1zc2wtcmhj czUtOGRjeHY2LW5vZGU1MR0wGwYJKoZIhvcNAQkBFg5hYmNAcmVkaGF0LmNvbTAe -----END PRIVATE KEY----- service_type: ingress service_id: rgw.ssl # adjust to match your existing RGW service placement: hosts: - hostname1 - hostname2 spec: backend_service: rgw.rgw.ssl.ceph13 # adjust to match your existing RGW service virtual_ip: IP_ADDRESS/CIDR # ex: 192.168.20.1/24 frontend_port: INTEGER # ex: 443 monitor_port: INTEGER # ex: 1969 use_tcp_mode_over_rgw: True cephadm shell --mount ingress.yaml:/var/lib/ceph/radosgw/ingress.yaml ceph config set mgr mgr/cephadm/container_image_haproxy HAPROXY_IMAGE_ID ceph config set mgr mgr/cephadm/container_image_keepalived KEEPALIVED_IMAGE_ID ceph config set mgr mgr/cephadm/container_image_haproxy registry.redhat.io/rhceph/rhceph-haproxy-rhel9:latest ceph config set mgr mgr/cephadm/container_image_keepalived registry.redhat.io/rhceph/keepalived-rhel9:latest ceph orch apply -i /var/lib/ceph/radosgw/ingress.yaml ip addr show wget HOST_NAME wget host01.example.com <?xml version="1.0" encoding="UTF-8"?> <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Owner> <ID>anonymous</ID> <DisplayName></DisplayName> </Owner> <Buckets> </Buckets> </ListAllMyBucketsResult> cephadm shell ceph nfs export create rgw --cluster-id NFS_CLUSTER_NAME --pseudo-path PATH_FROM_ROOT --user-id USER_ID ceph nfs export create rgw --cluster-id cluster1 --pseudo-path root/testnfs1/ --user-id nfsuser mount -t nfs IP_ADDRESS:PATH_FROM_ROOT -osync MOUNT_POINT mount -t nfs 10.0.209.0:/root/testnfs1 -osync /mnt/mount1 cat ./haproxy.cfg global log 127.0.0.1 local2 chroot /var/lib/haproxy pidfile /var/run/haproxy.pid maxconn 7000 user haproxy group haproxy daemon stats socket /var/lib/haproxy/stats defaults mode http log global option httplog option dontlognull option http-server-close option forwardfor except 127.0.0.0/8 option redispatch retries 3 timeout http-request 10s timeout queue 1m timeout connect 10s timeout client 30s timeout server 30s timeout http-keep-alive 10s timeout check 10s timeout client-fin 1s timeout server-fin 1s maxconn 6000 listen stats bind 0.0.0.0:1936 mode http log global maxconn 256 clitimeout 10m srvtimeout 10m contimeout 10m timeout queue 10m JTH start stats enable stats hide-version stats refresh 30s stats show-node ## stats auth admin:password stats uri /haproxy?stats stats admin if TRUE frontend main bind *:5000 acl url_static path_beg -i /static /images /javascript /stylesheets acl url_static path_end -i .jpg .gif .png .css .js use_backend static if url_static default_backend app maxconn 6000 backend static balance roundrobin fullconn 6000 server app8 host01:8080 check maxconn 2000 server app9 host02:8080 check maxconn 2000 server app10 host03:8080 check maxconn 2000 backend app balance roundrobin fullconn 6000 server app8 host01:8080 check maxconn 2000 server app9 host02:8080 check maxconn 2000 server app10 host03:8080 check maxconn 2000 ceph config set osd osd_pool_default_pg_num 50 ceph config set osd osd_pool_default_pgp_num 50 radosgw-admin realm create --rgw-realm REALM_NAME --default radosgw-admin zonegroup rename --rgw-zonegroup default --zonegroup-new-name NEW_ZONE_GROUP_NAME radosgw-admin zone rename --rgw-zone default --zone-new-name NEW_ZONE_NAME --rgw-zonegroup NEW_ZONE_GROUP_NAME radosgw-admin zonegroup modify --api-name NEW_ZONE_GROUP_NAME --rgw-zonegroup NEW_ZONE_GROUP_NAME radosgw-admin zonegroup modify --rgw-realm REALM_NAME --rgw-zonegroup NEW_ZONE_GROUP_NAME --endpoints http://ENDPOINT --master --default radosgw-admin zone modify --rgw-realm REALM_NAME --rgw-zonegroup NEW_ZONE_GROUP_NAME --rgw-zone NEW_ZONE_NAME --endpoints http://ENDPOINT --master --default radosgw-admin user create --uid USER_ID --display-name DISPLAY_NAME --access-key ACCESS_KEY --secret SECRET_KEY --system radosgw-admin period update --commit ceph orch ls | grep rgw ceph config set client.rgw.SERVICE_NAME rgw_realm REALM_NAME ceph config set client.rgw.SERVICE_NAME rgw_zonegroup ZONE_GROUP_NAME ceph config set client.rgw.SERVICE_NAME rgw_zone PRIMARY_ZONE_NAME ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_realm test_realm ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zonegroup us ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zone us-east-1 systemctl restart ceph-radosgw@rgw.`hostname -s` ceph orch restart _RGW_SERVICE_NAME_ ceph orch restart rgw.rgwsvcid.mons-1.jwgwwp cephadm shell radosgw-admin realm pull --url= URL_TO_PRIMARY_ZONE_GATEWAY --access-key= ACCESS_KEY --secret-key= SECRET_KEY radosgw-admin realm pull --url=http://10.74.249.26:80 --access-key=LIPEYZJLTWXRKXS9LPJC --secret-key=IsAje0AVDNXNw48LjMAimpCpI7VaxJYSnfD0FFKQ radosgw-admin period pull --url= URL_TO_PRIMARY_ZONE_GATEWAY --access-key= ACCESS_KEY --secret-key= SECRET_KEY radosgw-admin period pull --url=http://10.74.249.26:80 --access-key=LIPEYZJLTWXRKXS9LPJC --secret-key=IsAje0AVDNXNw48LjMAimpCpI7VaxJYSnfD0FFKQ radosgw-admin zone create --rgw-zonegroup=_ZONE_GROUP_NAME_ --rgw-zone=_SECONDARY_ZONE_NAME_ --endpoints=http://_RGW_SECONDARY_HOSTNAME_:_RGW_PRIMARY_PORT_NUMBER_1_ --access-key=_SYSTEM_ACCESS_KEY_ --secret=_SYSTEM_SECRET_KEY_ [--read-only] radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east-2 --endpoints=http://rgw2:80 --access-key=LIPEYZJLTWXRKXS9LPJC --secret-key=IsAje0AVDNXNw48LjMAimpCpI7VaxJYSnfD0FFKQ radosgw-admin zone rm --rgw-zone=default ceph osd pool rm default.rgw.log default.rgw.log --yes-i-really-really-mean-it ceph osd pool rm default.rgw.meta default.rgw.meta --yes-i-really-really-mean-it ceph osd pool rm default.rgw.control default.rgw.control --yes-i-really-really-mean-it ceph osd pool rm default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it ceph osd pool rm default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it ceph config set client.rgw. SERVICE_NAME rgw_realm REALM_NAME ceph config set client.rgw. SERVICE_NAME rgw_zonegroup ZONE_GROUP_NAME ceph config set client.rgw. SERVICE_NAME rgw_zone SECONDARY_ZONE_NAME ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_realm test_realm ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zonegroup us ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zone us-east-2 radosgw-admin period update --commit radosgw-admin period update --commit systemctl list-units | grep ceph systemctl start ceph- FSID @ DAEMON_NAME systemctl enable ceph- FSID @ DAEMON_NAME systemctl start ceph-62a081a6-88aa-11eb-a367-001a4a000672@rgw.test_realm.us-east-2.host04.ahdtsw.service systemctl enable ceph-62a081a6-88aa-11eb-a367-001a4a000672@rgw.test_realm.us-east-2.host04.ahdtsw.service radosgw-admin zone create --rgw-zonegroup={ ZONE_GROUP_NAME } --rgw-zone={ ZONE_NAME } --endpoints={http:// FQDN : PORT },{http:// FQDN : PORT } --tier-type=archive radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east --endpoints={http://example.com:8080} --tier-type=archive radosgw-admin zone modify --rgw-zone archive --sync_from primary --sync_from_all false --sync-from-rm secondary radosgw-admin period update --commit ceph config set client.rgw rgw_max_objs_per_shard 50000 <?xml version="1.0" ?> <LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Rule> <ID>delete-1-days-az</ID> <Filter> <Prefix></Prefix> <ArchiveZone /> 1 </Filter> <Status>Enabled</Status> <Expiration> <Days>1</Days> </Expiration> </Rule> </LifecycleConfiguration> radosgw-admin lc get --bucket BUCKET_NAME radosgw-admin lc get --bucket test-bkt { "prefix_map": { "": { "status": true, "dm_expiration": true, "expiration": 0, "noncur_expiration": 2, "mp_expiration": 0, "transitions": {}, "noncur_transitions": {} } }, "rule_map": [ { "id": "Rule 1", "rule": { "id": "Rule 1", "prefix": "", "status": "Enabled", "expiration": { "days": "", "date": "" }, "noncur_expiration": { "days": "2", "date": "" }, "mp_expiration": { "days": "", "date": "" }, "filter": { "prefix": "", "obj_tags": { "tagset": {} }, "archivezone": "" 1 }, "transitions": {}, "noncur_transitions": {}, "dm_expiration": true } } ] } radosgw-admin bucket link --uid NEW_USER_ID --bucket BUCKET_NAME --yes-i-really-mean-it radosgw-admin bucket link --uid arcuser1 --bucket arc1-deleted-da473fbbaded232dc5d1e434675c1068 --yes-i-really-mean-it radosgw-admin zone modify --rgw-zone= ZONE_NAME --master --default radosgw-admin zone modify --rgw-zone= ZONE_NAME --master --default --read-only=false radosgw-admin period update --commit systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw radosgw-admin realm pull --url= URL_TO_PRIMARY_ZONE_GATEWAY --access-key= ACCESS_KEY --secret= SECRET_KEY radosgw-admin zone modify --rgw-zone= ZONE_NAME --master --default radosgw-admin period update --commit ceph orch restart SERVICE_TYPE ceph orch restart rgw radosgw-admin zone modify --rgw-zone= ZONE_NAME --read-only radosgw-admin zone modify --rgw-zone= ZONE_NAME --read-only radosgw-admin period update --commit ceph orch restart SERVICE_TYPE ceph orch restart rgw radosgw-admin realm create --rgw-realm= REALM_NAME --default radosgw-admin realm create --rgw-realm=ldc1 --default radosgw-admin zonegroup create --rgw-zonegroup= ZONE_GROUP_NAME --endpoints=http:// RGW_NODE_NAME :80 --rgw-realm= REALM_NAME --master --default radosgw-admin zonegroup create --rgw-zonegroup=ldc1zg --endpoints=http://rgw1:80 --rgw-realm=ldc1 --master --default radosgw-admin zone create --rgw-zonegroup= ZONE_GROUP_NAME --rgw-zone= ZONE_NAME --master --default --endpoints= HTTP_FQDN [, HTTP_FQDN ] radosgw-admin zone create --rgw-zonegroup=ldc1zg --rgw-zone=ldc1z --master --default --endpoints=http://rgw.example.com radosgw-admin period update --commit ceph orch apply rgw SERVICE_NAME --realm= REALM_NAME --zone= ZONE_NAME --placement=" NUMBER_OF_DAEMONS HOST_NAME_1 HOST_NAME_2 " ceph orch apply rgw rgw --realm=ldc1 --zone=ldc1z --placement="1 host01" ceph config set client.rgw. SERVICE_NAME rgw_realm REALM_NAME ceph config set client.rgw. SERVICE_NAME rgw_zonegroup ZONE_GROUP_NAME ceph config set client.rgw. SERVICE_NAME rgw_zone ZONE_NAME ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_realm ldc1 ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zonegroup ldc1zg ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zone ldc1z systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw radosgw-admin realm create --rgw-realm= REALM_NAME --default radosgw-admin realm create --rgw-realm=ldc2 --default radosgw-admin zonegroup create --rgw-zonegroup= ZONE_GROUP_NAME --endpoints=http:// RGW_NODE_NAME :80 --rgw-realm= REALM_NAME --master --default radosgw-admin zonegroup create --rgw-zonegroup=ldc2zg --endpoints=http://rgw2:80 --rgw-realm=ldc2 --master --default radosgw-admin zone create --rgw-zonegroup= ZONE_GROUP_NAME --rgw-zone= ZONE_NAME --master --default --endpoints= HTTP_FQDN [, HTTP_FQDN ] radosgw-admin zone create --rgw-zonegroup=ldc2zg --rgw-zone=ldc2z --master --default --endpoints=http://rgw.example.com radosgw-admin period update --commit ceph orch apply rgw SERVICE_NAME --realm= REALM_NAME --zone= ZONE_NAME --placement=" NUMBER_OF_DAEMONS HOST_NAME_1 HOST_NAME_2 " ceph orch apply rgw rgw --realm=ldc2 --zone=ldc2z --placement="1 host01" ceph config set client.rgw. SERVICE_NAME rgw_realm REALM_NAME ceph config set client.rgw. SERVICE_NAME rgw_zonegroup ZONE_GROUP_NAME ceph config set client.rgw. SERVICE_NAME rgw_zone ZONE_NAME ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_realm ldc2 ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zonegroup ldc2zg ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zone ldc2z systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw radosgw-admin realm create --rgw-realm= REPLICATED_REALM_1 --default radosgw-admin realm create --rgw-realm=rdc1 --default radosgw-admin zonegroup create --rgw-zonegroup= RGW_ZONE_GROUP --endpoints=http://_RGW_NODE_NAME :80 --rgw-realm=_RGW_REALM_NAME --master --default radosgw-admin zonegroup create --rgw-zonegroup=rdc1zg --endpoints=http://rgw1:80 --rgw-realm=rdc1 --master --default radosgw-admin zone create --rgw-zonegroup= RGW_ZONE_GROUP --rgw-zone=_MASTER_RGW_NODE_NAME --master --default --endpoints= HTTP_FQDN [, HTTP_FQDN ] radosgw-admin zone create --rgw-zonegroup=rdc1zg --rgw-zone=rdc1z --master --default --endpoints=http://rgw.example.com radosgw-admin user create --uid=" SYNCHRONIZATION_USER " --display-name="Synchronization User" --system radosgw-admin zone modify --rgw-zone= RGW_ZONE --access-key= ACCESS_KEY --secret= SECRET_KEY radosgw-admin user create --uid="synchronization-user" --display-name="Synchronization User" --system radosgw-admin zone modify --rgw-zone=rdc1zg --access-key=3QV0D6ZMMCJZMSCXJ2QJ --secret=VpvQWcsfI9OPzUCpR4kynDLAbqa1OIKqRB6WEnH8 radosgw-admin period update --commit ceph orch apply rgw SERVICE_NAME --realm= REALM_NAME --zone= ZONE_NAME --placement=" NUMBER_OF_DAEMONS HOST_NAME_1 HOST_NAME_2 " ceph orch apply rgw rgw --realm=rdc1 --zone=rdc1z --placement="1 host01" ceph config set client.rgw. SERVICE_NAME rgw_realm REALM_NAME ceph config set client.rgw. SERVICE_NAME rgw_zonegroup ZONE_GROUP_NAME ceph config set client.rgw. SERVICE_NAME rgw_zone ZONE_NAME ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_realm rdc1 ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zonegroup rdc1zg ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zone rdc1z systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw radosgw-admin realm pull --url=https://tower-osd1.cephtips.com --access-key= ACCESS_KEY --secret-key= SECRET_KEY radosgw-admin realm pull --url=https://tower-osd1.cephtips.com --access-key=3QV0D6ZMMCJZMSCXJ2QJ --secret-key=VpvQWcsfI9OPzUCpR4kynDLAbqa1OIKqRB6WEnH8 radosgw-admin period pull --url=https://tower-osd1.cephtips.com --access-key= ACCESS_KEY --secret-key= SECRET_KEY radosgw-admin period pull --url=https://tower-osd1.cephtips.com --access-key=3QV0D6ZMMCJZMSCXJ2QJ --secret-key=VpvQWcsfI9OPzUCpR4kynDLAbqa1OIKqRB6WEnH8 radosgw-admin zone create --rgw-zone= RGW_ZONE --rgw-zonegroup= RGW_ZONE_GROUP --endpoints=https://tower-osd4.cephtips.com --access-key=_ACCESS_KEY --secret-key= SECRET_KEY radosgw-admin zone create --rgw-zone=rdc2z --rgw-zonegroup=rdc1zg --endpoints=https://tower-osd4.cephtips.com --access-key=3QV0D6ZMMCJZMSCXJ2QJ --secret-key=VpvQWcsfI9OPzUCpR4kynDLAbqa1OIKqRB6WEnH8 radosgw-admin period update --commit ceph orch apply rgw SERVICE_NAME --realm= REALM_NAME --zone= ZONE_NAME --placement=" NUMBER_OF_DAEMONS HOST_NAME_1 HOST_NAME_2 " ceph orch apply rgw rgw --realm=rdc1 --zone=rdc2z --placement="1 host04" ceph config set client.rgw. SERVICE_NAME rgw_realm REALM_NAME ceph config set client.rgw. SERVICE_NAME rgw_zonegroup ZONE_GROUP_NAME ceph config set client.rgw. SERVICE_NAME rgw_zone ZONE_NAME ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_realm rdc1 ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zonegroup rdc1zg ceph config set client.rgw.rgwsvcid.mons-1.jwgwwp rgw_zone rdc2z systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw radosgw-admin sync status radosgw-admin sync status realm 59762f08-470c-46de-b2b1-d92c50986e67 (ldc2) zonegroup 7cf8daf8-d279-4d5c-b73e-c7fd2af65197 (ldc2zg) zone 034ae8d3-ae0c-4e35-8760-134782cb4196 (ldc2z) metadata sync no sync (zone is master) radosgw-admin sync status --rgw-realm RGW_REALM_NAME radosgw-admin sync status --rgw-realm rdc1 realm 73c7b801-3736-4a89-aaf8-e23c96e6e29d (rdc1) zonegroup d67cc9c9-690a-4076-89b8-e8127d868398 (rdc1zg) zone 67584789-375b-4d61-8f12-d1cf71998b38 (rdc2z) metadata sync syncing full sync: 0/64 shards incremental sync: 64/64 shards metadata is caught up with master data sync source: 705ff9b0-68d5-4475-9017-452107cec9a0 (rdc1z) syncing full sync: 0/128 shards incremental sync: 128/128 shards data is caught up with source realm 73c7b801-3736-4a89-aaf8-e23c96e6e29d (rdc1) zonegroup d67cc9c9-690a-4076-89b8-e8127d868398 (rdc1zg) zone 67584789-375b-4d61-8f12-d1cf71998b38 (rdc2z) metadata sync syncing full sync: 0/64 shards incremental sync: 64/64 shards metadata is caught up with master data sync source: 705ff9b0-68d5-4475-9017-452107cec9a0 (rdc1z) syncing full sync: 0/128 shards incremental sync: 128/128 shards data is caught up with source radosgw-admin user create --uid=" LOCAL_USER" --display-name="Local user" --rgw-realm=_REALM_NAME --rgw-zonegroup= ZONE_GROUP_NAME --rgw-zone= ZONE_NAME radosgw-admin user create --uid="local-user" --display-name="Local user" --rgw-realm=ldc1 --rgw-zonegroup=ldc1zg --rgw-zone=ldc1z radosgw-admin sync info --bucket=buck { "sources": [ { "id": "pipe1", "source": { "zone": "us-east", "bucket": "buck:115b12b3-....4409.1" }, "dest": { "zone": "us-west", "bucket": "buck:115b12b3-....4409.1" }, } ], "dests": [ { "id": "pipe1", "source": { "zone": "us-west", "bucket": "buck:115b12b3-....4409.1" }, "dest": { "zone": "us-east", "bucket": "buck:115b12b3-....4409.1" }, }, { "id": "pipe1", "source": { "zone": "us-west", "bucket": "buck:115b12b3-....4409.1" }, "dest": { "zone": "us-west-2", "bucket": "buck:115b12b3-....4409.1" }, } ], } radosgw-admin sync policy get --bucket= BUCKET_NAME radosgw-admin sync policy get --bucket=mybucket radosgw-admin sync group create --bucket= BUCKET_NAME --group-id= GROUP_ID --status=enabled | allowed | forbidden radosgw-admin sync group create --group-id=mygroup1 --status=enabled radosgw-admin bucket sync run radosgw-admin bucket sync run radosgw-admin sync group modify --bucket= BUCKET_NAME --group-id= GROUP_ID --status=enabled | allowed | forbidden radosgw-admin sync group modify --group-id=mygroup1 --status=forbidden radosgw-admin bucket sync run radosgw-admin bucket sync run radosgw-admin sync group get --bucket= BUCKET_NAME --group-id= GROUP_ID radosgw-admin sync group get --group-id=mygroup radosgw-admin sync group remove --bucket= BUCKET_NAME --group-id= GROUP_ID radosgw-admin sync group remove --group-id=mygroup radosgw-admin sync group flow create --bucket= BUCKET_NAME --group-id= GROUP_ID --flow-id= FLOW_ID --flow-type=directional --source-zone= SOURCE_ZONE --dest-zone= DESTINATION_ZONE radosgw-admin sync group flow create --bucket= BUCKET_NAME --group-id= GROUP_ID --flow-id= FLOW_ID --flow-type=symmetrical --zones= ZONE_NAME1 , ZONE_NAME2 radosgw-admin sync group flow remove --bucket= BUCKET_NAME --group-id= GROUP_ID --flow-id= FLOW_ID --flow-type=directional --source-zone= SOURCE_ZONE --dest-zone= DESTINATION_ZONE radosgw-admin sync group flow remove --bucket= BUCKET_NAME --group-id= GROUP_ID --flow-id= FLOW_ID --flow-type=symmetrical --zones= ZONE_NAME1 , ZONE_NAME2 radosgw-admin sync group flow remove --group-id= GROUP_ID --flow-id= FLOW_ID --flow-type=symmetrical --zones= ZONE_NAME1 , ZONE_NAME2 radosgw-admin sync group pipe create --bucket= BUCKET_NAME --group-id= GROUP_ID --pipe-id= PIPE_ID --source-zones=' ZONE_NAME ',' ZONE_NAME2 '... --source-bucket= SOURCE_BUCKET --source-bucket-id= SOURCE_BUCKET_ID --dest-zones=' ZONE_NAME ',' ZONE_NAME2 '... --dest-bucket= DESTINATION_BUCKET --dest-bucket-id= DESTINATION_BUCKET_ID --prefix= SOURCE_PREFIX --prefix-rm --tags-add= KEY1=VALUE1 , KEY2=VALUE2 ,.. --tags-rm= KEY1=VALUE1 , KEY2=VALUE2 , ... --dest-owner= OWNER_ID --storage-class= STORAGE_CLASS --mode= USER --uid= USER_ID radosgw-admin sync group pipe modify --bucket= BUCKET_NAME --group-id= GROUP_ID --pipe-id= PIPE_ID --source-zones=' ZONE_NAME ',' ZONE_NAME2 '... --source-bucket= SOURCE_BUCKET1 --source-bucket-id= SOURCE_BUCKET_ID --dest-zones=' ZONE_NAME ',' ZONE_NAME2 '... --dest-bucket= DESTINATION_BUCKET1 --dest-bucket-id=_DESTINATION_BUCKET-ID radosgw-admin sync group pipe modify --group-id=zonegroup --pipe-id=pipe --dest-zones='primary','secondary','tertiary' --source-zones='primary','secondary','tertiary' --source-bucket=pri-bkt-1 --dest-bucket=pri-bkt-1 radosgw-admin sync group pipe remove --bucket= BUCKET_NAME --group-id= GROUP_ID --pipe-id= PIPE_ID --source-zones=' ZONE_NAME ',' ZONE_NAME2 '... --source-bucket= SOURCE_BUCKET , --source-bucket-id= SOURCE_BUCKET_ID --dest-zones=' ZONE_NAME ',' ZONE_NAME2 '... --dest-bucket= DESTINATION_BUCKET --dest-bucket-id= DESTINATION_BUCKET-ID radosgw-admin sync group pipe remove --group-id=zonegroup --pipe-id=pipe --dest-zones='primary','secondary','tertiary' --source-zones='primary','secondary','tertiary' --source-bucket=pri-bkt-1 --dest-bucket=pri-bkt-1 radosgw-admin sync group pipe remove --bucket= BUCKET_NAME --group-id= GROUP_ID --pipe-id= PIPE_ID radosgw-admin sync group pipe remove -bucket-name=mybuck --group-id=zonegroup --pipe-id=pipe radosgw-admin sync info --bucket= BUCKET_NAME --effective-zone-name= ZONE_NAME radosgw-admin sync info radosgw-admin sync group create --group-id=group1 --status=allowed radosgw-admin sync group flow create --group-id=group1 --flow-id=flow-mirror --flow-type=symmetrical --zones=us-east,us-west radosgw-admin sync group pipe create --group-id=group1 --pipe-id=pipe1 --source-zones='*' --source-bucket='*' --dest-zones='*' --dest-bucket='*' radosgw-admin sync group modify --group-id=group1 --status=enabled radosgw-admin period update --commit radosgw-admin sync info -bucket buck { "sources": [ { "id": "pipe1", "source": { "zone": "us-east", "bucket": "buck:115b12b3-....4409.1" }, "dest": { "zone": "us-west", "bucket": "buck:115b12b3-....4409.1" }, } ], "dests": [ { "id": "pipe1", "source": { "zone": "us-west", "bucket": "buck:115b12b3-....4409.1" }, "dest": { "zone": "us-east", "bucket": "buck:115b12b3-....4409.1" }, } ], } radosgw-admin sync group create --group-id= GROUP_ID --status=allowed radosgw-admin sync group create --group-id=group1 --status=allowed radosgw-admin sync group flow create --group-id= GROUP_ID --flow-id= FLOW_ID --flow-type=directional --source-zone= SOURCE_ZONE_NAME --dest-zone= DESTINATION_ZONE_NAME radosgw-admin sync group flow create --group-id=group1 --flow-id=us-west-backup --flow-type=directional --source-zone=us-west --dest-zone=us-west-2 radosgw-admin sync group pipe create --group-id= GROUP_ID --pipe-id= PIPE_ID --source-zones=' SOURCE_ZONE_NAME ' --dest-zones=' DESTINATION_ZONE_NAME ' radosgw-admin sync group pipe create --group-id=group1 --pipe-id=pipe1 --source-zones='us-west' --dest-zones='us-west-2' radosgw-admin period update --commit radosgw-admin sync info radosgw-admin sync group create --group-id= GROUP_ID --status=allowed --bucket= BUCKET_NAME radosgw-admin sync group create --group-id=group1 --status=allowed --bucket=buck radosgw-admin sync group flow create --bucket-name= BUCKET_NAME --group-id= GROUP_ID --flow-id= FLOW_ID --flow-type=directional --source-zone= SOURCE_ZONE_NAME --dest-zone= DESTINATION_ZONE_NAME radosgw-admin sync group flow create --bucket-name=buck --group-id=group1 --flow-id=us-west-backup --flow-type=directional --source-zone=us-west --dest-zone=us-west-2 radosgw-admin sync group pipe create --group-id= GROUP_ID --bucket-name= BUCKET_NAME --pipe-id= PIPE_ID --source-zones=' SOURCE_ZONE_NAME ' --dest-zones=' DESTINATION_ZONE_NAME ' radosgw-admin sync group pipe create --group-id=group1 --bucket-name=buck --pipe-id=pipe1 --source-zones='us-west' --dest-zones='us-west-2' radosgw-admin sync info --bucket-name= BUCKET_NAME radosgw-admin sync group modify --group-id=group1 --status=allowed radosgw-admin period update --commit radosgw-admin sync group create --bucket=buck --group-id=buck-default --status=enabled radosgw-admin sync group pipe create --bucket=buck --group-id=buck-default --pipe-id=pipe1 --source-zones='*' --dest-zones='*' radosgw-admin bucket sync info --bucket buck realm 33157555-f387-44fc-b4b4-3f9c0b32cd66 (india) zonegroup 594f1f63-de6f-4e1e-90b6-105114d7ad55 (shared) zone ffaa5ba4-c1bd-4c17-b176-2fe34004b4c5 (primary) bucket :buck[ffaa5ba4-c1bd-4c17-b176-2fe34004b4c5.16191.1] source zone e0e75beb-4e28-45ff-8d48-9710de06dcd0 bucket :buck[ffaa5ba4-c1bd-4c17-b176-2fe34004b4c5.16191.1] radosgw-admin sync info --bucket buck { "id": "pipe1", "source": { "zone": "secondary", "bucket": "buck:ffaa5ba4-c1bd-4c17-b176-2fe34004b4c5.16191.1" }, "dest": { "zone": "primary", "bucket": "buck:ffaa5ba4-c1bd-4c17-b176-2fe34004b4c5.16191.1" }, "params": { "source": { "filter": { "tags": [] } }, "dest": {}, "priority": 0, "mode": "system", "user": "" } }, { "id": "pipe1", "source": { "zone": "primary", "bucket": "buck:ffaa5ba4-c1bd-4c17-b176-2fe34004b4c5.16191.1" }, "dest": { "zone": "secondary", "bucket": "buck:ffaa5ba4-c1bd-4c17-b176-2fe34004b4c5.16191.1" }, "params": { "source": { "filter": { "tags": [] } }, "dest": {}, "priority": 0, "mode": "system", "user": "" } } radosgw-admin sync group create --bucket= BUCKET_NAME --group-id= GROUP_ID --status=enabled radosgw-admin sync group create --bucket=buck4 --group-id=buck4-default --status=enabled radosgw-admin sync group pipe create --bucket-name= BUCKET_NAME --group-id= GROUP_ID --pipe-id= PIPE_ID --source-zones= SOURCE_ZONE_NAME --source-bucket= SOURCE_BUCKET_NAME --dest-zones= DESTINATION_ZONE_NAME radosgw-admin sync group pipe create --bucket=buck4 --group-id=buck4-default --pipe-id=pipe1 --source-zones='*' --source-bucket=buck5 --dest-zones='*' radosgw-admin sync group pipe modify --bucket=buck4 --group-id=buck4-default --pipe-id=pipe1 --source-zones=us-west --source-bucket=buck5 --dest-zones='*' radosgw-admin sync info --bucket-name= BUCKET_NAME radosgw-admin sync info --bucket=buck4 { "sources": [], "dests": [], "hints": { "sources": [], "dests": [ "buck4:115b12b3-....14433.2" ] }, "resolved-hints-1": { "sources": [], "dests": [ { "id": "pipe1", "source": { "zone": "us-west", "bucket": "buck5" }, "dest": { "zone": "us-east", "bucket": "buck4:115b12b3-....14433.2" }, }, { "id": "pipe1", "source": { "zone": "us-west", "bucket": "buck5" }, "dest": { "zone": "us-west-2", "bucket": "buck4:115b12b3-....14433.2" }, } ] }, "resolved-hints": { "sources": [], "dests": [] } radosgw-admin sync group create --bucket= BUCKET_NAME --group-id= GROUP_ID --status=enabled radosgw-admin sync group create --bucket=buck6 --group-id=buck6-default --status=enabled radosgw-admin sync group pipe create --bucket-name= BUCKET_NAME --group-id= GROUP_ID --pipe-id= PIPE_ID --source-zones= SOURCE_ZONE_NAME --dest-zones= DESTINATION_ZONE_NAME --dest-bucket= DESTINATION_BUCKET_NAME radosgw-admin sync group pipe create --bucket=buck6 --group-id=buck6-default --pipe-id=pipe1 --source-zones='*' --dest-zones='*' --dest-bucket=buck5 radosgw-admin sync group pipe modify --bucket=buck6 --group-id=buck6-default --pipe-id=pipe1 --source-zones='*' --dest-zones='us-west' --dest-bucket=buck5 radosgw-admin sync info --bucket-name= BUCKET_NAME radosgw-admin sync info --bucket buck5 { "sources": [], "dests": [ { "id": "pipe1", "source": { "zone": "us-west", "bucket": "buck6:c7887c5b-f6ff-4d5f-9736-aa5cdb4a15e8.20493.4" }, "dest": { "zone": "us-east", "bucket": "buck5" }, "params": { "source": { "filter": { "tags": [] } }, "dest": {}, "priority": 0, "mode": "system", "user": "s3cmd" } }, ], "hints": { "sources": [], "dests": [ "buck5" ] }, "resolved-hints-1": { "sources": [], "dests": [] }, "resolved-hints": { "sources": [], "dests": [] } } radosgw-admin sync group create --bucket= BUCKET_NAME --group-id= GROUP_ID --status=enabled radosgw-admin sync group create --bucket=buck1 --group-id=buck8-default --status=enabled radosgw-admin sync group pipe create --bucket= BUCKET_NAME --group-id= GROUP_ID --pipe-id= PIPE_ID --tags-add= KEY1 = VALUE1 , KEY2 = VALUE2 --source-zones=' ZONE_NAME1 ',' ZONE_NAME2 ' --dest-zones=' ZONE_NAME1 ',' ZONE_NAME2 ' radosgw-admin sync group pipe create --bucket=buck1 --group-id=buck1-default --pipe-id=pipe-tags --tags-add=color=blue,color=red --source-zones='*' --dest-zones='*' radosgw-admin sync group pipe create --bucket= BUCKET_NAME --group-id= GROUP_ID --pipe-id= PIPE_ID --prefix= PREFIX --source-zones=' ZONE_NAME1 ',' ZONE_NAME2 ' --dest-zones=' ZONE_NAME1 ',' ZONE_NAME2 ' radosgw-admin sync group pipe create --bucket=buck1 --group-id=buck1-default --pipe-id=pipe-prefix --prefix=foo/ --source-zones='*' --dest-zones='*' \ radosgw-admin sync info --bucket= BUCKET_NAME radosgw-admin sync info --bucket=buck1 radosgw-admin sync group modify --group-id buck-default --status forbidden --bucket buck { "groups": [ { "id": "buck-default", "data_flow": {}, "pipes": [ { "id": "pipe1", "source": { "bucket": "*", "zones": [ "*" ] }, "dest": { "bucket": "*", "zones": [ "*" ] }, "params": { "source": { "filter": { "tags": [] } }, "dest": {}, "priority": 0, "mode": "system", } } ], "status": "forbidden" } ] } radosgw-admin sync info --bucket buck { "sources": [], "dests": [], "hints": { "sources": [], "dests": [] }, "resolved-hints-1": { "sources": [], "dests": [] }, "resolved-hints": { "sources": [], "dests": [] } } radosgw-admin realm create --rgw-realm= REALM_NAME radosgw-admin realm create --rgw-realm=test_realm radosgw-admin realm default --rgw-realm= REALM_NAME radosgw-admin realm default --rgw-realm=test_realm1 radosgw-admin realm default --rgw-realm=test_realm radosgw-admin realm delete --rgw-realm= REALM_NAME radosgw-admin realm delete --rgw-realm=test_realm radosgw-admin realm get --rgw-realm= REALM_NAME radosgw-admin realm get --rgw-realm=test_realm >filename.json { "id": "0a68d52e-a19c-4e8e-b012-a8f831cb3ebc", "name": "test_realm", "current_period": "b0c5bbef-4337-4edd-8184-5aeab2ec413b", "epoch": 1 } radosgw-admin realm set --rgw-realm= REALM_NAME --infile= IN_FILENAME radosgw-admin realm set --rgw-realm=test_realm --infile=filename.json radosgw-admin realm list radosgw-admin realm list-periods radosgw-admin realm pull --url= URL_TO_MASTER_ZONE_GATEWAY --access-key= ACCESS_KEY --secret= SECRET_KEY radosgw-admin realm rename --rgw-realm= REALM_NAME --realm-new-name= NEW_REALM_NAME radosgw-admin realm rename --rgw-realm=test_realm --realm-new-name=test_realm2 radosgw-admin period update --commit radosgw-admin period update --commit radosgw-admin zonegroup create --rgw-zonegroup= ZONE_GROUP_NAME [--rgw-realm= REALM_NAME ] [--master] radosgw-admin zonegroup create --rgw-zonegroup=zonegroup1 --rgw-realm=test_realm --default zonegroup modify --rgw-zonegroup= ZONE_GROUP_NAME radosgw-admin zonegroup modify --rgw-zonegroup=zonegroup1 radosgw-admin zonegroup default --rgw-zonegroup= ZONE_GROUP_NAME radosgw-admin zonegroup default --rgw-zonegroup=zonegroup2 radosgw-admin period update --commit radosgw-admin period update --commit radosgw-admin zonegroup default --rgw-zonegroup=us radosgw-admin period update --commit radosgw-admin zonegroup add --rgw-zonegroup= ZONE_GROUP_NAME --rgw-zone= ZONE_NAME radosgw-admin period update --commit radosgw-admin zonegroup remove --rgw-zonegroup= ZONE_GROUP_NAME --rgw-zone= ZONE_NAME radosgw-admin period update --commit radosgw-admin zonegroup rename --rgw-zonegroup= ZONE_GROUP_NAME --zonegroup-new-name= NEW_ZONE_GROUP_NAME radosgw-admin period update --commit radosgw-admin zonegroup delete --rgw-zonegroup= ZONE_GROUP_NAME radosgw-admin period update --commit radosgw-admin zonegroup list { "default_info": "90b28698-e7c3-462c-a42d-4aa780d24eda", "zonegroups": [ "us" ] } radosgw-admin zonegroup get [--rgw-zonegroup= ZONE_GROUP_NAME ] { "id": "90b28698-e7c3-462c-a42d-4aa780d24eda", "name": "us", "api_name": "us", "is_master": "true", "endpoints": [ "http:\/\/rgw1:80" ], "hostnames": [], "hostnames_s3website": [], "master_zone": "9248cab2-afe7-43d8-a661-a40bf316665e", "zones": [ { "id": "9248cab2-afe7-43d8-a661-a40bf316665e", "name": "us-east", "endpoints": [ "http:\/\/rgw1" ], "log_meta": "true", "log_data": "true", "bucket_index_max_shards": 11, "read_only": "false" }, { "id": "d1024e59-7d28-49d1-8222-af101965a939", "name": "us-west", "endpoints": [ "http:\/\/rgw2:80" ], "log_meta": "false", "log_data": "true", "bucket_index_max_shards": 11, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "ae031368-8715-4e27-9a99-0c9468852cfe" } radosgw-admin zonegroup set --infile zonegroup.json radosgw-admin period update --commit { "zonegroups": [ { "key": "90b28698-e7c3-462c-a42d-4aa780d24eda", "val": { "id": "90b28698-e7c3-462c-a42d-4aa780d24eda", "name": "us", "api_name": "us", "is_master": "true", "endpoints": [ "http:\/\/rgw1:80" ], "hostnames": [], "hostnames_s3website": [], "master_zone": "9248cab2-afe7-43d8-a661-a40bf316665e", "zones": [ { "id": "9248cab2-afe7-43d8-a661-a40bf316665e", "name": "us-east", "endpoints": [ "http:\/\/rgw1" ], "log_meta": "true", "log_data": "true", "bucket_index_max_shards": 11, "read_only": "false" }, { "id": "d1024e59-7d28-49d1-8222-af101965a939", "name": "us-west", "endpoints": [ "http:\/\/rgw2:80" ], "log_meta": "false", "log_data": "true", "bucket_index_max_shards": 11, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "ae031368-8715-4e27-9a99-0c9468852cfe" } } ], "master_zonegroup": "90b28698-e7c3-462c-a42d-4aa780d24eda", "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } } radosgw-admin zonegroup-map set --infile zonegroupmap.json radosgw-admin period update --commit radosgw-admin zone create --rgw-zone= ZONE_NAME [--zonegroup= ZONE_GROUP_NAME ] [--endpoints= ENDPOINT_PORT [,<endpoint:port>] [--master] [--default] --access-key ACCESS_KEY --secret SECRET_KEY radosgw-admin period update --commit radosgw-admin period update --commit radosgw-admin zonegroup remove --rgw-zonegroup= ZONE_GROUP_NAME --rgw-zone= ZONE_NAME radosgw-admin period update --commit radosgw-admin zone delete --rgw-zone= ZONE_NAME radosgw-admin period update --commit ceph osd pool delete DELETED_ZONE_NAME .rgw.control DELETED_ZONE_NAME .rgw.control --yes-i-really-really-mean-it ceph osd pool delete DELETED_ZONE_NAME .rgw.data.root DELETED_ZONE_NAME .rgw.data.root --yes-i-really-really-mean-it ceph osd pool delete DELETED_ZONE_NAME .rgw.log DELETED_ZONE_NAME .rgw.log --yes-i-really-really-mean-it ceph osd pool delete DELETED_ZONE_NAME .rgw.users.uid DELETED_ZONE_NAME .rgw.users.uid --yes-i-really-really-mean-it radosgw-admin zone modify [options] --access-key=<key> --secret/--secret-key=<key> --master --default --endpoints=<list> radosgw-admin period update --commit radosgw-admin zone list radosgw-admin zone get [--rgw-zone= ZONE_NAME ] { "domain_root": ".rgw", "control_pool": ".rgw.control", "gc_pool": ".rgw.gc", "log_pool": ".log", "intent_log_pool": ".intent-log", "usage_log_pool": ".usage", "user_keys_pool": ".users", "user_email_pool": ".users.email", "user_swift_pool": ".users.swift", "user_uid_pool": ".users.uid", "system_key": { "access_key": "", "secret_key": ""}, "placement_pools": [ { "key": "default-placement", "val": { "index_pool": ".rgw.buckets.index", "data_pool": ".rgw.buckets"} } ] } radosgw-admin zone set --rgw-zone=test-zone --infile zone.json radosgw-admin period update --commit radosgw-admin zone rename --rgw-zone= ZONE_NAME --zone-new-name= NEW_ZONE_NAME radosgw-admin period update --commit firewall-cmd --zone=public --add-port=636/tcp firewall-cmd --zone=public --add-port=636/tcp --permanent certutil -d /etc/openldap/certs -A -t "TC,," -n "msad-frog-MSAD-FROG-CA" -i /path/to/ldap.pem setsebool -P httpd_can_network_connect on chmod 644 /etc/openldap/certs/* ldapwhoami -H ldaps://rh-directory-server.example.com -d 9 radosgw-admin metadata list user ldapsearch -x -D "uid=ceph,ou=People,dc=example,dc=com" -W -H ldaps://example.com -b "ou=People,dc=example,dc=com" -s sub 'uid=ceph' ceph config set client.rgw OPTION VALUE ceph config set client.rgw rgw_ldap_secret /etc/bindpass service_type: rgw service_id: rgw.1 service_name: rgw.rgw.1 placement: label: rgw extra_container_args: - -v - /etc/bindpass:/etc/bindpass ceph config set client.rgw OPTION VALUE ceph config set client.rgw rgw_ldap_uri ldaps://:636 ceph config set client.rgw rgw_ldap_binddn "ou=poc,dc=example,dc=local" ceph config set client.rgw rgw_ldap_searchdn "ou=poc,dc=example,dc=local" ceph config set client.rgw rgw_ldap_dnattr "uid" ceph config set client.rgw rgw_s3_auth_use_ldap true systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw "objectclass=inetorgperson" "(&(uid=joe)(objectclass=inetorgperson))" "(&(uid=@USERNAME@)(memberOf=cn=ceph-users,ou=groups,dc=mycompany,dc=com))" export RGW_ACCESS_KEY_ID=" USERNAME " export RGW_SECRET_ACCESS_KEY=" PASSWORD " radosgw-token --encode --ttype=ldap radosgw-token --encode --ttype=ad export RGW_ACCESS_KEY_ID="ewogICAgIlJHV19UT0tFTiI6IHsKICAgICAgICAidmVyc2lvbiI6IDEsCiAgICAgICAgInR5cGUiOiAibGRhcCIsCiAgICAgICAgImlkIjogImNlcGgiLAogICAgICAgICJrZXkiOiAiODAwI0dvcmlsbGEiCiAgICB9Cn0K" cat .aws/credentials [default] aws_access_key_id = ewogICaGbnjlwe9UT0tFTiI6IHsKICAgICAgICAidmVyc2lvbiI6IDEsCiAgICAgICAgInR5cGUiOiAiYWQiLAogICAgICAgICJpZCI6ICJjZXBoIiwKICAgICAgICAia2V5IjogInBhc3M0Q2VwaCIKICAgIH0KfQo= aws_secret_access_key = aws s3 ls --endpoint http://host03 2023-12-11 17:08:50 mybucket 2023-12-24 14:55:44 mybucket2 radosgw-admin user info --uid dir1 { "user_id": "dir1", "display_name": "dir1", "email": "", "suspended": 0, "max_buckets": 1000, "subusers": [], "keys": [], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "default_storage_class": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "ldap", "mfa_ids": [] } radosgw-admin metadata list user ldapsearch -x -D "uid=ceph,ou=People,dc=example,dc=com" -W -H ldaps://example.com -b "ou=People,dc=example,dc=com" -s sub 'uid=ceph' ceph config set client.rgw OPTION VALUE ceph config set client.rgw rgw_ldap_secret /etc/bindpass service_type: rgw service_id: rgw.1 service_name: rgw.rgw.1 placement: label: rgw extra_container_args: - -v - /etc/bindpass:/etc/bindpass ceph config set client.rgw OPTION VALUE ceph config set client.rgw rgw_ldap_uri ldaps://_FQDN_:636 ceph config set client.rgw rgw_ldap_binddn "_BINDDN_" ceph config set client.rgw rgw_ldap_searchdn "_SEARCHDN_" ceph config set client.rgw rgw_ldap_dnattr "cn" ceph config set client.rgw rgw_s3_auth_use_ldap true rgw_ldap_binddn "uid=ceph,cn=users,cn=accounts,dc=example,dc=com" rgw_ldap_searchdn "cn=users,cn=accounts,dc=example,dc=com" systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw export RGW_ACCESS_KEY_ID=" USERNAME " export RGW_SECRET_ACCESS_KEY=" PASSWORD " radosgw-token --encode --ttype=ldap radosgw-token --encode --ttype=ad export RGW_ACCESS_KEY_ID="ewogICAgIlJHV19UT0tFTiI6IHsKICAgICAgICAidmVyc2lvbiI6IDEsCiAgICAgICAgInR5cGUiOiAibGRhcCIsCiAgICAgICAgImlkIjogImNlcGgiLAogICAgICAgICJrZXkiOiAiODAwI0dvcmlsbGEiCiAgICB9Cn0K" cat .aws/credentials [default] aws_access_key_id = ewogICaGbnjlwe9UT0tFTiI6IHsKICAgICAgICAidmVyc2lvbiI6IDEsCiAgICAgICAgInR5cGUiOiAiYWQiLAogICAgICAgICJpZCI6ICJjZXBoIiwKICAgICAgICAia2V5IjogInBhc3M0Q2VwaCIKICAgIH0KfQo= aws_secret_access_key = aws s3 ls --endpoint http://host03 2023-12-11 17:08:50 mybucket 2023-12-24 14:55:44 mybucket2 radosgw-admin user info --uid dir1 { "user_id": "dir1", "display_name": "dir1", "email": "", "suspended": 0, "max_buckets": 1000, "subusers": [], "keys": [], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "default_storage_class": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "ldap", "mfa_ids": [] } openstack service create --name=swift --description="Swift Service" object-store openstack endpoint create --region REGION_NAME swift admin " URL " openstack endpoint create --region REGION_NAME swift public " URL " openstack endpoint create --region REGION_NAME swift internal " URL " openstack endpoint create --region us-west swift admin "http://radosgw.example.com:8080/swift/v1" openstack endpoint create --region us-west swift public "http://radosgw.example.com:8080/swift/v1" openstack endpoint create --region us-west swift internal "http://radosgw.example.com:8080/swift/v1" openstack endpoint list --service=swift openstack endpoint show ENDPOINT_ID mkdir /var/ceph/nss openssl x509 -in /etc/keystone/ssl/certs/ca.pem -pubkey | certutil -d /var/ceph/nss -A -n ca -t "TCu,Cu,Tuw" openssl x509 -in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | certutil -A -d /var/ceph/nss -n signing_cert -t "P,P,P" ceph config set client.rgw nss_db_path "/var/lib/ceph/radosgw/ceph-rgw.rgw01/nss" ceph config set client.rgw rgw_keystone_verify_ssl TRUE / FALSE ceph config set client.rgw rgw_s3_auth_use_keystone TRUE / FALSE ceph config set client.rgw rgw_keystone_api_version API_VERSION ceph config set client.rgw rgw_keystone_url KEYSTONE_URL : ADMIN_PORT ceph config set client.rgw rgw_keystone_accepted_roles ACCEPTED_ROLES_ ceph config set client.rgw rgw_keystone_accepted_admin_roles ACCEPTED_ADMIN_ROLES ceph config set client.rgw rgw_keystone_admin_domain default ceph config set client.rgw rgw_keystone_admin_project SERVICE_NAME ceph config set client.rgw rgw_keystone_admin_user KEYSTONE_TENANT_USER_NAME ceph config set client.rgw rgw_keystone_admin_password KEYSTONE_TENANT_USER_PASSWORD ceph config set client.rgw rgw_keystone_implicit_tenants KEYSTONE_IMPLICIT_TENANT_NAME ceph config set client.rgw rgw_swift_versioning_enabled TRUE / FALSE ceph config set client.rgw rgw_swift_enforce_content_length TRUE / FALSE ceph config set client.rgw rgw_swift_account_in_url TRUE / FALSE ceph config set client.rgw rgw_trust_forwarded_https TRUE / FALSE ceph config set client.rgw rgw_max_attr_name_len MAXIMUM_LENGTH_OF_METADATA_NAMES ceph config set client.rgw rgw_max_attrs_num_in_req MAXIMUM_NUMBER_OF_METADATA_ITEMS ceph config set client.rgw rgw_max_attr_size MAXIMUM_LENGTH_OF_METADATA_VALUE ceph config set client.rgw rgw_keystone_accepted_reader_roles SwiftSystemReader ceph config set client.rgw rgw_keystone_verify_ssl false ceph config set client.rgw rgw_s3_auth_use_keystone true ceph config set client.rgw rgw_keystone_api_version 3 ceph config set client.rgw rgw_keystone_url http://<public Keystone endpoint>:5000/ ceph config set client.rgw rgw_keystone_accepted_roles 'member, Member, admin' ceph config set client.rgw rgw_keystone_accepted_admin_roles 'ResellerAdmin, swiftoperator' ceph config set client.rgw rgw_keystone_admin_domain default ceph config set client.rgw rgw_keystone_admin_project service ceph config set client.rgw rgw_keystone_admin_user swift ceph config set client.rgw rgw_keystone_admin_password password ceph config set client.rgw rgw_keystone_implicit_tenants true ceph config set client.rgw rgw_swift_versioning_enabled true ceph config set client.rgw rgw_swift_enforce_content_length true ceph config set client.rgw rgw_swift_account_in_url true ceph config set client.rgw rgw_trust_forwarded_https true ceph config set client.rgw rgw_max_attr_name_len 128 ceph config set client.rgw rgw_max_attrs_num_in_req 90 ceph config set client.rgw rgw_max_attr_size 1024 ceph config set client.rgw rgw_keystone_accepted_reader_roles SwiftSystemReader systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw grubby --update-kernel=ALL --args="intel_iommu=on" dnf install -y qatlib-service qatlib qatzip qatengine usermod -aG qat root cat /etc/sysconfig/qat ServicesEnabled=asym POLICY=8 cat /etc/sysconfig/qat ServicesEnabled=dc POLICY=8 cat /etc/sysconfig/qat ServicesEnabled=asym,dc POLICY=8 sudo vim /etc/security/limits.conf root - memlock 500000 ceph - memlock 500000 sudo su -l USDUSER systemctl enable qat systemctl reboot service_type: rgw service_id: rgw_qat placement: label: rgw extra_container_args: - "-v /etc/group:/etc/group:ro" - "--group-add=keep-groups" - "--cap-add=SYS_ADMIN" - "--cap-add=SYS_PTRACE" - "--cap-add=IPC_LOCK" - "--security-opt seccomp=unconfined" - "--ulimit memlock=209715200:209715200" - "--device=/dev/qat_adf_ctl:/dev/qat_adf_ctl" - "--device=/dev/vfio/vfio:/dev/vfio/vfio" - "--device=/dev/vfio/333:/dev/vfio/333" - "--device=/dev/vfio/334:/dev/vfio/334" - "--device=/dev/vfio/335:/dev/vfio/335" - "--device=/dev/vfio/336:/dev/vfio/336" - "--device=/dev/vfio/337:/dev/vfio/337" - "--device=/dev/vfio/338:/dev/vfio/338" - "--device=/dev/vfio/339:/dev/vfio/339" - "--device=/dev/vfio/340:/dev/vfio/340" - "--device=/dev/vfio/341:/dev/vfio/341" - "--device=/dev/vfio/342:/dev/vfio/342" - "--device=/dev/vfio/343:/dev/vfio/343" - "--device=/dev/vfio/344:/dev/vfio/344" - "--device=/dev/vfio/345:/dev/vfio/345" - "--device=/dev/vfio/346:/dev/vfio/346" - "--device=/dev/vfio/347:/dev/vfio/347" - "--device=/dev/vfio/348:/dev/vfio/348" - "--device=/dev/vfio/349:/dev/vfio/349" - "--device=/dev/vfio/350:/dev/vfio/350" - "--device=/dev/vfio/351:/dev/vfio/351" - "--device=/dev/vfio/352:/dev/vfio/352" - "--device=/dev/vfio/353:/dev/vfio/353" - "--device=/dev/vfio/354:/dev/vfio/354" - "--device=/dev/vfio/355:/dev/vfio/355" - "--device=/dev/vfio/356:/dev/vfio/356" - "--device=/dev/vfio/357:/dev/vfio/357" - "--device=/dev/vfio/358:/dev/vfio/358" - "--device=/dev/vfio/359:/dev/vfio/359" - "--device=/dev/vfio/360:/dev/vfio/360" - "--device=/dev/vfio/361:/dev/vfio/361" - "--device=/dev/vfio/362:/dev/vfio/362" - "--device=/dev/vfio/363:/dev/vfio/363" - "--device=/dev/vfio/364:/dev/vfio/364" - "--device=/dev/vfio/365:/dev/vfio/365" - "--device=/dev/vfio/366:/dev/vfio/366" - "--device=/dev/vfio/367:/dev/vfio/367" - "--device=/dev/vfio/368:/dev/vfio/368" - "--device=/dev/vfio/369:/dev/vfio/369" - "--device=/dev/vfio/370:/dev/vfio/370" - "--device=/dev/vfio/371:/dev/vfio/371" - "--device=/dev/vfio/372:/dev/vfio/372" - "--device=/dev/vfio/373:/dev/vfio/373" - "--device=/dev/vfio/374:/dev/vfio/374" - "--device=/dev/vfio/375:/dev/vfio/375" - "--device=/dev/vfio/376:/dev/vfio/376" - "--device=/dev/vfio/377:/dev/vfio/377" - "--device=/dev/vfio/378:/dev/vfio/378" - "--device=/dev/vfio/379:/dev/vfio/379" - "--device=/dev/vfio/380:/dev/vfio/380" - "--device=/dev/vfio/381:/dev/vfio/381" - "--device=/dev/vfio/382:/dev/vfio/382" - "--device=/dev/vfio/383:/dev/vfio/383" - "--device=/dev/vfio/384:/dev/vfio/384" - "--device=/dev/vfio/385:/dev/vfio/385" - "--device=/dev/vfio/386:/dev/vfio/386" - "--device=/dev/vfio/387:/dev/vfio/387" - "--device=/dev/vfio/388:/dev/vfio/388" - "--device=/dev/vfio/389:/dev/vfio/389" - "--device=/dev/vfio/390:/dev/vfio/390" - "--device=/dev/vfio/391:/dev/vfio/391" - "--device=/dev/vfio/392:/dev/vfio/392" - "--device=/dev/vfio/393:/dev/vfio/393" - "--device=/dev/vfio/394:/dev/vfio/394" - "--device=/dev/vfio/395:/dev/vfio/395" - "--device=/dev/vfio/396:/dev/vfio/396" - "--device=/dev/vfio/devices/vfio0:/dev/vfio/devices/vfio0" - "--device=/dev/vfio/devices/vfio1:/dev/vfio/devices/vfio1" - "--device=/dev/vfio/devices/vfio2:/dev/vfio/devices/vfio2" - "--device=/dev/vfio/devices/vfio3:/dev/vfio/devices/vfio3" - "--device=/dev/vfio/devices/vfio4:/dev/vfio/devices/vfio4" - "--device=/dev/vfio/devices/vfio5:/dev/vfio/devices/vfio5" - "--device=/dev/vfio/devices/vfio6:/dev/vfio/devices/vfio6" - "--device=/dev/vfio/devices/vfio7:/dev/vfio/devices/vfio7" - "--device=/dev/vfio/devices/vfio8:/dev/vfio/devices/vfio8" - "--device=/dev/vfio/devices/vfio9:/dev/vfio/devices/vfio9" - "--device=/dev/vfio/devices/vfio10:/dev/vfio/devices/vfio10" - "--device=/dev/vfio/devices/vfio11:/dev/vfio/devices/vfio11" - "--device=/dev/vfio/devices/vfio12:/dev/vfio/devices/vfio12" - "--device=/dev/vfio/devices/vfio13:/dev/vfio/devices/vfio13" - "--device=/dev/vfio/devices/vfio14:/dev/vfio/devices/vfio14" - "--device=/dev/vfio/devices/vfio15:/dev/vfio/devices/vfio15" - "--device=/dev/vfio/devices/vfio16:/dev/vfio/devices/vfio16" - "--device=/dev/vfio/devices/vfio17:/dev/vfio/devices/vfio17" - "--device=/dev/vfio/devices/vfio18:/dev/vfio/devices/vfio18" - "--device=/dev/vfio/devices/vfio19:/dev/vfio/devices/vfio19" - "--device=/dev/vfio/devices/vfio20:/dev/vfio/devices/vfio20" - "--device=/dev/vfio/devices/vfio21:/dev/vfio/devices/vfio21" - "--device=/dev/vfio/devices/vfio22:/dev/vfio/devices/vfio22" - "--device=/dev/vfio/devices/vfio23:/dev/vfio/devices/vfio23" - "--device=/dev/vfio/devices/vfio24:/dev/vfio/devices/vfio24" - "--device=/dev/vfio/devices/vfio25:/dev/vfio/devices/vfio25" - "--device=/dev/vfio/devices/vfio26:/dev/vfio/devices/vfio26" - "--device=/dev/vfio/devices/vfio27:/dev/vfio/devices/vfio27" - "--device=/dev/vfio/devices/vfio28:/dev/vfio/devices/vfio28" - "--device=/dev/vfio/devices/vfio29:/dev/vfio/devices/vfio29" - "--device=/dev/vfio/devices/vfio30:/dev/vfio/devices/vfio30" - "--device=/dev/vfio/devices/vfio31:/dev/vfio/devices/vfio31" - "--device=/dev/vfio/devices/vfio32:/dev/vfio/devices/vfio32" - "--device=/dev/vfio/devices/vfio33:/dev/vfio/devices/vfio33" - "--device=/dev/vfio/devices/vfio34:/dev/vfio/devices/vfio34" - "--device=/dev/vfio/devices/vfio35:/dev/vfio/devices/vfio35" - "--device=/dev/vfio/devices/vfio36:/dev/vfio/devices/vfio36" - "--device=/dev/vfio/devices/vfio37:/dev/vfio/devices/vfio37" - "--device=/dev/vfio/devices/vfio38:/dev/vfio/devices/vfio38" - "--device=/dev/vfio/devices/vfio39:/dev/vfio/devices/vfio39" - "--device=/dev/vfio/devices/vfio40:/dev/vfio/devices/vfio40" - "--device=/dev/vfio/devices/vfio41:/dev/vfio/devices/vfio41" - "--device=/dev/vfio/devices/vfio42:/dev/vfio/devices/vfio42" - "--device=/dev/vfio/devices/vfio43:/dev/vfio/devices/vfio43" - "--device=/dev/vfio/devices/vfio44:/dev/vfio/devices/vfio44" - "--device=/dev/vfio/devices/vfio45:/dev/vfio/devices/vfio45" - "--device=/dev/vfio/devices/vfio46:/dev/vfio/devices/vfio46" - "--device=/dev/vfio/devices/vfio47:/dev/vfio/devices/vfio47" - "--device=/dev/vfio/devices/vfio48:/dev/vfio/devices/vfio48" - "--device=/dev/vfio/devices/vfio49:/dev/vfio/devices/vfio49" - "--device=/dev/vfio/devices/vfio50:/dev/vfio/devices/vfio50" - "--device=/dev/vfio/devices/vfio51:/dev/vfio/devices/vfio51" - "--device=/dev/vfio/devices/vfio52:/dev/vfio/devices/vfio52" - "--device=/dev/vfio/devices/vfio53:/dev/vfio/devices/vfio53" - "--device=/dev/vfio/devices/vfio54:/dev/vfio/devices/vfio54" - "--device=/dev/vfio/devices/vfio55:/dev/vfio/devices/vfio55" - "--device=/dev/vfio/devices/vfio56:/dev/vfio/devices/vfio56" - "--device=/dev/vfio/devices/vfio57:/dev/vfio/devices/vfio57" - "--device=/dev/vfio/devices/vfio58:/dev/vfio/devices/vfio58" - "--device=/dev/vfio/devices/vfio59:/dev/vfio/devices/vfio59" - "--device=/dev/vfio/devices/vfio60:/dev/vfio/devices/vfio60" - "--device=/dev/vfio/devices/vfio61:/dev/vfio/devices/vfio61" - "--device=/dev/vfio/devices/vfio62:/dev/vfio/devices/vfio62" - "--device=/dev/vfio/devices/vfio63:/dev/vfio/devices/vfio63" networks: - 172.17.8.0/24 spec: rgw_frontend_port: 8000 plugin crypto accelerator = crypto_qat qat compressor enabled=true [user@client ~]USD vi bucket-encryption.json { "Rules": [ { "ApplyServerSideEncryptionByDefault": { "SSEAlgorithm": "AES256" } } ] } aws --endpoint-url=pass:q[_RADOSGW_ENDPOINT_URL_]:pass:q[_PORT_] s3api put-bucket-encryption --bucket pass:q[_BUCKET_NAME_] --server-side-encryption-configuration pass:q[_file://PATH_TO_BUCKET_ENCRYPTION_CONFIGURATION_FILE/BUCKET_ENCRYPTION_CONFIGURATION_FILE.json_] [user@client ~]USD aws --endpoint-url=http://host01:80 s3api put-bucket-encryption --bucket testbucket --server-side-encryption-configuration file://bucket-encryption.json aws --endpoint-url=pass:q[_RADOSGW_ENDPOINT_URL_]:pass:q[_PORT_] s3api get-bucket-encryption --bucket BUCKET_NAME [user@client ~]USD aws --profile ceph --endpoint=http://host01:80 s3api get-bucket-encryption --bucket testbucket { "ServerSideEncryptionConfiguration": { "Rules": [ { "ApplyServerSideEncryptionByDefault": { "SSEAlgorithm": "AES256" } } ] } } aws --endpoint-url= RADOSGW_ENDPOINT_URL : PORT s3api delete-bucket-encryption --bucket BUCKET_NAME [user@client ~]USD aws --endpoint-url=http://host01:80 s3api delete-bucket-encryption --bucket testbucket aws --endpoint-url= RADOSGW_ENDPOINT_URL : PORT s3api get-bucket-encryption --bucket BUCKET_NAME [user@client ~]USD aws --endpoint=http://host01:80 s3api get-bucket-encryption --bucket testbucket An error occurred (ServerSideEncryptionConfigurationNotFoundError) when calling the GetBucketEncryption operation: The server side encryption configuration was not found frontend http_web *:80 mode http default_backend rgw frontend rgw\u00ad-https bind *:443 ssl crt /etc/ssl/private/example.com.pem default_backend rgw backend rgw balance roundrobin mode http server rgw1 10.0.0.71:8080 check server rgw2 10.0.0.80:8080 check frontend http_web *:80 mode http default_backend rgw frontend rgw\u00ad-https bind *:443 ssl crt /etc/ssl/private/example.com.pem http-request set-header X-Forwarded-Proto https if { ssl_fc } http-request set-header X-Forwarded-Proto https here we set the incoming HTTPS port on the load balancer (eg : 443) http-request set-header X-Forwarded-Port 443 default_backend rgw backend rgw balance roundrobin mode http server rgw1 10.0.0.71:8080 check server rgw2 10.0.0.80:8080 check ceph config set client.rgw rgw_trust_forwarded_https true systemctl enable haproxy systemctl start haproxy ceph config set client.rgw rgw_crypt_vault_secret_engine transit compat=0 ceph config set client.rgw rgw_crypt_vault_secret_engine transit compat=1 ceph config set client.rgw rgw_crypt_vault_secret_engine transit compat=2 vault policy write rgw-kv-policy -<<EOF path "secret/data/*" { capabilities = ["read"] } EOF vault policy write rgw-transit-policy -<<EOF path "transit/keys/*" { capabilities = [ "create", "update" ] denied_parameters = {"exportable" = [], "allow_plaintext_backup" = [] } } path "transit/keys/*" { capabilities = ["read", "delete"] } path "transit/keys/" { capabilities = ["list"] } path "transit/keys/+/rotate" { capabilities = [ "update" ] } path "transit/*" { capabilities = [ "update" ] } EOF vault policy write old-rgw-transit-policy -<<EOF path "transit/export/encryption-key/*" { capabilities = ["read"] } EOF ceph config set client.rgw rgw_crypt_s3_kms_backend vault ceph config set client.rgw rgw_crypt_vault_auth agent ceph config set client.rgw rgw_crypt_vault_addr http:// VAULT_SERVER :8100 vault read auth/approle/role/rgw-ap/role-id -format=json | \ jq -r .data.role_id > PATH_TO_FILE vault read auth/approle/role/rgw-ap/role-id -format=json | \ jq -r .data.secret_id > PATH_TO_FILE pid_file = "/run/kv-vault-agent-pid" auto_auth { method "AppRole" { mount_path = "auth/approle" config = { role_id_file_path ="/root/vault_configs/kv-agent-role-id" secret_id_file_path ="/root/vault_configs/kv-agent-secret-id" remove_secret_id_file_after_reading ="false" } } } cache { use_auto_auth_token = true } listener "tcp" { address = "127.0.0.1:8100" tls_disable = true } vault { address = "http://10.8.128.9:8200" } /usr/local/bin/vault agent -config=/usr/local/etc/vault/rgw-agent.hcl ceph config set client.rgw rgw_crypt_vault_secret_engine kv ceph config set client.rgw rgw_crypt_vault_secret_engine transit ceph config set client.rgw rgw_crypt_vault_namespace testnamespace1 ceph config set client.rgw rgw_crypt_vault_prefix /v1/secret/data ceph config set client.rgw rgw_crypt_vault_prefix /v1/transit/export/encryption-key http://vault-server:8200/v1/transit/export/encryption-key systemctl restart ceph- CLUSTER_ID@SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw cephadm shell ceph config set client.rgw rgw_crypt_sse_s3_backend vault ceph config set client.rgw rgw_crypt_sse_s3_vault_auth agent ceph config set client.rgw rgw_crypt_sse_s3_vault_addr http:// VAULT_AGENT : VAULT_AGENT_PORT ceph config set client.rgw rgw_crypt_sse_s3_vault_auth agent ceph config set client.rgw rgw_crypt_sse_s3_vault_addr http://vaultagent:8100 vault read auth/approle/role/rgw-ap/role-id -format=json | \ jq -r .rgw-ap-role-id > PATH_TO_FILE vault read auth/approle/role/rgw-ap/role-id -format=json | \ jq -r .rgw-ap-secret-id > PATH_TO_FILE pid_file = "/run/rgw-vault-agent-pid" auto_auth { method "AppRole" { mount_path = "auth/approle" config = { role_id_file_path ="/usr/local/etc/vault/.rgw-ap-role-id" secret_id_file_path ="/usr/local/etc/vault/.rgw-ap-secret-id" remove_secret_id_file_after_reading ="false" } } } cache { use_auto_auth_token = true } listener "tcp" { address = "127.0.0.1:8100" tls_disable = true } vault { address = "https://vaultserver:8200" } /usr/local/bin/vault agent -config=/usr/local/etc/vault/rgw-agent.hcl ceph config set client.rgw rgw_crypt_sse_s3_vault_secret_engine kv ceph config set client.rgw rgw_crypt_sse_s3_vault_secret_engine transit ceph config set client.rgw rgw_crypt_sse_s3_vault_namespace company/testnamespace1 ceph config set client.rgw rgw_crypt_sse_s3_vault_prefix /v1/secret/data ceph config set client.rgw rgw_crypt_sse_s3_vault_prefix /v1/transit http://vaultserver:8200/v1/transit ceph config set client.rgw rgw_crypt_sse_s3_vault_verify_ssl true ceph config set client.rgw rgw_crypt_sse_s3_vault_ssl_cacert PATH_TO_CA_CERTIFICATE ceph config set client.rgw rgw_crypt_sse_s3_vault_ssl_clientcert PATH_TO_CLIENT_CERTIFICATE ceph config set client.rgw rgw_crypt_sse_s3_vault_ssl_clientkey PATH_TO_PRIVATE_KEY ceph config set client.rgw rgw_crypt_sse_s3_vault_verify_ssl true ceph config set client.rgw rgw_crypt_sse_s3_vault_ssl_cacert /etc/ceph/vault.ca ceph config set client.rgw rgw_crypt_sse_s3_vault_ssl_clientcert /etc/ceph/vault.crt ceph config set client.rgw rgw_crypt_sse_s3_vault_ssl_clientkey /etc/ceph/vault.key systemctl restart ceph- CLUSTER_ID@SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw vault secrets enable -path secret kv-v2 vault kv put secret/ PROJECT_NAME / BUCKET_NAME key=USD(openssl rand -base64 32) vault kv put secret/myproject/mybucketkey key=USD(openssl rand -base64 32) ====== Metadata ====== Key Value --- ---- created_time 2020-02-21T17:01:09.095824999Z deletion_time n/a destroyed false version 1 vault secrets enable transit vault write -f transit/keys/ BUCKET_NAME exportable=true vault write -f transit/keys/mybucketkey exportable=true vault read transit/export/encryption-key/ BUCKET_NAME / VERSION_NUMBER vault read transit/export/encryption-key/mybucketkey/1 Key Value --- ----- keys map[1:-gbTI9lNpqv/V/2lDcmH2Nq1xKn6FPDWarCmFM2aNsQ=] name mybucketkey type aes256-gcm96 [user@client ~]USD aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id myproject/mybucketkey [user@client ~]USD aws s3api --endpoint http://rgw_host:8080 put-object --bucket my-bucket --key obj1 --body test_file_to_upload --server-side-encryption AES256 [user@client ~]USD aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id mybucketkey [user@client ~]USD aws s3api --endpoint http://rgw_host:8080 put-object --bucket my-bucket --key obj1 --body test_file_to_upload --server-side-encryption AES256 [user@host01 ~]USD SEED=USD(head -10 /dev/urandom | sha512sum | cut -b 1-30) [user@host01 ~]USD echo USDSEED 492dedb20cf51d1405ef6a1316017e radosgw-admin mfa create --uid= USERID --totp-serial= SERIAL --totp-seed= SEED --totp-seed-type= SEED_TYPE --totp-seconds= TOTP_SECONDS --totp-window= TOTP_WINDOW radosgw-admin mfa create --uid=johndoe --totp-serial=MFAtest --totp-seed=492dedb20cf51d1405ef6a1316017e radosgw-admin mfa check --uid= USERID --totp-serial= SERIAL --totp-pin= PIN radosgw-admin mfa check --uid=johndoe --totp-serial=MFAtest --totp-pin=870305 ok radosgw-admin mfa resync --uid= USERID --totp-serial= SERIAL --totp-pin= PREVIOUS_PIN --totp=pin= CURRENT_PIN radosgw-admin mfa resync --uid=johndoe --totp-serial=MFAtest --totp-pin=802021 --totp-pin=439996 radosgw-admin mfa check --uid= USERID --totp-serial= SERIAL --totp-pin= PIN radosgw-admin mfa check --uid=johndoe --totp-serial=MFAtest --totp-pin=870305 ok radosgw-admin mfa list --uid= USERID radosgw-admin mfa list --uid=johndoe { "entries": [ { "type": 2, "id": "MFAtest", "seed": "492dedb20cf51d1405ef6a1316017e", "seed_type": "hex", "time_ofs": 0, "step_size": 30, "window": 2 } ] } radosgw-admin mfa get --uid= USERID --totp-serial= SERIAL radosgw-admin mfa remove --uid= USERID --totp-serial= SERIAL radosgw-admin mfa remove --uid=johndoe --totp-serial=MFAtest radosgw-admin mfa get --uid= USERID --totp-serial= SERIAL radosgw-admin mfa get --uid=johndoe --totp-serial=MFAtest MFA serial id not found radosgw-admin zonegroup --rgw-zonegroup= ZONE_GROUP_NAME get > FILE_NAME .json radosgw-admin zonegroup --rgw-zonegroup=default get > zonegroup.json { "name": "default", "api_name": "", "is_master": "true", "endpoints": [], "hostnames": [], "master_zone": "", "zones": [{ "name": "default", "endpoints": [], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 5 }], "placement_targets": [{ "name": "default-placement", "tags": [] }, { "name": "special-placement", "tags": [] }], "default_placement": "default-placement" } radosgw-admin zonegroup set < zonegroup.json radosgw-admin zone get > zone.json { "domain_root": ".rgw", "control_pool": ".rgw.control", "gc_pool": ".rgw.gc", "log_pool": ".log", "intent_log_pool": ".intent-log", "usage_log_pool": ".usage", "user_keys_pool": ".users", "user_email_pool": ".users.email", "user_swift_pool": ".users.swift", "user_uid_pool": ".users.uid", "system_key": { "access_key": "", "secret_key": "" }, "placement_pools": [{ "key": "default-placement", "val": { "index_pool": ".rgw.buckets.index", "data_pool": ".rgw.buckets", "data_extra_pool": ".rgw.buckets.extra" } }, { "key": "special-placement", "val": { "index_pool": ".rgw.buckets.index", "data_pool": ".rgw.buckets.special", "data_extra_pool": ".rgw.buckets.extra" } }] } radosgw-admin zone set < zone.json radosgw-admin period update --commit curl -i http://10.0.0.1/swift/v1/TestContainer/file.txt -X PUT -H "X-Storage-Policy: special-placement" -H "X-Auth-Token: AUTH_rgwtxxxxxx" radosgw-admin zonegroup placement add --rgw-zonegroup="default" --placement-id="indexless-placement" radosgw-admin zone placement add --rgw-zone="default" --placement-id="indexless-placement" --data-pool="default.rgw.buckets.data" --index-pool="default.rgw.buckets.index" --data_extra_pool="default.rgw.buckets.non-ec" --placement-index-type="indexless" radosgw-admin zonegroup placement default --placement-id "indexless-placement" radosgw-admin period update --commit ceph orch restart SERVICE_TYPE ceph orch restart rgw ln: failed to access '/tmp/rgwrbi-object-list.4053207': No such file or directory /usr/bin/rgw-restore-bucket-index -b bucket-large-1 -p local-zone.rgw.buckets.data marker is d8a347a4-99b6-4312-a5c1-75b83904b3d4.41610.2 bucket_id is d8a347a4-99b6-4312-a5c1-75b83904b3d4.41610.2 number of bucket index shards is 5 data pool is local-zone.rgw.buckets.data NOTICE: This tool is currently considered EXPERIMENTAL. The list of objects that we will attempt to restore can be found in "/tmp/rgwrbi-object-list.49946". Please review the object names in that file (either below or in another window/terminal) before proceeding. Type "proceed!" to proceed, "view" to view object list, or "q" to quit: view Viewing Type "proceed!" to proceed, "view" to view object list, or "q" to quit: proceed! Proceeding NOTICE: Bucket stats are currently incorrect. They can be restored with the following command after 2 minutes: radosgw-admin bucket list --bucket=bucket-large-1 --allow-unordered --max-entries=1073741824 Would you like to take the time to recalculate bucket stats now? [yes/no] yes Done real 2m16.530s user 0m1.082s sys 0m0.870s time rgw-restore-bucket-index --proceed serp-bu-ver-1 default.rgw.buckets.data NOTICE: This tool is currently considered EXPERIMENTAL. marker is e871fb65-b87f-4c16-a7c3-064b66feb1c4.25076.5 bucket_id is e871fb65-b87f-4c16-a7c3-064b66feb1c4.25076.5 Error: this bucket appears to be versioned, and this tool cannot work with versioned buckets. Bucket _BUCKET_NAME_ already has too many log generations (4) from previous reshards that peer zones haven't finished syncing. Resharding is not recommended until the old generations sync, but you can force a reshard with `--yes-i-really-mean-it`. number of objects expected in a bucket / 100,000 ceph config set client.rgw rgw_override_bucket_index_max_shards VALUE ceph config set client.rgw rgw_override_bucket_index_max_shards 12 ceph orch restart SERVICE_TYPE ceph orch restart rgw number of objects expected in a bucket / 100,000 radosgw-admin zonegroup get > zonegroup.json bucket_index_max_shards = VALUE bucket_index_max_shards = 12 radosgw-admin zonegroup set < zonegroup.json radosgw-admin period update --commit radosgw-admin reshard status --bucket BUCKET_NAME radosgw-admin reshard status --bucket data radosgw-admin sync status radosgw-admin period get ceph config set client.rgw OPTION VALUE ceph config set client.rgw rgw_reshard_num_logs 23 radosgw-admin reshard add --bucket BUCKET --num-shards NUMBER radosgw-admin reshard add --bucket data --num-shards 10 radosgw-admin reshard list radosgw-admin bucket layout --bucket data { "layout": { "resharding": "None", "current_index": { "gen": 1, "layout": { "type": "Normal", "normal": { "num_shards": 23, "hash_type": "Mod" } } }, "logs": [ { "gen": 0, "layout": { "type": "InIndex", "in_index": { "gen": 0, "layout": { "num_shards": 11, "hash_type": "Mod" } } } }, { "gen": 1, "layout": { "type": "InIndex", "in_index": { "gen": 1, "layout": { "num_shards": 23, "hash_type": "Mod" } } } } ] } } radosgw-admin reshard status --bucket BUCKET radosgw-admin reshard status --bucket data radosgw-admin reshard process radosgw-admin reshard cancel --bucket BUCKET radosgw-admin reshard cancel --bucket data radosgw-admin reshard status --bucket BUCKET radosgw-admin reshard status --bucket data radosgw-admin sync status radosgw-admin zonegroup modify --rgw-zonegroup= ZONEGROUP_NAME --enable-feature=resharding radosgw-admin zonegroup modify --rgw-zonegroup=us --enable-feature=resharding radosgw-admin period update --commit radosgw-admin zone modify --rgw-zone= ZONE_NAME --enable-feature=resharding radosgw-admin zone modify --rgw-zone=us-east --enable-feature=resharding radosgw-admin period update --commit radosgw-admin period get "zones": [ { "id": "505b48db-6de0-45d5-8208-8c98f7b1278d", "name": "us_east", "endpoints": [ "http://10.0.208.11:8080" ], "log_meta": "false", "log_data": "true", "bucket_index_max_shards": 11, "read_only": "false", "tier_type": "", "sync_from_all": "true", "sync_from": [], "redirect_zone": "", "supported_features": [ "resharding" ] "default_placement": "default-placement", "realm_id": "26cf6f23-c3a0-4d57-aae4-9b0010ee55cc", "sync_policy": { "groups": [] }, "enabled_features": [ "resharding" ] radosgw-admin sync status realm 26cf6f23-c3a0-4d57-aae4-9b0010ee55cc (usa) zonegroup 33a17718-6c77-493e-99fe-048d3110a06e (us) zone 505b48db-6de0-45d5-8208-8c98f7b1278d (us_east) zonegroup features enabled: resharding radosgw-admin zonegroup modify --rgw-zonegroup= ZONEGROUP_NAME --disable-feature=resharding radosgw-admin zonegroup modify --rgw-zonegroup=us --disable-feature=resharding radosgw-admin period update --commit radosgw-admin bi list --bucket= BUCKET > BUCKET .list.backup radosgw-admin bi list --bucket=data > data.list.backup radosgw-admin bucket reshard --bucket= BUCKET --num-shards= NUMBER radosgw-admin bucket reshard --bucket=data --num-shards=100 radosgw-admin reshard status --bucket bucket radosgw-admin reshard status --bucket data radosgw-admin reshard stale-instances list radosgw-admin reshard stale-instances rm radosgw-admin reshard status --bucket BUCKET radosgw-admin reshard status --bucket data [root@host01 ~] radosgw-admin zone placement modify --rgw-zone=default --placement-id=default-placement --compression=zlib { "placement_pools": [ { "key": "default-placement", "val": { "index_pool": "default.rgw.buckets.index", "data_pool": "default.rgw.buckets.data", "data_extra_pool": "default.rgw.buckets.non-ec", "index_type": 0, "compression": "zlib" } } ], } radosgw-admin bucket stats --bucket= BUCKET_NAME { "usage": { "rgw.main": { "size": 1075028, "size_actual": 1331200, "size_utilized": 592035, "size_kb": 1050, "size_kb_actual": 1300, "size_kb_utilized": 579, "num_objects": 104 } }, } radosgw-admin user <create|modify|info|rm|suspend|enable|check|stats> <--uid= USER_ID |--subuser= SUB_USER_NAME > [other-options] radosgw-admin --tenant testx --uid tester --display-name "Test User" --access_key TESTER --secret test123 user create radosgw-admin --tenant testx --uid tester --display-name "Test User" --subuser tester:swift --key-type swift --access full subuser create radosgw-admin key create --subuser 'testxUSDtester:swift' --key-type swift --secret test123 radosgw-admin user create --uid= USER_ID [--key-type= KEY_TYPE ] [--gen-access-key|--access-key= ACCESS_KEY ] [--gen-secret | --secret= SECRET_KEY ] [--email= EMAIL ] --display-name= DISPLAY_NAME radosgw-admin user create --uid=janedoe --access-key=11BS02LGFB6AL6H1ADMW --secret=vzCEkuryfn060dfee4fgQPqFrncKEIkh3ZcdOANY --email=jane@example.com --display-name=Jane Doe { "user_id": "janedoe", "display_name": "Jane Doe", "email": "jane@example.com", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "janedoe", "access_key": "11BS02LGFB6AL6H1ADMW", "secret_key": "vzCEkuryfn060dfee4fgQPqFrncKEIkh3ZcdOANY"}], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1}, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1}, "temp_url_keys": []} radosgw-admin subuser create --uid= USER_ID --subuser= SUB_USER_ID --access=[ read | write | readwrite | full ] radosgw-admin subuser create --uid=janedoe --subuser=janedoe:swift --access=full { "user_id": "janedoe", "display_name": "Jane Doe", "email": "jane@example.com", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [ { "id": "janedoe:swift", "permissions": "full-control"}], "keys": [ { "user": "janedoe", "access_key": "11BS02LGFB6AL6H1ADMW", "secret_key": "vzCEkuryfn060dfee4fgQPqFrncKEIkh3ZcdOANY"}], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1}, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1}, "temp_url_keys": []} radosgw-admin user info --uid=janedoe radosgw-admin user info --uid=janedoe --tenant=test radosgw-admin user modify --uid=janedoe --display-name="Jane E. Doe" radosgw-admin subuser modify --subuser=janedoe:swift --access=full radosgw-admin user suspend --uid=johndoe radosgw-admin user enable --uid=johndoe radosgw-admin user rm --uid= USER_ID [--purge-keys] [--purge-data] radosgw-admin user rm --uid=johndoe --purge-data radosgw-admin subuser rm --subuser=johndoe:swift --purge-keys radosgw-admin subuser rm --subuser= SUB_USER_ID radosgw-admin subuser rm --subuser=johndoe:swift radosgw-admin user rename --uid= CURRENT_USER_NAME --new-uid= NEW_USER_NAME radosgw-admin user rename --uid=user1 --new-uid=user2 { "user_id": "user2", "display_name": "user 2", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "user2", "access_key": "59EKHI6AI9F8WOW8JQZJ", "secret_key": "XH0uY3rKCUcuL73X0ftjXbZqUbk0cavD11rD8MsA" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw" } radosgw-admin user rename --uid USER_NAME --new-uid NEW_USER_NAME --tenant TENANT radosgw-admin user rename --uid=testUSDuser1 --new-uid=testUSDuser2 --tenant test 1000 objects processed in tvtester1. Next marker 80_tVtester1_99 2000 objects processed in tvtester1. Next marker 64_tVtester1_44 3000 objects processed in tvtester1. Next marker 48_tVtester1_28 4000 objects processed in tvtester1. Next marker 2_tVtester1_74 5000 objects processed in tvtester1. Next marker 14_tVtester1_53 6000 objects processed in tvtester1. Next marker 87_tVtester1_61 7000 objects processed in tvtester1. Next marker 6_tVtester1_57 8000 objects processed in tvtester1. Next marker 52_tVtester1_91 9000 objects processed in tvtester1. Next marker 34_tVtester1_74 9900 objects processed in tvtester1. Next marker 9_tVtester1_95 1000 objects processed in tvtester2. Next marker 82_tVtester2_93 2000 objects processed in tvtester2. Next marker 64_tVtester2_9 3000 objects processed in tvtester2. Next marker 48_tVtester2_22 4000 objects processed in tvtester2. Next marker 32_tVtester2_42 5000 objects processed in tvtester2. Next marker 16_tVtester2_36 6000 objects processed in tvtester2. Next marker 89_tVtester2_46 7000 objects processed in tvtester2. Next marker 70_tVtester2_78 8000 objects processed in tvtester2. Next marker 51_tVtester2_41 9000 objects processed in tvtester2. Next marker 33_tVtester2_32 9900 objects processed in tvtester2. Next marker 9_tVtester2_83 { "user_id": "testUSDuser2", "display_name": "User 2", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "testUSDuser2", "access_key": "user2", "secret_key": "123456789" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw" } radosgw-admin user info --uid= NEW_USER_NAME radosgw-admin user info --uid=user2 radosgw-admin user info --uid= TENANT USD USER_NAME radosgw-admin user info --uid=testUSDuser2 radosgw-admin key create --subuser=johndoe:swift --key-type=swift --gen-secret { "user_id": "johndoe", "rados_uid": 0, "display_name": "John Doe", "email": "john@example.com", "suspended": 0, "subusers": [ { "id": "johndoe:swift", "permissions": "full-control"}], "keys": [ { "user": "johndoe", "access_key": "QFAMEDSJP5DEKJO0DDXY", "secret_key": "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87"}], "swift_keys": [ { "user": "johndoe:swift", "secret_key": "E9T2rUZNu2gxUjcwUBO8n\/Ev4KX6\/GprEuH4qhu1"}]} radosgw-admin key create --uid=johndoe --key-type=s3 --gen-access-key --gen-secret radosgw-admin user info --uid=johndoe radosgw-admin user info --uid=johndoe { "user_id": "johndoe", "keys": [ { "user": "johndoe", "access_key": "0555b35654ad1656d804", "secret_key": "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==" } ], } radosgw-admin key rm --uid= USER_ID --access-key ACCESS_KEY radosgw-admin key rm --uid=johndoe --access-key 0555b35654ad1656d804 radosgw-admin caps add --uid= USER_ID --caps= CAPS --caps="[users|buckets|metadata|usage|zone]=[*|read|write|read, write]" radosgw-admin caps add --uid=johndoe --caps="users=*" radosgw-admin caps remove --uid=johndoe --caps={caps} radosgw-admin role create --role-name= ROLE_NAME [--path==" PATH_TO_FILE "] [--assume-role-policy-doc= TRUST_RELATIONSHIP_POLICY_DOCUMENT ] radosgw-admin role create --role-name=S3Access1 --path=/application_abc/component_xyz/ --assume-role-policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\} { "RoleId": "ca43045c-082c-491a-8af1-2eebca13deec", "RoleName": "S3Access1", "Path": "/application_abc/component_xyz/", "Arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1", "CreateDate": "2022-06-17T10:18:29.116Z", "MaxSessionDuration": 3600, "AssumeRolePolicyDocument": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}" } radosgw-admin role get --role-name= ROLE_NAME radosgw-admin role get --role-name=S3Access1 { "RoleId": "ca43045c-082c-491a-8af1-2eebca13deec", "RoleName": "S3Access1", "Path": "/application_abc/component_xyz/", "Arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1", "CreateDate": "2022-06-17T10:18:29.116Z", "MaxSessionDuration": 3600, "AssumeRolePolicyDocument": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}" } radosgw-admin role list radosgw-admin role list [ { "RoleId": "85fb46dd-a88a-4233-96f5-4fb54f4353f7", "RoleName": "kvm-sts", "Path": "/application_abc/component_xyz/", "Arn": "arn:aws:iam:::role/application_abc/component_xyz/kvm-sts", "CreateDate": "2022-09-13T11:55:09.39Z", "MaxSessionDuration": 7200, "AssumeRolePolicyDocument": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/kvm\"]},\"Action\":[\"sts:AssumeRole\"]}]}" }, { "RoleId": "9116218d-4e85-4413-b28d-cdfafba24794", "RoleName": "kvm-sts-1", "Path": "/application_abc/component_xyz/", "Arn": "arn:aws:iam:::role/application_abc/component_xyz/kvm-sts-1", "CreateDate": "2022-09-16T00:05:57.483Z", "MaxSessionDuration": 3600, "AssumeRolePolicyDocument": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/kvm\"]},\"Action\":[\"sts:AssumeRole\"]}]}" } ] radosgw-admin role-trust-policy modify --role-name= ROLE_NAME --assume-role-policy-doc= TRUST_RELATIONSHIP_POLICY_DOCUMENT radosgw-admin role-trust-policy modify --role-name=S3Access1 --assume-role-policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\} { "RoleId": "ca43045c-082c-491a-8af1-2eebca13deec", "RoleName": "S3Access1", "Path": "/application_abc/component_xyz/", "Arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1", "CreateDate": "2022-06-17T10:18:29.116Z", "MaxSessionDuration": 3600, "AssumeRolePolicyDocument": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}" } radosgw-admin role-policy get --role-name= ROLE_NAME --policy-name= POLICY_NAME radosgw-admin role-policy get --role-name=S3Access1 --policy-name=Policy1 { "Permission policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:*\"],\"Resource\":\"arn:aws:s3:::example_bucket\"}]}" } radosgw-admin role policy delete --role-name= ROLE_NAME --policy-name= POLICY_NAME radosgw-admin role policy delete --role-name=S3Access1 --policy-name=Policy1 radosgw-admin role delete --role-name= ROLE_NAME radosgw-admin role delete --role-name=S3Access1 radosgw-admin role-policy put --role-name= ROLE_NAME --policy-name= POLICY_NAME --policy-doc= PERMISSION_POLICY_DOCUMENT radosgw-admin role-policy put --role-name=S3Access1 --policy-name=Policy1 --policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Action\":\[\"s3:*\"\],\"Resource\":\"arn:aws:s3:::example_bucket\"\}\]\} radosgw-admin role-policy list --role-name= ROLE_NAME radosgw-admin role-policy list --role-name=S3Access1 [ "Policy1" ] radosgw-admin role policy delete --role-name= ROLE_NAME --policy-name= POLICY_NAME radosgw-admin role policy delete --role-name=S3Access1 --policy-name=Policy1 radosgw-admin role update --role-name= ROLE_NAME --max-session-duration=7200 radosgw-admin role update --role-name=test-sts-role --max-session-duration=7200 radosgw-admin role list [ { "RoleId": "d4caf33f-caba-42f3-8bd4-48c84b4ea4d3", "RoleName": "test-sts-role", "Path": "/", "Arn": "arn:aws:iam:::role/test-role", "CreateDate": "2022-09-07T20:01:15.563Z", "MaxSessionDuration": 7200, <<<<<< "AssumeRolePolicyDocument": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/kvm\"]},\"Action\":[\"sts:AssumeRole\"]}]}" } ] radosgw-admin quota set --quota-scope=user --uid= USER_ID [--max-objects= NUMBER_OF_OBJECTS ] [--max-size= MAXIMUM_SIZE_IN_BYTES ] radosgw-admin quota set --quota-scope=user --uid=johndoe --max-objects=1024 --max-size=1024 radosgw-admin quota enable --quota-scope=user --uid= USER_ID radosgw-admin quota disable --quota-scope=user --uid= USER_ID radosgw-admin quota set --uid= USER_ID --quota-scope=bucket --bucket= BUCKET_NAME [--max-objects= NUMBER_OF_OBJECTS ] [--max-size= MAXIMUM_SIZE_IN_BYTES ] radosgw-admin quota enable --quota-scope=bucket --uid= USER_ID radosgw-admin quota disable --quota-scope=bucket --uid= USER_ID radosgw-admin user info --uid= USER_ID radosgw-admin user info --uid= USER_ID --tenant= TENANT radosgw-admin user stats --uid= USER_ID --sync-stats radosgw-admin user stats --uid= USER_ID radosgw-admin global quota get radosgw-admin global quota set --quota-scope bucket --max-objects 1024 radosgw-admin global quota enable --quota-scope bucket radosgw-admin bucket list [ "34150b2e9174475db8e191c188e920f6/swcontainer", "s3bucket1", "34150b2e9174475db8e191c188e920f6/swimpfalse", "c278edd68cfb4705bb3e07837c7ad1a8/ec2container", "c278edd68cfb4705bb3e07837c7ad1a8/demoten1", "c278edd68cfb4705bb3e07837c7ad1a8/demo-ct", "c278edd68cfb4705bb3e07837c7ad1a8/demopostup", "34150b2e9174475db8e191c188e920f6/postimpfalse", "c278edd68cfb4705bb3e07837c7ad1a8/demoten2", "c278edd68cfb4705bb3e07837c7ad1a8/postupsw" ] radosgw-admin bucket link --bucket= ORIGINAL_NAME --bucket-new-name= NEW_NAME --uid= USER_ID radosgw-admin bucket link --bucket=s3bucket1 --bucket-new-name=s3newb --uid=testuser radosgw-admin bucket link --bucket= tenant / ORIGINAL_NAME --bucket-new-name= NEW_NAME --uid= TENANT USD USER_ID radosgw-admin bucket link --bucket=test/s3bucket1 --bucket-new-name=s3newb --uid=testUSDtestuser radosgw-admin bucket list [ "34150b2e9174475db8e191c188e920f6/swcontainer", "34150b2e9174475db8e191c188e920f6/swimpfalse", "c278edd68cfb4705bb3e07837c7ad1a8/ec2container", "s3newb", "c278edd68cfb4705bb3e07837c7ad1a8/demoten1", "c278edd68cfb4705bb3e07837c7ad1a8/demo-ct", "c278edd68cfb4705bb3e07837c7ad1a8/demopostup", "34150b2e9174475db8e191c188e920f6/postimpfalse", "c278edd68cfb4705bb3e07837c7ad1a8/demoten2", "c278edd68cfb4705bb3e07837c7ad1a8/postupsw" ] radosgw-admin bucket list [ "34150b2e9174475db8e191c188e920f6/swcontainer", "s3bucket1", "34150b2e9174475db8e191c188e920f6/swimpfalse", "c278edd68cfb4705bb3e07837c7ad1a8/ec2container", "c278edd68cfb4705bb3e07837c7ad1a8/demoten1", "c278edd68cfb4705bb3e07837c7ad1a8/demo-ct", "c278edd68cfb4705bb3e07837c7ad1a8/demopostup", "34150b2e9174475db8e191c188e920f6/postimpfalse", "c278edd68cfb4705bb3e07837c7ad1a8/demoten2", "c278edd68cfb4705bb3e07837c7ad1a8/postupsw" ] radosgw-admin bucket rm --bucket= BUCKET_NAME radosgw-admin bucket rm --bucket=s3bucket1 radosgw-admin bucket rm --bucket= BUCKET --purge-objects --bypass-gc radosgw-admin bucket rm --bucket=s3bucket1 --purge-objects --bypass-gc radosgw-admin bucket list [ "34150b2e9174475db8e191c188e920f6/swcontainer", "34150b2e9174475db8e191c188e920f6/swimpfalse", "c278edd68cfb4705bb3e07837c7ad1a8/ec2container", "c278edd68cfb4705bb3e07837c7ad1a8/demoten1", "c278edd68cfb4705bb3e07837c7ad1a8/demo-ct", "c278edd68cfb4705bb3e07837c7ad1a8/demopostup", "34150b2e9174475db8e191c188e920f6/postimpfalse", "c278edd68cfb4705bb3e07837c7ad1a8/demoten2", "c278edd68cfb4705bb3e07837c7ad1a8/postupsw" ] radosgw-admin bucket link --uid= USER --bucket= BUCKET radosgw-admin bucket link --uid=user2 --bucket=data radosgw-admin bucket list --uid=user2 [ "data" ] radosgw-admin bucket chown --uid= user --bucket= bucket radosgw-admin bucket chown --uid=user2 --bucket=data radosgw-admin bucket list --bucket=data radosgw-admin bucket link --bucket= CURRENT_TENANT / BUCKET --uid= NEW_TENANT USD USER radosgw-admin bucket link --bucket=test/data --uid=test2USDuser2 radosgw-admin bucket list --uid=testUSDuser2 [ "data" ] radosgw-admin bucket chown --bucket= NEW_TENANT / BUCKET --uid= NEW_TENANT USD USER radosgw-admin bucket chown --bucket='test2/data' --uid='testUSDtuser2' radosgw-admin bucket list --bucket=test2/data ceph config set client.rgw rgw_keystone_implicit_tenants true swift list s3cmd ls radosgw-admin bucket link --bucket=/ BUCKET --uid=' TENANT USD USER ' radosgw-admin bucket link --bucket=/data --uid='testUSDtenanted-user' radosgw-admin bucket list --uid='testUSDtenanted-user' [ "data" ] radosgw-admin bucket chown --bucket=' tenant / bucket name ' --uid=' tenant USD user ' radosgw-admin bucket chown --bucket='test/data' --uid='testUSDtenanted-user' radosgw-admin bucket list --bucket=test/data radosgw-admin bucket radoslist --bucket BUCKET_NAME radosgw-admin bucket radoslist --bucket mybucket head /usr/bin/rgw-orphan-list mkdir orphans cd orphans rgw-orphan-list Available pools: .rgw.root default.rgw.control default.rgw.meta default.rgw.log default.rgw.buckets.index default.rgw.buckets.data rbd default.rgw.buckets.non-ec ma.rgw.control ma.rgw.meta ma.rgw.log ma.rgw.buckets.index ma.rgw.buckets.data ma.rgw.buckets.non-ec Which pool do you want to search for orphans? rgw-orphan-list -h rgw-orphan-list POOL_NAME / DIRECTORY rgw-orphan-list default.rgw.buckets.data /orphans 2023-09-12 08:41:14 ceph-host01 Computing delta 2023-09-12 08:41:14 ceph-host01 Computing results 10 potential orphans found out of a possible 2412 (0%). <<<<<<< orphans detected The results can be found in './orphan-list-20230912124113.out'. Intermediate files are './rados-20230912124113.intermediate' and './radosgw-admin-20230912124113.intermediate'. *** *** WARNING: This is EXPERIMENTAL code and the results should be used *** only with CAUTION! *** Done at 2023-09-12 08:41:14. ls -l -rw-r--r--. 1 root root 770 Sep 12 03:59 orphan-list-20230912075939.out -rw-r--r--. 1 root root 0 Sep 12 03:59 rados-20230912075939.error -rw-r--r--. 1 root root 248508 Sep 12 03:59 rados-20230912075939.intermediate -rw-r--r--. 1 root root 0 Sep 12 03:59 rados-20230912075939.issues -rw-r--r--. 1 root root 0 Sep 12 03:59 radosgw-admin-20230912075939.error -rw-r--r--. 1 root root 247738 Sep 12 03:59 radosgw-admin-20230912075939.intermediate cat ./orphan-list-20230912124113.out a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.0 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.1 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.2 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.3 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.4 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.5 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.6 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.7 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.8 a9c042bc-be24-412c-9052-dda6b2f01f55.16749.1_key1.cherylf.433-bucky-4865-0.9 rados -p POOL_NAME rm OBJECT_NAME rados -p default.rgw.buckets.data rm myobject radosgw-admin bucket check --bucket= BUCKET_NAME radosgw-admin bucket check --bucket=mybucket radosgw-admin bucket check --fix --bucket= BUCKET_NAME radosgw-admin bucket check --fix --bucket=mybucket radosgw-admin topic list radosgw-admin topic get --topic=topic1 radosgw-admin topic rm --topic=topic1 client.put_bucket_notification_configuration( Bucket=bucket_name, NotificationConfiguration={ 'TopicConfigurations': [ { 'Id': notification_name, 'TopicArn': topic_arn, 'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*', 's3:ObjectLifecycle:Expiration:*'] }]}) { "Role": "arn:aws:iam::account-id:role/role-name", "Rules": [ { "ID": "String", "Status": "Enabled", "Priority": 1, "DeleteMarkerReplication": { "Status": "Enabled"|"Disabled" }, "Destination": { "Bucket": "BUCKET_NAME" } } ] } cat replication.json { "Role": "arn:aws:iam::account-id:role/role-name", "Rules": [ { "ID": "pipe-bkt", "Status": "Enabled", "Priority": 1, "DeleteMarkerReplication": { "Status": "Disabled" }, "Destination": { "Bucket": "testbucket" } } ] } aws --endpoint-url=RADOSGW_ENDPOINT_URL s3api put-bucket-replication --bucket BUCKET_NAME --replication-configuration file://REPLICATION_CONFIIRATION_FILE.json aws --endpoint-url=http://host01:80 s3api put-bucket-replication --bucket testbucket --replication-configuration file://replication.json radosgw-admin sync policy get --bucket BUCKET_NAME radosgw-admin sync policy get --bucket testbucket { "groups": [ { "id": "s3-bucket-replication:disabled", "data_flow": {}, "pipes": [], "status": "allowed" }, { "id": "s3-bucket-replication:enabled", "data_flow": {}, "pipes": [ { "id": "", "source": { "bucket": "*", "zones": [ "*" ] }, "dest": { "bucket": "testbucket", "zones": [ "*" ] }, "params": { "source": {}, "dest": {}, "priority": 1, "mode": "user", "user": "s3cmd" } } ], "status": "enabled" } ] } aws s3api get-bucket-replication --bucket BUCKET_NAME --endpoint-url=RADOSGW_ENDPOINT_URL aws s3api get-bucket-replication --bucket testbucket --endpoint-url=http://host01:80 { "ReplicationConfiguration": { "Role": "", "Rules": [ { "ID": "pipe-bkt", "Status": "Enabled", "Priority": 1, "Destination": { Bucket": "testbucket" } } ] } } aws s3api delete-bucket-replication --bucket BUCKET_NAME --endpoint-url=RADOSGW_ENDPOINT_URL aws s3api delete-bucket-replication --bucket testbucket --endpoint-url=http://host01:80 radosgw-admin sync policy get --bucket=BUCKET_NAME radosgw-admin sync policy get --bucket=testbucket cat user_policy.json { "Version":"2012-10-17", "Statement": { "Effect":"Deny", "Action": [ "s3:PutReplicationConfiguration", "s3:GetReplicationConfiguration", "s3:DeleteReplicationConfiguration" ], "Resource": "arn:aws:s3:::*", } } aws --endpoint-url=ENDPOINT_URL iam put-user-policy --user-name USER_NAME --policy-name USER_POLICY_NAME --policy-document POLICY_DOCUMENT_PATH aws --endpoint-url=http://host01:80 iam put-user-policy --user-name newuser1 --policy-name userpolicy --policy-document file://user_policy.json aws --endpoint-url=ENDPOINT_URL iam get-user-policy --user-name USER_NAME --policy-name USER_POLICY_NAME --region us aws --endpoint-url=http://host01:80 iam get-user-policy --user-name newuser1 --policy-name userpolicy --region us [user@client ~]USD vi lifecycle.json { "Rules": [ { "Filter": { "Prefix": "images/" }, "Status": "Enabled", "Expiration": { "Days": 1 }, "ID": "ImageExpiration" } ] } aws --endpoint-url= RADOSGW_ENDPOINT_URL : PORT s3api put-bucket-lifecycle-configuration --bucket BUCKET_NAME --lifecycle-configuration file:// PATH_TO_LIFECYCLE_CONFIGURATION_FILE / LIFECYCLE_CONFIGURATION_FILE .json [user@client ~]USD aws --endpoint-url=http://host01:80 s3api put-bucket-lifecycle-configuration --bucket testbucket --lifecycle-configuration file://lifecycle.json aws --endpoint-url= RADOSGW_ENDPOINT_URL : PORT s3api get-bucket-lifecycle-configuration --bucket BUCKET_NAME [user@client ~]USD aws --endpoint-url=http://host01:80 s3api get-bucket-lifecycle-configuration --bucket testbucket { "Rules": [ { "Expiration": { "Days": 1 }, "ID": "ImageExpiration", "Filter": { "Prefix": "images/" }, "Status": "Enabled" } ] } radosgw-admin lc get --bucket= BUCKET_NAME radosgw-admin lc get --bucket=testbucket { "prefix_map": { "images/": { "status": true, "dm_expiration": false, "expiration": 1, "noncur_expiration": 0, "mp_expiration": 0, "transitions": {}, "noncur_transitions": {} } }, "rule_map": [ { "id": "ImageExpiration", "rule": { "id": "ImageExpiration", "prefix": "", "status": "Enabled", "expiration": { "days": "1", "date": "" }, "mp_expiration": { "days": "", "date": "" }, "filter": { "prefix": "images/", "obj_tags": { "tagset": {} } }, "transitions": {}, "noncur_transitions": {}, "dm_expiration": false } } ] } aws --endpoint-url= RADOSGW_ENDPOINT_URL : PORT s3api delete-bucket-lifecycle --bucket BUCKET_NAME [user@client ~]USD aws --endpoint-url=http://host01:80 s3api delete-bucket-lifecycle --bucket testbucket aws --endpoint-url= RADOSGW_ENDPOINT_URL : PORT s3api get-bucket-lifecycle-configuration --bucket BUCKET_NAME aws --endpoint-url=http://host01:80 s3api get-bucket-lifecycle-configuration --bucket testbucket radosgw-admin lc get --bucket= BUCKET_NAME radosgw-admin lc get --bucket=testbucket [user@client ~]USD vi lifecycle.json { "Rules": [ { "Filter": { "Prefix": "images/" }, "Status": "Enabled", "Expiration": { "Days": 1 }, "ID": "ImageExpiration" }, { "Filter": { "Prefix": "docs/" }, "Status": "Enabled", "Expiration": { "Days": 30 }, "ID": "DocsExpiration" } ] } aws --endpoint-url= RADOSGW_ENDPOINT_URL : PORT s3api put-bucket-lifecycle-configuration --bucket BUCKET_NAME --lifecycle-configuration file:// PATH_TO_LIFECYCLE_CONFIGURATION_FILE / LIFECYCLE_CONFIGURATION_FILE .json [user@client ~]USD aws --endpoint-url=http://host01:80 s3api put-bucket-lifecycle-configuration --bucket testbucket --lifecycle-configuration file://lifecycle.json aws --endpointurl= RADOSGW_ENDPOINT_URL : PORT s3api get-bucket-lifecycle-configuration --bucket BUCKET_NAME [user@client ~]USD aws -endpoint-url=http://host01:80 s3api get-bucket-lifecycle-configuration --bucket testbucket { "Rules": [ { "Expiration": { "Days": 30 }, "ID": "DocsExpiration", "Filter": { "Prefix": "docs/" }, "Status": "Enabled" }, { "Expiration": { "Days": 1 }, "ID": "ImageExpiration", "Filter": { "Prefix": "images/" }, "Status": "Enabled" } ] } radosgw-admin lc get --bucket= BUCKET_NAME radosgw-admin lc get --bucket=testbucket { "prefix_map": { "docs/": { "status": true, "dm_expiration": false, "expiration": 1, "noncur_expiration": 0, "mp_expiration": 0, "transitions": {}, "noncur_transitions": {} }, "images/": { "status": true, "dm_expiration": false, "expiration": 1, "noncur_expiration": 0, "mp_expiration": 0, "transitions": {}, "noncur_transitions": {} } }, "rule_map": [ { "id": "DocsExpiration", "rule": { "id": "DocsExpiration", "prefix": "", "status": "Enabled", "expiration": { "days": "30", "date": "" }, "noncur_expiration": { "days": "", "date": "" }, "mp_expiration": { "days": "", "date": "" }, "filter": { "prefix": "docs/", "obj_tags": { "tagset": {} } }, "transitions": {}, "noncur_transitions": {}, "dm_expiration": false } }, { "id": "ImageExpiration", "rule": { "id": "ImageExpiration", "prefix": "", "status": "Enabled", "expiration": { "days": "1", "date": "" }, "mp_expiration": { "days": "", "date": "" }, "filter": { "prefix": "images/", "obj_tags": { "tagset": {} } }, "transitions": {}, "noncur_transitions": {}, "dm_expiration": false } } ] } cephadm shell radosgw-admin lc list [ { "bucket": ":testbucket:8b63d584-9ea1-4cf3-8443-a6a15beca943.54187.1", "started": "Thu, 01 Jan 1970 00:00:00 GMT", "status" : "UNINITIAL" }, { "bucket": ":testbucket1:8b635499-9e41-4cf3-8443-a6a15345943.54187.2", "started": "Thu, 01 Jan 1970 00:00:00 GMT", "status" : "UNINITIAL" } ] radosgw-admin lc process --bucket= BUCKET_NAME radosgw-admin lc process --bucket=testbucket1 radosgw-admin lc process radosgw-admin lc list [ { "bucket": ":testbucket:8b63d584-9ea1-4cf3-8443-a6a15beca943.54187.1", "started": "Thu, 17 Mar 2022 21:48:50 GMT", "status" : "COMPLETE" } { "bucket": ":testbucket1:8b635499-9e41-4cf3-8443-a6a15345943.54187.2", "started": "Thu, 17 Mar 2022 20:38:50 GMT", "status" : "COMPLETE" } ] cephadm shell ceph config set client.rgw rgw_lifecycle_work_time %D:%D-%D:%D ceph config set client.rgw rgw_lifecycle_work_time 06:00-08:00 ceph config get client.rgw rgw_lifecycle_work_time 06:00-08:00 ceph osd pool create POOL_NAME ceph osd pool create test.hot.data radosgw-admin zonegroup placement add --rgw-zonegroup default --placement-id PLACEMENT_TARGET --storage-class STORAGE_CLASS radosgw-admin zonegroup placement add --rgw-zonegroup default --placement-id default-placement --storage-class hot.test { "key": "default-placement", "val": { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD", "hot.test" ] } } radosgw-admin zone placement add --rgw-zone default --placement-id PLACEMENT_TARGET --storage-class STORAGE_CLASS --data-pool DATA_POOL radosgw-admin zone placement add --rgw-zone default --placement-id default-placement --storage-class hot.test --data-pool test.hot.data { "key": "default-placement", "val": { "index_pool": "test_zone.rgw.buckets.index", "storage_classes": { "STANDARD": { "data_pool": "test.hot.data" }, "hot.test": { "data_pool": "test.hot.data", } }, "data_extra_pool": "", "index_type": 0 } ceph osd pool application enable POOL_NAME rgw ceph osd pool application enable test.hot.data rgw enabled application 'rgw' on pool 'test.hot.data' aws s3api create-bucket --bucket testbucket10 --create-bucket-configuration LocationConstraint=default:default-placement --endpoint-url http://10.0.0.80:8080 aws --endpoint=http://10.0.0.80:8080 s3api put-object --bucket testbucket10 --key compliance-upload --body /root/test2.txt ceph osd pool create POOL_NAME ceph osd pool create test.cold.data radosgw-admin zonegroup placement add --rgw-zonegroup default --placement-id PLACEMENT_TARGET --storage-class STORAGE_CLASS radosgw-admin zonegroup placement add --rgw-zonegroup default --placement-id default-placement --storage-class cold.test { "key": "default-placement", "val": { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD", "cold.test" ] } } radosgw-admin zone placement add --rgw-zone default --placement-id PLACEMENT_TARGET --storage-class STORAGE_CLASS --data-pool DATA_POOL radosgw-admin zone placement add --rgw-zone default --placement-id default-placement --storage-class cold.test --data-pool test.cold.data ceph osd pool application enable POOL_NAME rgw ceph osd pool application enable test.cold.data rgw enabled application 'rgw' on pool 'test.cold.data' radosgw-admin zonegroup get { "id": "3019de59-ddde-4c5c-b532-7cdd29de09a1", "name": "default", "api_name": "default", "is_master": "true", "endpoints": [], "hostnames": [], "hostnames_s3website": [], "master_zone": "adacbe1b-02b4-41b8-b11d-0d505b442ed4", "zones": [ { "id": "adacbe1b-02b4-41b8-b11d-0d505b442ed4", "name": "default", "endpoints": [], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 11, "read_only": "false", "tier_type": "", "sync_from_all": "true", "sync_from": [], "redirect_zone": "" } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "hot.test", "cold.test", "STANDARD" ] } ], "default_placement": "default-placement", "realm_id": "", "sync_policy": { "groups": [] } } radosgw-admin zone get { "id": "adacbe1b-02b4-41b8-b11d-0d505b442ed4", "name": "default", "domain_root": "default.rgw.meta:root", "control_pool": "default.rgw.control", "gc_pool": "default.rgw.log:gc", "lc_pool": "default.rgw.log:lc", "log_pool": "default.rgw.log", "intent_log_pool": "default.rgw.log:intent", "usage_log_pool": "default.rgw.log:usage", "roles_pool": "default.rgw.meta:roles", "reshard_pool": "default.rgw.log:reshard", "user_keys_pool": "default.rgw.meta:users.keys", "user_email_pool": "default.rgw.meta:users.email", "user_swift_pool": "default.rgw.meta:users.swift", "user_uid_pool": "default.rgw.meta:users.uid", "otp_pool": "default.rgw.otp", "system_key": { "access_key": "", "secret_key": "" }, "placement_pools": [ { "key": "default-placement", "val": { "index_pool": "default.rgw.buckets.index", "storage_classes": { "cold.test": { "data_pool": "test.cold.data" }, "hot.test": { "data_pool": "test.hot.data" }, "STANDARD": { "data_pool": "default.rgw.buckets.data" } }, "data_extra_pool": "default.rgw.buckets.non-ec", "index_type": 0 } } ], "realm_id": "", "notif_pool": "default.rgw.log:notif" } aws s3api create-bucket --bucket testbucket10 --create-bucket-configuration LocationConstraint=default:default-placement --endpoint-url http://10.0.0.80:8080 radosgw-admin bucket list --bucket testbucket10 { "ETag": "\"211599863395c832a3dfcba92c6a3b90\"", "Size": 540, "StorageClass": "STANDARD", "Key": "obj1", "VersionId": "W95teRsXPSJI4YWJwwSG30KxSCzSgk-", "IsLatest": true, "LastModified": "2023-11-23T10:38:07.214Z", "Owner": { "DisplayName": "test-user", "ID": "test-user" } } vi lifecycle.json { "Rules": [ { "Filter": { "Prefix": "" }, "Status": "Enabled", "Transitions": [ { "Days": 5, "StorageClass": "hot.test" }, { "Days": 20, "StorageClass": "cold.test" } ], "Expiration": { "Days": 365 }, "ID": "double transition and expiration" } ] } aws s3api put-bucket-lifecycle-configuration --bucket testbucket10 --lifecycle-configuration file://lifecycle.json aws s3api get-bucket-lifecycle-configuration --bucket testbucke10 { "Rules": [ { "Expiration": { "Days": 365 }, "ID": "double transition and expiration", "Prefix": "", "Status": "Enabled", "Transitions": [ { "Days": 20, "StorageClass": "cold.test" }, { "Days": 5, "StorageClass": "hot.test" } ] } ] } radosgw-admin bucket list --bucket testbucket10 { "ETag": "\"211599863395c832a3dfcba92c6a3b90\"", "Size": 540, "StorageClass": "cold.test", "Key": "obj1", "VersionId": "W95teRsXPSJI4YWJwwSG30KxSCzSgk-", "IsLatest": true, "LastModified": "2023-11-23T10:38:07.214Z", "Owner": { "DisplayName": "test-user", "ID": "test-user" } } aws --endpoint=http:// RGW_PORT :8080 s3api create-bucket --bucket BUCKET_NAME --object-lock-enabled-for-bucket aws --endpoint=http://rgw.ceph.com:8080 s3api create-bucket --bucket worm-bucket --object-lock-enabled-for-bucket aws --endpoint=http:// RGW_PORT :8080 s3api put-object-lock-configuration --bucket BUCKET_NAME --object-lock-configuration '{ "ObjectLockEnabled": "Enabled", "Rule": { "DefaultRetention": { "Mode": " RETENTION_MODE ", "Days": NUMBER_OF_DAYS }}}' aws --endpoint=http://rgw.ceph.com:8080 s3api put-object-lock-configuration --bucket worm-bucket --object-lock-configuration '{ "ObjectLockEnabled": "Enabled", "Rule": { "DefaultRetention": { "Mode": "COMPLIANCE", "Days": 10 }}}' aws --endpoint=http:// RGW_PORT :8080 s3api put-object --bucket BUCKET_NAME --object-lock-mode RETENTION_MODE --object-lock-retain-until-date " DATE " --key compliance-upload --body TEST_FILE aws --endpoint=http://rgw.ceph.com:8080 s3api put-object --bucket worm-bucket --object-lock-mode COMPLIANCE --object-lock-retain-until-date "2022-05-31" --key compliance-upload --body test.dd { "ETag": "\"d560ea5652951637ba9c594d8e6ea8c1\"", "VersionId": "Nhhk5kRS6Yp6dZXVWpZZdRcpSpBKToD" } aws --endpoint=http:// RGW_PORT :8080 s3api put-object --bucket BUCKET_NAME --object-lock-mode RETENTION_MODE --object-lock-retain-until-date " DATE " --key compliance-upload --body PATH aws --endpoint=http://rgw.ceph.com:8080 s3api put-object --bucket worm-bucket --object-lock-mode COMPLIANCE --object-lock-retain-until-date "2022-05-31" --key compliance-upload --body /etc/fstab { "ETag": "\"d560ea5652951637ba9c594d8e6ea8c1\"", "VersionId": "Nhhk5kRS6Yp6dZXVWpZZdRcpSpBKToD" } aws --endpoint=http://rgw.ceph.com:8080 s3api put-object-legal-hold --bucket worm-bucket --key compliance-upload --legal-hold Status=ON aws --endpoint=http://rgw.ceph.com:8080 s3api list-objects --bucket worm-bucket aws --endpoint=http://rgw.ceph.com:8080 s3api list-objects --bucket worm-bucket { "Versions": [ { "ETag": "\"d560ea5652951637ba9c594d8e6ea8c1\"", "Size": 288, "StorageClass": "STANDARD", "Key": "hosts", "VersionId": "Nhhk5kRS6Yp6dZXVWpZZdRcpSpBKToD", "IsLatest": true, "LastModified": "2022-06-17T08:51:17.392000+00:00", "Owner": { "DisplayName": "Test User in Tenant test", "ID": "testUSDtest.user" } } } ] } aws --endpoint=http://rgw.ceph.com:8080 s3api get-object --bucket worm-bucket --key compliance-upload --version-id 'IGOU.vdIs3SPduZglrB-RBaK.sfXpcd' download.1 { "AcceptRanges": "bytes", "LastModified": "2022-06-17T08:51:17+00:00", "ContentLength": 288, "ETag": "\"d560ea5652951637ba9c594d8e6ea8c1\"", "VersionId": "Nhhk5kRS6Yp6dZXVWpZZdRcpSpBKToD", "ContentType": "binary/octet-stream", "Metadata": {}, "ObjectLockMode": "COMPLIANCE", "ObjectLockRetainUntilDate": "2023-06-17T08:51:17+00:00" } radosgw-admin usage show --uid=johndoe --start-date=2022-06-01 --end-date=2022-07-01 radosgw-admin usage show --show-log-entries=false radosgw-admin usage trim --start-date=2022-06-01 --end-date=2022-07-31 radosgw-admin usage trim --uid=johndoe radosgw-admin usage trim --uid=johndoe --end-date=2021-04-31 radosgw-admin metadata get bucket: BUCKET_NAME radosgw-admin metadata get bucket.instance: BUCKET : BUCKET_ID radosgw-admin metadata get user: USER radosgw-admin metadata set user: USER radosgw-admin metadata list radosgw-admin metadata list bucket radosgw-admin metadata list bucket.instance radosgw-admin metadata list user .bucket.meta.prodtx:test%25star:default.84099.6 .bucket.meta.testcont:default.4126.1 .bucket.meta.prodtx:testcont:default.84099.4 prodtx/testcont prodtx/test%25star testcont prodtxUSDprodt test2.buckets prodtxUSDprodt.buckets test2 radosgw-admin ratelimit set --ratelimit-scope=user --uid= USER_ID [--max-read-ops= NUMBER_OF_OPERATIONS ] [--max-read-bytes= NUMBER_OF_BYTES ] [--max-write-ops= NUMBER_OF_OPERATIONS ] [--max-write-bytes= NUMBER_OF_BYTES ] radosgw-admin ratelimit set --ratelimit-scope=user --uid=testing --max-read-ops=1024 --max-write-bytes=10240 radosgw-admin ratelimit get --ratelimit-scope=user --uid= USER_ID radosgw-admin ratelimit get --ratelimit-scope=user --uid=testing { "user_ratelimit": { "max_read_ops": 1024, "max_write_ops": 0, "max_read_bytes": 0, "max_write_bytes": 10240, "enabled": false } } radosgw-admin ratelimit enable --ratelimit-scope=user --uid= USER_ID radosgw-admin ratelimit enable --ratelimit-scope=user --uid=testing { "user_ratelimit": { "max_read_ops": 1024, "max_write_ops": 0, "max_read_bytes": 0, "max_write_bytes": 10240, "enabled": true } } radosgw-admin ratelimit disable --ratelimit-scope=user --uid= USER_ID radosgw-admin ratelimit disable --ratelimit-scope=user --uid=testing radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket= BUCKET_NAME [--max-read-ops= NUMBER_OF_OPERATIONS ] [--max-read-bytes= NUMBER_OF_BYTES ] [--max-write-ops= NUMBER_OF_OPERATIONS ] [--max-write-bytes= NUMBER_OF_BYTES ] radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=mybucket --max-read-ops=1024 --max-write-bytes=10240 radosgw-admin ratelimit get --ratelimit-scope=bucket --bucket= BUCKET_NAME radosgw-admin ratelimit get --ratelimit-scope=bucket --bucket=mybucket { "bucket_ratelimit": { "max_read_ops": 1024, "max_write_ops": 0, "max_read_bytes": 0, "max_write_bytes": 10240, "enabled": false } } radosgw-admin ratelimit enable --ratelimit-scope=bucket --bucket= BUCKET_NAME radosgw-admin ratelimit enable --ratelimit-scope=bucket --bucket=mybucket { "bucket_ratelimit": { "max_read_ops": 1024, "max_write_ops": 0, "max_read_bytes": 0, "max_write_bytes": 10240, "enabled": true } } radosgw-admin ratelimit disable --ratelimit-scope=bucket --bucket= BUCKET_NAME radosgw-admin ratelimit disable --ratelimit-scope=bucket --bucket=mybucket radosgw-admin global ratelimit get radosgw-admin global ratelimit get { "bucket_ratelimit": { "max_read_ops": 1024, "max_write_ops": 0, "max_read_bytes": 0, "max_write_bytes": 0, "enabled": false }, "user_ratelimit": { "max_read_ops": 0, "max_write_ops": 0, "max_read_bytes": 0, "max_write_bytes": 0, "enabled": false }, "anonymous_ratelimit": { "max_read_ops": 0, "max_write_ops": 0, "max_read_bytes": 0, "max_write_bytes": 0, "enabled": false } } radosgw-admin global ratelimit set --ratelimit-scope=bucket [--max-read-ops= NUMBER_OF_OPERATIONS ] [--max-read-bytes= NUMBER_OF_BYTES ] [--max-write-ops= NUMBER_OF_OPERATIONS ] [--max-write-bytes= NUMBER_OF_BYTES ] radosgw-admin global ratelimit set --ratelimit-scope bucket --max-read-ops=1024 radosgw-admin global ratelimit enable --ratelimit-scope=bucket radosgw-admin global ratelimit enable --ratelimit-scope bucket radosgw-admin global ratelimit set --ratelimit-scope=user [--max-read-ops= NUMBER_OF_OPERATIONS ] [--max-read-bytes= NUMBER_OF_BYTES ] [--max-write-ops= NUMBER_OF_OPERATIONS ] [--max-write-bytes= NUMBER_OF_BYTES ] radosgw-admin global ratelimit set --ratelimit-scope=user --max-read-ops=1024 radosgw-admin global ratelimit enable --ratelimit-scope=user radosgw-admin global ratelimit enable --ratelimit-scope=user radosgw-admin global ratelimit set --ratelimit-scope=anonymous [--max-read-ops= NUMBER_OF_OPERATIONS ] [--max-read-bytes= NUMBER_OF_BYTES ] [--max-write-ops= NUMBER_OF_OPERATIONS ] [--max-write-bytes= NUMBER_OF_BYTES ] radosgw-admin global ratelimit set --ratelimit-scope=anonymous --max-read-ops=1024 radosgw-admin global ratelimit enable --ratelimit-scope=anonymous radosgw-admin global ratelimit enable --ratelimit-scope=anonymous radosgw-admin gc list radosgw-admin gc list ceph config set client.rgw rgw_gc_max_concurrent_io 20 ceph config set client.rgw rgw_gc_max_trim_chunk 64 ceph config set client.rgw rgw_lc_max_worker 7 ceph config set client.rgw rgw_lc_max_wp_worker 7 radosgw-admin user create --uid= USER_NAME --display-name=" DISPLAY_NAME " [--access-key ACCESS_KEY --secret-key SECRET_KEY ] radosgw-admin user create --uid=test-user --display-name="test-user" --access-key a21e86bce636c3aa1 --secret-key cf764951f1fdde5e { "user_id": "test-user", "display_name": "test-user", "email": "", "suspended": 0, "max_buckets": 1000, "subusers": [], "keys": [ { "user": "test-user", "access_key": "a21e86bce636c3aa1", "secret_key": "cf764951f1fdde5e" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "default_storage_class": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw", "mfa_ids": [] } radosgw-admin zonegroup placement add --rgw-zonegroup = ZONE_GROUP_NAME --placement-id= PLACEMENT_ID --storage-class = STORAGE_CLASS_NAME --tier-type=cloud-s3 radosgw-admin zonegroup placement add --rgw-zonegroup=default --placement-id=default-placement --storage-class=CLOUDTIER --tier-type=cloud-s3 [ { "key": "default-placement", "val": { "name": "default-placement", "tags": [], "storage_classes": [ "CLOUDTIER", "STANDARD" ], "tier_targets": [ { "key": "CLOUDTIER", "val": { "tier_type": "cloud-s3", "storage_class": "CLOUDTIER", "retain_head_object": "false", "s3": { "endpoint": "", "access_key": "", "secret": "", "host_style": "path", "target_storage_class": "", "target_path": "", "acl_mappings": [], "multipart_sync_threshold": 33554432, "multipart_min_part_size": 33554432 } } } ] } } ] radosgw-admin zonegroup placement modify --rgw-zonegroup ZONE_GROUP_NAME --placement-id PLACEMENT_ID --storage-class STORAGE_CLASS_NAME --tier-config=endpoint= AWS_ENDPOINT_URL , access_key= AWS_ACCESS_KEY ,secret= AWS_SECRET_KEY , target_path=" TARGET_BUCKET_ON_AWS ", multipart_sync_threshold=44432, multipart_min_part_size=44432, retain_head_object=true region= REGION_NAME radosgw-admin zonegroup placement modify --rgw-zonegroup default --placement-id default-placement --storage-class CLOUDTIER --tier-config=endpoint=http://10.0.210.010:8080, access_key=a21e86bce636c3aa2,secret=cf764951f1fdde5f, target_path="dfqe-bucket-01", multipart_sync_threshold=44432, multipart_min_part_size=44432, retain_head_object=true region=us-east-1 [ { "key": "default-placement", "val": { "name": "default-placement", "tags": [], "storage_classes": [ "CLOUDTIER", "STANDARD", "cold.test", "hot.test" ], "tier_targets": [ { "key": "CLOUDTIER", "val": { "tier_type": "cloud-s3", "storage_class": "CLOUDTIER", "retain_head_object": "true", "s3": { "endpoint": "http://10.0.210.010:8080", "access_key": "a21e86bce636c3aa2", "secret": "cf764951f1fdde5f", "region": "", "host_style": "path", "target_storage_class": "", "target_path": "dfqe-bucket-01", "acl_mappings": [], "multipart_sync_threshold": 44432, "multipart_min_part_size": 44432 } } } ] } } ] ] ceph orch restart CEPH_OBJECT_GATEWAY_SERVICE_NAME ceph orch restart rgw.rgw.1 Scheduled to restart rgw.rgw.1.host03.vkfldf on host 'host03' s3cmd --configure Enter new values or accept defaults in brackets with Enter. Refer to user manual for detailed description of all options. Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables. Access Key: a21e86bce636c3aa2 Secret Key: cf764951f1fdde5f Default Region [US]: Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3. S3 Endpoint [s3.amazonaws.com]: 10.0.210.78:80 Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used if the target S3 system supports dns based buckets. DNS-style bucket+hostname:port template for accessing a bucket [%(bucket)s.s3.amazonaws.com]: 10.0.210.78:80 Encryption password is used to protect your files from reading by unauthorized persons while in transfer to S3 Encryption password: Path to GPG program [/usr/bin/gpg]: When using secure HTTPS protocol all communication with Amazon S3 servers is protected from 3rd party eavesdropping. This method is slower than plain HTTP, and can only be proxied with Python 2.7 or newer Use HTTPS protocol [Yes]: No On some networks all internet access must go through a HTTP proxy. Try setting it here if you can't connect to S3 directly HTTP Proxy server name: New settings: Access Key: a21e86bce636c3aa2 Secret Key: cf764951f1fdde5f Default Region: US S3 Endpoint: 10.0.210.78:80 DNS-style bucket+hostname:port template for accessing a bucket: 10.0.210.78:80 Encryption password: Path to GPG program: /usr/bin/gpg Use HTTPS protocol: False HTTP Proxy server name: HTTP Proxy server port: 0 Test access with supplied credentials? [Y/n] Y Please wait, attempting to list all buckets Success. Your access key and secret key worked fine :-) Now verifying that encryption works Not configured. Never mind. Save settings? [y/N] y Configuration saved to '/root/.s3cfg' s3cmd mb s3:// NAME_OF_THE_BUCKET_FOR_S3 s3cmd mb s3://awstestbucket Bucket 's3://awstestbucket/' created s3cmd put FILE_NAME s3:// NAME_OF_THE_BUCKET_ON_S3 s3cmd put test.txt s3://awstestbucket upload: 'test.txt' -> 's3://awstestbucket/test.txt' [1 of 1] 21 of 21 100% in 1s 16.75 B/s done <LifecycleConfiguration> <Rule> <ID> RULE_NAME </ID> <Filter> <Prefix></Prefix> </Filter> <Status>Enabled</Status> <Transition> <Days> DAYS </Days> <StorageClass> STORAGE_CLASS_NAME </StorageClass> </Transition> </Rule> </LifecycleConfiguration> cat lc_cloud.xml <LifecycleConfiguration> <Rule> <ID>Archive all objects</ID> <Filter> <Prefix></Prefix> </Filter> <Status>Enabled</Status> <Transition> <Days>2</Days> <StorageClass>CLOUDTIER</StorageClass> </Transition> </Rule> </LifecycleConfiguration> s3cmd setlifecycle FILE_NAME s3:// NAME_OF_THE_BUCKET_FOR_S3 s3cmd setlifecycle lc_config.xml s3://awstestbucket s3://awstestbucket/: Lifecycle Policy updated cephadm shell ceph orch restart CEPH_OBJECT_GATEWAY_SERVICE_NAME ceph orch restart rgw.rgw.1 Scheduled to restart rgw.rgw.1.host03.vkfldf on host 'host03' radosgw-admin lc list [ { "bucket": ":awstestbucket:552a3adb-39e0-40f6-8c84-00590ed70097.54639.1", "started": "Mon, 26 Sep 2022 18:32:07 GMT", "status": "COMPLETE" } ] [root@client ~]USD radosgw-admin bucket list [ "awstestbucket" ] [root@host01 ~]USD aws s3api list-objects --bucket awstestbucket --endpoint=http://10.0.209.002:8080 { "Contents": [ { "Key": "awstestbucket/test", "LastModified": "2022-08-25T16:14:23.118Z", "ETag": "\"378c905939cc4459d249662dfae9fd6f\"", "Size": 29, "StorageClass": "STANDARD", "Owner": { "DisplayName": "test-user", "ID": "test-user" } } ] } s3cmd ls s3://awstestbucket 2022-08-25 09:57 0 s3://awstestbucket/test.txt s3cmd info s3://awstestbucket/test.txt s3://awstestbucket/test.txt (object): File size: 0 Last mod: Mon, 03 Aug 2022 09:57:49 GMT MIME type: text/plain Storage: CLOUDTIER MD5 sum: 991d2528bb41bb839d1a9ed74b710794 SSE: none Policy: none CORS: none ACL: test-user: FULL_CONTROL x-amz-meta-s3cmd-attrs: atime:1664790668/ctime:1664790668/gid:0/gname:root/md5:991d2528bb41bb839d1a9ed74b710794/mode:33188/mtime:1664790668/uid:0/uname:root [client@client01 ~]USD aws configure AWS Access Key ID [****************6VVP]: AWS Secret Access Key [****************pXqy]: Default region name [us-east-1]: Default output format [json]: [client@client01 ~]USD aws s3 ls s3://dfqe-bucket-01/awstest PRE awstestbucket/ [client@client01 ~]USD aws s3 cp s3://dfqe-bucket-01/awstestbucket/test.txt . download: s3://dfqe-bucket-01/awstestbucket/test.txt to ./test.txt radosgw-admin user create --uid= USER_NAME --display-name=" DISPLAY_NAME " [--access-key ACCESS_KEY --secret-key SECRET_KEY ] radosgw-admin user create --uid=test-user --display-name="test-user" --access-key a21e86bce636c3aa1 --secret-key cf764951f1fdde5e { "user_id": "test-user", "display_name": "test-user", "email": "", "suspended": 0, "max_buckets": 1000, "subusers": [], "keys": [ { "user": "test-user", "access_key": "a21e86bce636c3aa1", "secret_key": "cf764951f1fdde5e" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "default_storage_class": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw", "mfa_ids": [] } aws s3 --ca-bundle CA_PERMISSION --profile rgw --endpoint ENDPOINT_URL --region default mb s3:// BUCKET_NAME [root@host01 ~]USD aws s3 --ca-bundle /etc/pki/ca-trust/source/anchors/myCA.pem --profile rgw --endpoint https://host02.example.com:8043 --region default mb s3://transition radosgw-admin bucket stats --bucket transition { "bucket": "transition", "num_shards": 11, "tenant": "", "zonegroup": "b29b0e50-1301-4330-99fc-5cdcfc349acf", "placement_rule": "default-placement", "explicit_placement": { "data_pool": "", "data_extra_pool": "", "index_pool": "" }, [root@host01 ~]USD oc project openshift-storage [root@host01 ~]USD oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.11.6 True False 4d1h Cluster version is 4.11.6 [root@host01 ~]USD oc get storagecluster NAME AGE PHASE EXTERNAL CREATED AT VERSION ocs-storagecluster 4d Ready 2023-06-27T15:23:01Z 4.11.0 noobaa namespacestore create azure-blob az --account-key=' ACCOUNT_KEY ' --account-name=' ACCOUNT_NAME' --target-blob-container='_AZURE_CONTAINER_NAME ' [root@host01 ~]USD noobaa namespacestore create azure-blob az --account-key='iq3+6hRtt9bQ46QfHKQ0nSm2aP+tyMzdn8dBSRW4XWrFhY+1nwfqEj4hk2q66nmD85E/o5OrrUqo+AStkKwm9w==' --account-name='transitionrgw' --target-blob-container='mcgnamespace' [root@host01 ~]USD noobaa bucketclass create namespace-bucketclass single aznamespace-bucket-class --resource az -n openshift-storage noobaa obc create OBC_NAME --bucketclass aznamespace-bucket-class -n openshift-storage [root@host01 ~]USD noobaa obc create rgwobc --bucketclass aznamespace-bucket-class -n openshift-storage radosgw-admin zonegroup placement add --rgw-zonegroup = ZONE_GROUP_NAME --placement-id= PLACEMENT_ID --storage-class = STORAGE_CLASS_NAME --tier-type=cloud-s3 radosgw-admin zonegroup placement add --rgw-zonegroup=default --placement-id=default-placement --storage-class=AZURE --tier-type=cloud-s3 [ { "key": "default-placement", "val": { "name": "default-placement", "tags": [], "storage_classes": [ "AZURE", "STANDARD" ], "tier_targets": [ { "key": "AZURE", "val": { "tier_type": "cloud-s3", "storage_class": "AZURE", "retain_head_object": "false", "s3": { "endpoint": "", "access_key": "", "secret": "", "host_style": "path", "target_storage_class": "", "target_path": "", "acl_mappings": [], "multipart_sync_threshold": 33554432, "multipart_min_part_size": 33554432 } } } ] } } ] radosgw-admin zonegroup placement modify --rgw-zonegroup ZONE_GROUP_NAME --placement-id PLACEMENT_ID --storage-class STORAGE_CLASS_NAME --tier-config=endpoint= ENDPOINT_URL , access_key= ACCESS_KEY ,secret= SECRET_KEY , target_path=" TARGET_BUCKET_ON ", multipart_sync_threshold=44432, multipart_min_part_size=44432, retain_head_object=true region= REGION_NAME radosgw-admin zonegroup placement modify --rgw-zonegroup default --placement-id default-placement --storage-class AZURE --tier-config=endpoint="https://s3-openshift-storage.apps.ocp410.0e73azopenshift.com", access_key=a21e86bce636c3aa2,secret=cf764951f1fdde5f, target_path="dfqe-bucket-01", multipart_sync_threshold=44432, multipart_min_part_size=44432, retain_head_object=true region=us-east-1 [ { "key": "default-placement", "val": { "name": "default-placement", "tags": [], "storage_classes": [ "AZURE", "STANDARD", "cold.test", "hot.test" ], "tier_targets": [ { "key": "AZURE", "val": { "tier_type": "cloud-s3", "storage_class": "AZURE", "retain_head_object": "true", "s3": { "endpoint": "https://s3-openshift-storage.apps.ocp410.0e73azopenshift.com", "access_key": "a21e86bce636c3aa2", "secret": "cf764951f1fdde5f", "region": "", "host_style": "path", "target_storage_class": "", "target_path": "dfqe-bucket-01", "acl_mappings": [], "multipart_sync_threshold": 44432, "multipart_min_part_size": 44432 } } } ] } } ] ] ceph orch restart CEPH_OBJECT_GATEWAY_SERVICE_NAME ceph orch restart client.rgw.objectgwhttps.host02.udyllp Scheduled to restart client.rgw.objectgwhttps.host02.udyllp on host 'host02 cat transition.json { "Rules": [ { "Filter": { "Prefix": "" }, "Status": "Enabled", "Transitions": [ { "Days": 30, "StorageClass": " STORAGE_CLASS " } ], "ID": " TRANSITION_ID " } ] } [root@host01 ~]USD cat transition.json { "Rules": [ { "Filter": { "Prefix": "" }, "Status": "Enabled", "Transitions": [ { "Days": 30, "StorageClass": "AZURE" } ], "ID": "Transition Objects in bucket to AZURE Blob after 30 days" } ] } aws s3api --ca-bundle CA_PERMISSION --profile rgw --endpoint ENDPOINT_URL --region default put-bucket-lifecycle-configuration --lifecycle-configuration file:// BUCKET .json --bucket BUCKET_NAME [root@host01 ~]USD aws s3api --ca-bundle /etc/pki/ca-trust/source/anchors/myCA.pem --profile rgw --endpoint https://host02.example.com:8043 --region default put-bucket-lifecycle-configuration --lifecycle-configuration file://transition.json --bucket transition aws s3api --ca-bundle CA_PERMISSION --profile rgw --endpoint ENDPOINT_URL --region default get-bucket-lifecycle-configuration --lifecycle-configuration file:// BUCKET .json --bucket BUCKET_NAME [root@host01 ~]USD aws s3api --ca-bundle /etc/pki/ca-trust/source/anchors/myCA.pem --profile rgw --endpoint https://host02.example.com:8043 --region default get-bucket-lifecycle-configuration --bucket transition { "Rules": [ { "ID": "Transition Objects in bucket to AZURE Blob after 30 days", "Prefix": "", "Status": "Enabled", "Transitions": [ { "Days": 30, "StorageClass": "AZURE" } ] } ] } radosgw-admin lc list [ { "bucket": ":transition:d9c4f708-5598-4c44-9d36-849552a08c4d.169377.1", "started": "Thu, 01 Jan 1970 00:00:00 GMT", "status": "UNINITIAL" } ] cephadm shell ceph orch daemon CEPH_OBJECT_GATEWAY_DAEMON_NAME ceph orch daemon restart rgw.objectgwhttps.host02.udyllp ceph orch daemon restart rgw.objectgw.host02.afwvyq ceph orch daemon restart rgw.objectgw.host05.ucpsrr for i in 1 2 3 4 5 do aws s3 --ca-bundle /etc/pki/ca-trust/source/anchors/myCA.pem --profile rgw --endpoint https://host02.example.com:8043 --region default cp /etc/hosts s3://transition/transitionUSDi done aws s3 --ca-bundle /etc/pki/ca-trust/source/anchors/myCA.pem --profile rgw --endpoint https://host02.example.com:8043 --region default ls s3://transition 2023-06-30 10:24:01 3847 transition1 2023-06-30 10:24:04 3847 transition2 2023-06-30 10:24:07 3847 transition3 2023-06-30 10:24:09 3847 transition4 2023-06-30 10:24:13 3847 transition5 rados ls -p default.rgw.buckets.data | grep transition d9c4f708-5598-4c44-9d36-849552a08c4d.169377.1_transition1 d9c4f708-5598-4c44-9d36-849552a08c4d.169377.1_transition4 d9c4f708-5598-4c44-9d36-849552a08c4d.169377.1_transition2 d9c4f708-5598-4c44-9d36-849552a08c4d.169377.1_transition3 d9c4f708-5598-4c44-9d36-849552a08c4d.169377.1_transition5 radosgw-admin lc process radosgw-admin lc list [ { "bucket": ":transition:d9c4f708-5598-4c44-9d36-849552a08c4d.170017.5", "started": "Mon, 30 Jun 2023-06-30 16:52:56 GMT", "status": "COMPLETE" } ] [root@host01 ~]USD aws s3api list-objects --bucket awstestbucket --endpoint=http://10.0.209.002:8080 { "Contents": [ { "Key": "awstestbucket/test", "LastModified": "2023-06-25T16:14:23.118Z", "ETag": "\"378c905939cc4459d249662dfae9fd6f\"", "Size": 29, "StorageClass": "STANDARD", "Owner": { "DisplayName": "test-user", "ID": "test-user" } } ] } [root@host01 ~]USD aws s3 --ca-bundle /etc/pki/ca-trust/source/anchors/myCA.pem --profile rgw --endpoint https://host02.example.com:8043 --region default ls s3://transition 2023-06-30 17:52:56 0 transition1 2023-06-30 17:51:59 0 transition2 2023-06-30 17:51:59 0 transition3 2023-06-30 17:51:58 0 transition4 2023-06-30 17:51:59 0 transition5 [root@host01 ~]USD aws s3api --ca-bundle /etc/pki/ca-trust/source/anchors/myCA.pem --profile rgw --endpoint https://host02.example.com:8043 --region default head-object --key transition1 --bucket transition { "AcceptRanges": "bytes", "LastModified": "2023-06-31T16:52:56+00:00", "ContentLength": 0, "ETag": "\"46ecb42fd0def0e42f85922d62d06766\"", "ContentType": "binary/octet-stream", "Metadata": {}, "StorageClass": "CLOUDTIER" } radosgw-admin account create [--account-name={name}] [--account-id={id}] [--email={email}] radosgw-admin account create --account-name=user1 --account-id=12345 --email=user1@example.com radosgw-admin user create --uid={userid} --display-name={name} --account-id={accountid} --account-root --gen-secret --gen-access-key radosgw-admin user create --uid=rootuser1 --display-name="Root User One" --account-id=account123 --account-root --gen-secret --gen-access-key radosgw-admin account rm --account-id={accountid} radosgw-admin account rm --account-id=account123 radosgw-admin account stats --account-id={accountid} --sync-stats { "account": "account123", "data_size": 3145728000, # Total size in bytes (3 GB) "num_objects": 12000, # Total number of objects "num_buckets": 5, # Total number of buckets "usage": { "total_size": 3145728000, # Total size in bytes (3 GB) "num_objects": 12000 } } radosgw-admin quota set --quota-scope=account --account-id={accountid} --max-size=10G radosgw-admin quota enable --quota-scope=account --account-id={accountid} { "status": "OK", "message": "Quota enabled for account account123" } radosgw-admin quota set --quota-scope=bucket --account-id={accountid} --max-objects=1000000 radosgw-admin quota enable --quota-scope=bucket --account-id={accountid} { "status": "OK", "message": "Quota enabled for bucket in account account123" } radosgw-admin quota set --quota-scope=account --account-id RGW12345678901234568 --max-buckets 10000 { "id": "RGW12345678901234568", "tenant": "tenant1", "name": "account1", "email": "tenataccount1", "quota": { "enabled": true, "check_on_raw": false, "max_size": 10737418240, "max_size_kb": 10485760, "max_objects": 100 }, "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "max_users": 1000, "max_roles": 1000, "max_groups": 1000, "max_buckets": 1000, "max_access_keys": 4 } radosgw-admin quota enable --quota-scope=account --account-id RGW12345678901234568 { "id": "RGW12345678901234568", "tenant": "tenant1", "name": "account1", "email": "tenataccount1", "quota": { "enabled": true, "check_on_raw": false, "max_size": 10737418240, "max_size_kb": 10485760, "max_objects": 100 }, "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "max_users": 1000, "max_roles": 1000, "max_groups": 1000, "max_buckets": 1000, "max_access_keys": 4 } radosgw-admin account get --account-id RGW12345678901234568 { "id": "RGW12345678901234568", "tenant": "tenant1", "name": "account1", "email": "tenataccount1", "quota": { "enabled": true, "check_on_raw": false, "max_size": 10737418240, "max_size_kb": 10485760, "max_objects": 100 }, "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "max_users": 1000, "max_roles": 1000, "max_groups": 1000, "max_buckets": 1000, "max_access_keys": 4 } ceph versions { "mon": { "ceph version 19.1.1-63.el9cp (8fa7b56d5e9f208c4233b0a8273665087bded8ae) squid (rc)": 3 }, "mgr": { "ceph version 19.1.1-63.el9cp (8fa7b56d5e9f208c4233b0a8273665087bded8ae) squid (rc)": 3 }, "osd": { "ceph version 19.1.1-63.el9cp (8fa7b56d5e9f208c4233b0a8273665087bded8ae) squid (rc)": 9 }, "rgw": { "ceph version 19.1.1-63.el9cp (8fa7b56d5e9f208c4233b0a8273665087bded8ae) squid (rc)": 3 }, "overall": { "ceph version 19.1.1-63.el9cp (8fa7b56d5e9f208c4233b0a8273665087bded8ae) squid (rc)": 18 } } radosgw-admin user modify --uid={userid} --account-id={accountid} {"TopicConfigurations": [{ "Id": "ID1", "TopicArn": "arn:aws:sns:default::topic1", "Events": ["s3:ObjectCreated:*"]}]} {"TopicConfigurations": [{ "Id": "ID1", "TopicArn": "arn:aws:sns:default:RGW00000000000000001:topic1", "Events": ["s3:ObjectCreated:*"]}]} radosgw-admin topic rm --topic topic1 radosgw-admin user modify --uid <user_ID> --account-id <Account_ID> --account-root radosgw-admin user policy attach --uid <user_ID> --policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess radosgw-admin user modify --uid <user_ID> --account-root=0 radosgw-admin user create --uid= name --display-name=" USER_NAME " radosgw-admin user create --uid="testuser" --display-name="Jane Doe" { "user_id": "testuser", "display_name": "Jane Doe", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "testuser", "access_key": "CEP28KDIQXBKU4M15PDC", "secret_key": "MARoio8HFc8JxhEilES3dKFVj8tV3NOOYymihTLO" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw" } radosgw-admin subuser create --uid= NAME --subuser= NAME :swift --access=full radosgw-admin subuser create --uid=testuser --subuser=testuser:swift --access=full { "user_id": "testuser", "display_name": "First User", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [ { "id": "testuser:swift", "permissions": "full-control" } ], "keys": [ { "user": "testuser", "access_key": "O8JDE41XMI74O185EHKD", "secret_key": "i4Au2yxG5wtr1JK01mI8kjJPM93HNAoVWOSTdJd6" } ], "swift_keys": [ { "user": "testuser:swift", "secret_key": "13TLtdEW7bCqgttQgPzxFxziu0AgabtOc6vM8DLA" } ], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw" } radosgw-admin key create --subuser= NAME :swift --key-type=swift --gen-secret radosgw-admin key create --subuser=testuser:swift --key-type=swift --gen-secret { "user_id": "testuser", "display_name": "First User", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [ { "id": "testuser:swift", "permissions": "full-control" } ], "keys": [ { "user": "testuser", "access_key": "O8JDE41XMI74O185EHKD", "secret_key": "i4Au2yxG5wtr1JK01mI8kjJPM93HNAoVWOSTdJd6" } ], "swift_keys": [ { "user": "testuser:swift", "secret_key": "a4ioT4jEP653CDcdU8p4OuhruwABBRZmyNUbnSSt" } ], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw" } subscription-manager repos --enable=rhel-9-for-x86_64-highavailability-rpms dnf install python3-boto3 vi s3test.py import boto3 endpoint = "" # enter the endpoint URL along with the port "http:// URL : PORT " access_key = ' ACCESS ' secret_key = ' SECRET ' s3 = boto3.client( 's3', endpoint_url=endpoint, aws_access_key_id=access_key, aws_secret_access_key=secret_key ) s3.create_bucket(Bucket='my-new-bucket') response = s3.list_buckets() for bucket in response['Buckets']: print("{name}\t{created}".format( name = bucket['Name'], created = bucket['CreationDate'] )) python3 s3test.py my-new-bucket 2022-05-31T17:09:10.000Z sudo yum install python-setuptools sudo easy_install pip sudo pip install --upgrade setuptools sudo pip install --upgrade python-swiftclient swift -A http:// IP_ADDRESS : PORT /auth/1.0 -U testuser:swift -K ' SWIFT_SECRET_KEY ' list swift -A http://10.10.143.116:80/auth/1.0 -U testuser:swift -K '244+fz2gSqoHwR3lYtSbIyomyPHf3i7rgSJrF/IA' list my-new-bucket