Developer Guide
Developer Guide Red Hat Ceph Storage 8 Using the various application programming interfaces for Red Hat Ceph Storage Red Hat Ceph Storage Documentation Team
Accept: application/vnd.ceph.api.v MAJOR . MINOR +json Accept: application/vnd.ceph.api.v1.0+json curl -X POST "https://example.com:8443/api/auth" -H "Accept: application/vnd.ceph.api.v1.0+json" -H "Content-Type: application/json" -d '{"username": user1, "password": password1}' curl -H "Authorization: Bearer TOKEN " root@host01 ~]# cephadm shell ceph mgr module enable dashboard ceph dashboard set-ssl-certificate HOST_NAME -i CERT_FILE ceph dashboard set-ssl-certificate-key HOST_NAME -i KEY_FILE ceph dashboard set-ssl-certificate -i dashboard.crt ceph dashboard set-ssl-certificate-key -i dashboard.key ceph dashboard set-ssl-certificate host01 -i dashboard.crt ceph dashboard set-ssl-certificate-key host01 -i dashboard.key ceph dashboard create-self-signed-cert echo -n " PASSWORD " > PATH_TO_FILE / PASSWORD_FILE ceph dashboard ac-user-create USER_NAME -i PASSWORD_FILE ROLE echo -n "p@ssw0rd" > /root/dash-password.txt ceph dashboard ac-user-create user1 -i /root/dash-password.txt administrator https:// HOST_NAME :8443 https://host01:8443 curl --silent --user USER 'https:// CEPH_MANAGER : CEPH_MANAGER_PORT /api/cluster_conf' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/cluster_conf' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/cluster_conf', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/cluster_conf', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/cluster_conf curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/cluster_conf/ ARGUMENT ' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/cluster_conf/ ARGUMENT ' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/cluster_conf/ ARGUMENT ', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/cluster_conf/ ARGUMENT ', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/cluster_conf/ ARGUMENT curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/osd/flags' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/osd/flags' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/osd/flags', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/osd/flags', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/osd/flags curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/crush_rule' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/crush_rule' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/crush_rule', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/crush_rule', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/crush_rule curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/monitor' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/monitor' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/monitor', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/monitor', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/monitor curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/monitor/ NAME ' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/monitor/ NAME ' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/monitor/ NAME ', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/monitor/ NAME ', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/monitor/ NAME curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/osd' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/osd' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/osd/', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/osd/', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/osd curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID ' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID ' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/osd/ ID ', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/osd/ ID ', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/osd/ ID curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID /command' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID /command' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/osd/ ID /command', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/osd/ ID /command', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/osd/ ID /command curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/pool' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/pool' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/pool', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/pool', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/pool curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/pool/ ID ' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/pool/ ID ' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/pool/ ID ', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/pool/ ID ', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/pool/ ID curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/host' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/host' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/host', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/host', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/host curl --silent --user USER 'https:// CEPH_MANAGER :8080/api/host/ HOST_NAME ' curl --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/host/ HOST_NAME ' python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/host/ HOST_NAME ', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.get('https:// CEPH_MANAGER :8080/api/host/ HOST_NAME ', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() https:// CEPH_MANAGER :8080/api/host/ HOST_NAME echo -En '{" OPTION ": VALUE }' | curl --request PATCH --data @- --silent --user USER 'https:// CEPH_MANAGER :8080/api/osd/flags' echo -En '{" OPTION ": VALUE }' | curl --request PATCH --data @- --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/osd/flags' python >> import requests >> result = requests.patch('https:// CEPH_MANAGER :8080/api/osd/flags', json={" OPTION ": VALUE }, auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.patch('https:// CEPH_MANAGER :8080/api/osd/flags', json={" OPTION ": VALUE }, auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() echo -En '{" STATE ": VALUE }' | curl --request PATCH --data @- --silent --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID ' echo -En '{" STATE ": VALUE }' | curl --request PATCH --data @- --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID ' python >> import requests >> result = requests.patch('https:// CEPH_MANAGER :8080/api/osd/ ID ', json={" STATE ": VALUE }, auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.patch('https:// CEPH_MANAGER :8080/api/osd/ ID ', json={" STATE ": VALUE }, auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() echo -En '{"reweight": VALUE }' | curl --request PATCH --data @- --silent --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID ' echo -En '{"reweight": VALUE }' | curl --request PATCH --data @- --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID ' python >> import requests >> result = requests.patch('https:// CEPH_MANAGER :8080/osd/ ID ', json={"reweight": VALUE }, auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.patch('https:// CEPH_MANAGER :8080/api/osd/ ID ', json={"reweight": VALUE }, auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() echo -En '{" OPTION ": VALUE }' | curl --request PATCH --data @- --silent --user USER 'https:// CEPH_MANAGER :8080/api/pool/ ID ' echo -En '{" OPTION ": VALUE }' | curl --request PATCH --data @- --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/pool/ ID ' python >> import requests >> result = requests.patch('https:// CEPH_MANAGER :8080/api/pool/ ID ', json={" OPTION ": VALUE }, auth=(" USER , " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.patch('https:// CEPH_MANAGER :8080/api/pool/ ID ', json={" OPTION ": VALUE }, auth=(" USER , " PASSWORD "), verify=False) >> print result.json() echo -En '{"command": " COMMAND "}' | curl --request POST --data @- --silent --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID /command' echo -En '{"command": " COMMAND "}' | curl --request POST --data @- --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/osd/ ID /command' python >> import requests >> result = requests.post('https:// CEPH_MANAGER :8080/api/osd/ ID /command', json={"command": " COMMAND "}, auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.post('https:// CEPH_MANAGER :8080/api/osd/ ID /command', json={"command": " COMMAND "}, auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() echo -En '{"name": " NAME ", "pg_num": NUMBER }' | curl --request POST --data @- --silent --user USER 'https:// CEPH_MANAGER :8080/api/pool' echo -En '{"name": " NAME ", "pg_num": NUMBER }' | curl --request POST --data @- --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/pool' python >> import requests >> result = requests.post('https:// CEPH_MANAGER :8080/api/pool', json={"name": " NAME ", "pg_num": NUMBER }, auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.post('https:// CEPH_MANAGER :8080/api/pool', json={"name": " NAME ", "pg_num": NUMBER }, auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() mon_allow_pool_delete = true curl --request DELETE --silent --user USER 'https:// CEPH_MANAGER :8080/api/pool/ ID ' curl --request DELETE --silent --insecure --user USER 'https:// CEPH_MANAGER :8080/api/pool/ ID ' python >> import requests >> result = requests.delete('https:// CEPH_MANAGER :8080/api/pool/ ID ', auth=(" USER ", " PASSWORD ")) >> print result.json() python >> import requests >> result = requests.delete('https:// CEPH_MANAGER :8080/api/pool/ ID ', auth=(" USER ", " PASSWORD "), verify=False) >> print result.json() PUT /admin/user?caps&format=json HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME Content-Type: text/plain Authorization: AUTHORIZATION_TOKEN usage=read Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET httpRequest.addHeader("Authorization", "AWS " + this.getAccessKey() + ":" + base64Sha1Hmac(headerString.toString(), this.getSecretKey())); import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.ZoneId; import org.apache.http.HttpEntity; import org.apache.http.NameValuePair; import org.apache.http.Header; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpDelete; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.apache.http.client.utils.URIBuilder; import java.util.Base64; import java.util.Base64.Encoder; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import javax.crypto.spec.SecretKeySpec; import javax.crypto.Mac; import java.util.Map; import java.util.Iterator; import java.util.Set; import java.util.Map.Entry; public class CephAdminAPI { /* * Each call must specify an access key, secret key, endpoint and format. */ String accessKey; String secretKey; String endpoint; String scheme = "http"; //http only. int port = 80; /* * A constructor that takes an access key, secret key, endpoint and format. */ public CephAdminAPI(String accessKey, String secretKey, String endpoint){ this.accessKey = accessKey; this.secretKey = secretKey; this.endpoint = endpoint; } /* * Accessor methods for access key, secret key, endpoint and format. */ public String getEndpoint(){ return this.endpoint; } public void setEndpoint(String endpoint){ this.endpoint = endpoint; } public String getAccessKey(){ return this.accessKey; } public void setAccessKey(String accessKey){ this.accessKey = accessKey; } public String getSecretKey(){ return this.secretKey; } public void setSecretKey(String secretKey){ this.secretKey = secretKey; } /* * Takes an HTTP Method, a resource and a map of arguments and * returns a CloseableHTTPResponse. */ public CloseableHttpResponse execute(String HTTPMethod, String resource, String subresource, Map arguments) { String httpMethod = HTTPMethod; String requestPath = resource; StringBuffer request = new StringBuffer(); StringBuffer headerString = new StringBuffer(); HttpRequestBase httpRequest; CloseableHttpClient httpclient; URI uri; CloseableHttpResponse httpResponse = null; try { uri = new URIBuilder() .setScheme(this.scheme) .setHost(this.getEndpoint()) .setPath(requestPath) .setPort(this.port) .build(); if (subresource != null){ uri = new URIBuilder(uri) .setCustomQuery(subresource) .build(); } for (Iterator iter = arguments.entrySet().iterator(); iter.hasNext();) { Entry entry = (Entry)iter.next(); uri = new URIBuilder(uri) .setParameter(entry.getKey().toString(), entry.getValue().toString()) .build(); } request.append(uri); headerString.append(HTTPMethod.toUpperCase().trim() + "\n\n\n"); OffsetDateTime dateTime = OffsetDateTime.now(ZoneId.of("GMT")); DateTimeFormatter formatter = DateTimeFormatter.RFC_1123_DATE_TIME; String date = dateTime.format(formatter); headerString.append(date + "\n"); headerString.append(requestPath); if (HTTPMethod.equalsIgnoreCase("PUT")){ httpRequest = new HttpPut(uri); } else if (HTTPMethod.equalsIgnoreCase("POST")){ httpRequest = new HttpPost(uri); } else if (HTTPMethod.equalsIgnoreCase("GET")){ httpRequest = new HttpGet(uri); } else if (HTTPMethod.equalsIgnoreCase("DELETE")){ httpRequest = new HttpDelete(uri); } else { System.err.println("The HTTP Method must be PUT, POST, GET or DELETE."); throw new IOException(); } httpRequest.addHeader("Date", date); httpRequest.addHeader("Authorization", "AWS " + this.getAccessKey() + ":" + base64Sha1Hmac(headerString.toString(), this.getSecretKey())); httpclient = HttpClients.createDefault(); httpResponse = httpclient.execute(httpRequest); } catch (URISyntaxException e){ System.err.println("The URI is not formatted properly."); e.printStackTrace(); } catch (IOException e){ System.err.println("There was an error making the request."); e.printStackTrace(); } return httpResponse; } /* * Takes a uri and a secret key and returns a base64-encoded * SHA-1 HMAC. */ public String base64Sha1Hmac(String uri, String secretKey) { try { byte[] keyBytes = secretKey.getBytes("UTF-8"); SecretKeySpec signingKey = new SecretKeySpec(keyBytes, "HmacSHA1"); Mac mac = Mac.getInstance("HmacSHA1"); mac.init(signingKey); byte[] rawHmac = mac.doFinal(uri.getBytes("UTF-8")); Encoder base64 = Base64.getEncoder(); return base64.encodeToString(rawHmac); } catch (Exception e) { throw new RuntimeException(e); } } } import java.io.IOException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.HttpEntity; import org.apache.http.util.EntityUtils; import java.util.*; public class CephAdminAPIClient { public static void main (String[] args){ CephAdminAPI adminApi = new CephAdminAPI ("FFC6ZQ6EMIF64194158N", "Xac39eCAhlTGcCAUreuwe1ZuH5oVQFa51lbEMVoT", "ceph-client"); /* * Create a user */ Map requestArgs = new HashMap(); requestArgs.put("access", "usage=read, write; users=read, write"); requestArgs.put("display-name", "New User"); requestArgs.put("email", "new-user@email.com"); requestArgs.put("format", "json"); requestArgs.put("uid", "new-user"); CloseableHttpResponse response = adminApi.execute("PUT", "/admin/user", null, requestArgs); System.out.println(response.getStatusLine()); HttpEntity entity = response.getEntity(); try { System.out.println("\nResponse Content is: " + EntityUtils.toString(entity, "UTF-8") + "\n"); response.close(); } catch (IOException e){ System.err.println ("Encountered an I/O exception."); e.printStackTrace(); } /* * Get a user */ requestArgs = new HashMap(); requestArgs.put("format", "json"); requestArgs.put("uid", "new-user"); response = adminApi.execute("GET", "/admin/user", null, requestArgs); System.out.println(response.getStatusLine()); entity = response.getEntity(); try { System.out.println("\nResponse Content is: " + EntityUtils.toString(entity, "UTF-8") + "\n"); response.close(); } catch (IOException e){ System.err.println ("Encountered an I/O exception."); e.printStackTrace(); } /* * Modify a user */ requestArgs = new HashMap(); requestArgs.put("display-name", "John Doe"); requestArgs.put("email", "johndoe@email.com"); requestArgs.put("format", "json"); requestArgs.put("uid", "new-user"); requestArgs.put("max-buckets", "100"); response = adminApi.execute("POST", "/admin/user", null, requestArgs); System.out.println(response.getStatusLine()); entity = response.getEntity(); try { System.out.println("\nResponse Content is: " + EntityUtils.toString(entity, "UTF-8") + "\n"); response.close(); } catch (IOException e){ System.err.println ("Encountered an I/O exception."); e.printStackTrace(); } /* * Create a subuser */ requestArgs = new HashMap(); requestArgs.put("format", "json"); requestArgs.put("uid", "new-user"); requestArgs.put("subuser", "foobar"); response = adminApi.execute("PUT", "/admin/user", "subuser", requestArgs); System.out.println(response.getStatusLine()); entity = response.getEntity(); try { System.out.println("\nResponse Content is: " + EntityUtils.toString(entity, "UTF-8") + "\n"); response.close(); } catch (IOException e){ System.err.println ("Encountered an I/O exception."); e.printStackTrace(); } /* * Delete a user */ requestArgs = new HashMap(); requestArgs.put("format", "json"); requestArgs.put("uid", "new-user"); response = adminApi.execute("DELETE", "/admin/user", null, requestArgs); System.out.println(response.getStatusLine()); entity = response.getEntity(); try { System.out.println("\nResponse Content is: " + EntityUtils.toString(entity, "UTF-8") + "\n"); response.close(); } catch (IOException e){ System.err.println ("Encountered an I/O exception."); e.printStackTrace(); } } } radosgw-admin user create --uid=" USER_NAME " --display-name=" DISPLAY_NAME " [user@client ~]USD radosgw-admin user create --uid="admin-api-user" --display-name="Admin API User" { "user_id": "admin-api-user", "display_name": "Admin API User", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "admin-api-user", "access_key": "NRWGT19TWMYOB1YDBV1Y", "secret_key": "gr1VEGIV7rxcP3xvXDFCo4UDwwl2YoNrmtRlIAty" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "temp_url_keys": [] } radosgw-admin caps add --uid=" USER_NAME " --caps="users=*" [user@client ~]USD radosgw-admin caps add --uid=admin-api-user --caps="users=*" { "user_id": "admin-api-user", "display_name": "Admin API User", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "admin-api-user", "access_key": "NRWGT19TWMYOB1YDBV1Y", "secret_key": "gr1VEGIV7rxcP3xvXDFCo4UDwwl2YoNrmtRlIAty" } ], "swift_keys": [], "caps": [ { "type": "users", "perm": "*" } ], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "temp_url_keys": [] } users=read or user-info-without-keys=read GET /admin/user?format=json HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME `users=write` PUT /admin/user?format=json HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME `users=write` POST /admin/user?format=json HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME `users=write` DELETE /admin/user?format=json HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME `users=write` PUT /admin/user?subuser&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `users=write` POST /admin/user?subuser&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `users=write` DELETE /admin/user?subuser&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `users=write` PUT /admin/user?caps&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `users=write` DELETE /admin/user?caps&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `users=write` PUT /admin/user?key&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `users=write` DELETE /admin/user?key&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME POST Action=CreateTopic &Name= TOPIC_NAME [&Attributes.entry.1.key=amqp-exchange&Attributes.entry.1.value= EXCHANGE ] [&Attributes.entry.2.key=amqp-ack-level&Attributes.entry.2.value=none|broker|routable] [&Attributes.entry.3.key=verify-ssl&Attributes.entry.3.value=true|false] [&Attributes.entry.4.key=kafka-ack-level&Attributes.entry.4.value=none|broker] [&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=true|false] [&Attributes.entry.6.key=ca-location&Attributes.entry.6.value= FILE_PATH ] [&Attributes.entry.7.key=OpaqueData&Attributes.entry.7.value= OPAQUE_DATA ] [&Attributes.entry.8.key=push-endpoint&Attributes.entry.8.value= ENDPOINT ] [&Attributes.entry.9.key=persistent&Attributes.entry.9.value=true|false] <CreateTopicResponse xmlns="https://sns.amazonaws.com/doc/2010-03-31/"> <CreateTopicResult> <TopicArn></TopicArn> </CreateTopicResult> <ResponseMetadata> <RequestId></RequestId> </ResponseMetadata> </CreateTopicResponse> client.create_topic(Name='my-topic' , Attributes={'push-endpoint': 'amqp://127.0.0.1:5672', 'amqp-exchange': 'ex1', 'amqp-ack-level': 'broker'}) " POST Action=GetTopic &TopicArn= TOPIC_ARN <GetTopicResponse> <GetTopicRersult> <Topic> <User></User> <Name></Name> <EndPoint> <EndpointAddress></EndpointAddress> <EndpointArgs></EndpointArgs> <EndpointTopic></EndpointTopic> <HasStoredSecret></HasStoredSecret> <Persistent></Persistent> </EndPoint> <TopicArn></TopicArn> <OpaqueData></OpaqueData> </Topic> </GetTopicResult> <ResponseMetadata> <RequestId></RequestId> </ResponseMetadata> </GetTopicResponse> POST Action=ListTopics <ListTopicdResponse xmlns="https://sns.amazonaws.com/doc/2020-03-31/"> <ListTopicsRersult> <Topics> <member> <User></User> <Name></Name> <EndPoint> <EndpointAddress></EndpointAddress> <EndpointArgs></EndpointArgs> <EndpointTopic></EndpointTopic> </EndPoint> <TopicArn></TopicArn> <OpaqueData></OpaqueData> </member> </Topics> </ListTopicsResult> <ResponseMetadata> <RequestId></RequestId> </ResponseMetadata> </ListTopicsResponse> POST Action=DeleteTopic &TopicArn= TOPIC_ARN <DeleteTopicResponse xmlns="https://sns.amazonaws.com/doc/2020-03-31/"> <ResponseMetadata> <RequestId></RequestId> </ResponseMetadata> </DeleteTopicResponse> radosgw-admin topic list --uid= USER_ID radosgw-admin topic list --uid=example radosgw-admin topic get --uid= USER_ID --topic= TOPIC_NAME radosgw-admin topic get --uid=example --topic=example-topic radosgw-admin topic rm --uid= USER_ID --topic= TOPIC_NAME radosgw-admin topic rm --uid=example --topic=example-topic radosgw-admin notification list --bucket= BUCKET_NAME radosgw-admin notification list --bucket bkt2 { "notifications": [ { "TopicArn": "arn:aws:sns:default::topic1", "Id": "notif1", "Events": [ "s3:ObjectCreated:*", "s3:ObjectRemoved:*" ], "Filter": { "S3Key": {}, "S3Metadata": {}, "S3Tags": {} } }, { "TopicArn": "arn:aws:sns:default::topic1", "Id": "notif2", "Events": [ "s3:ObjectSynced:*" ], "Filter": { "S3Key": {}, "S3Metadata": {}, "S3Tags": {} } } ] } radosgw-admin notification get --bucket BUCKET_NAME --notification-id NOTIFICATION_ID radosgw-admin notification get --bucket bkt2 --notification-id notif2 { "TopicArn": "arn:aws:sns:default::topic1", "Id": "notif2", "Events": [ "s3:ObjectSynced:*" ], "Filter": { "S3Key": {}, "S3Metadata": {}, "S3Tags": {} } } radosgw-admin notification rm --bucket BUCKET_NAME [--notification-id NOTIFICATION_ID ] radosgw-admin notification rm --bucket bkt2 --notification-id notif1 {"Records":[ { "eventVersion":"2.1", "eventSource":"ceph:s3", "awsRegion":"us-east-1", "eventTime":"2019-11-22T13:47:35.124724Z", "eventName":"ObjectCreated:Put", "userIdentity":{ "principalId":"tester" }, "requestParameters":{ "sourceIPAddress":"" }, "responseElements":{ "x-amz-request-id":"503a4c37-85eb-47cd-8681-2817e80b4281.5330.903595", "x-amz-id-2":"14d2-zone1-zonegroup1" }, "s3":{ "s3SchemaVersion":"1.0", "configurationId":"mynotif1", "bucket":{ "name":"mybucket1", "ownerIdentity":{ "principalId":"tester" }, "arn":"arn:aws:s3:us-east-1::mybucket1", "id":"503a4c37-85eb-47cd-8681-2817e80b4281.5332.38" }, "object":{ "key":"myimage1.jpg", "size":"1024", "eTag":"37b51d194a7513e45b56f6524f2d51f2", "versionId":"", "sequencer": "F7E6D75DC742D108", "metadata":[], "tags":[] } }, "eventId":"", "opaqueData":"me@example.com" } ]} `buckets=read` GET /admin/bucket?format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME GET /admin/bucket?index&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `buckets=write` DELETE /admin/bucket?format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `buckets=write` PUT /admin/bucket?format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `buckets=write` POST /admin/bucket?format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `buckets=read` GET /admin/bucket?policy&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME `buckets=write` DELETE /admin/bucket?object&format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME GET /admin/user?quota&uid= UID &quota-type=user PUT /admin/user?quota&uid= UID &quota-type=user `buckets=read` GET /admin/bucket?format=json HTTP/1.1 Host FULLY_QUALIFIED_DOMAIN_NAME PUT /admin/user?quota&uid= UID &quota-type=bucket `usage=read` GET /admin/usage?format=json HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME `usage=write` DELETE /admin/usage?format=json HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME HTTP/1.1 PUT /buckets/bucket/object.mpeg Host: cname.domain.com Date: Mon, 2 Jan 2012 00:01:01 +0000 Content-Encoding: mpeg Content-Length: 9999999 Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET firewall-cmd --zone=public --add-port=8080/tcp --permanent firewall-cmd --reload yum install dnsmasq echo "address=/. FQDN_OF_GATEWAY_NODE / IP_OF_GATEWAY_NODE " | tee --append /etc/dnsmasq.conf systemctl start dnsmasq systemctl enable dnsmasq systemctl stop NetworkManager systemctl disable NetworkManager echo "DNS1= IP_OF_GATEWAY_NODE " | tee --append /etc/sysconfig/network-scripts/ifcfg-eth0 echo " IP_OF_GATEWAY_NODE FQDN_OF_GATEWAY_NODE " | tee --append /etc/hosts systemctl restart network systemctl enable network systemctl restart dnsmasq [user@rgw ~]USD ping mybucket. FQDN_OF_GATEWAY_NODE yum install ruby gem install aws-s3 [user@dev ~]USD mkdir ruby_aws_s3 [user@dev ~]USD cd ruby_aws_s3 [user@dev ~]USD vim conn.rb #!/usr/bin/env ruby require 'aws/s3' require 'resolv-replace' AWS::S3::Base.establish_connection!( :server => ' FQDN_OF_GATEWAY_NODE ', :port => '8080', :access_key_id => ' MY_ACCESS_KEY ', :secret_access_key => ' MY_SECRET_KEY ' ) #!/usr/bin/env ruby require 'aws/s3' require 'resolv-replace' AWS::S3::Base.establish_connection!( :server => 'testclient.englab.pnq.redhat.com', :port => '8080', :access_key_id => '98J4R9P22P5CDL65HKP8', :secret_access_key => '6C+jcaP0dp0+FZfrRNgyGA9EzRy25pURldwje049' ) [user@dev ~]USD chmod +x conn.rb [user@dev ~]USD ./conn.rb | echo USD? [user@dev ~]USD vim create_bucket.rb #!/usr/bin/env ruby load 'conn.rb' AWS::S3::Bucket.create('my-new-bucket1') [user@dev ~]USD chmod +x create_bucket.rb [user@dev ~]USD ./create_bucket.rb [user@dev ~]USD vim list_owned_buckets.rb #!/usr/bin/env ruby load 'conn.rb' AWS::S3::Service.buckets.each do |bucket| puts "{bucket.name}\t{bucket.creation_date}" end [user@dev ~]USD chmod +x list_owned_buckets.rb [user@dev ~]USD ./list_owned_buckets.rb my-new-bucket1 2020-01-21 10:33:19 UTC [user@dev ~]USD vim create_object.rb #!/usr/bin/env ruby load 'conn.rb' AWS::S3::S3Object.store( 'hello.txt', 'Hello World!', 'my-new-bucket1', :content_type => 'text/plain' ) [user@dev ~]USD chmod +x create_object.rb [user@dev ~]USD ./create_object.rb [user@dev ~]USD vim list_bucket_content.rb #!/usr/bin/env ruby load 'conn.rb' new_bucket = AWS::S3::Bucket.find('my-new-bucket1') new_bucket.each do |object| puts "{object.key}\t{object.about['content-length']}\t{object.about['last-modified']}" end [user@dev ~]USD chmod +x list_bucket_content.rb [user@dev ~]USD ./list_bucket_content.rb hello.txt 12 Fri, 22 Jan 2020 15:54:52 GMT [user@dev ~]USD vim del_empty_bucket.rb #!/usr/bin/env ruby load 'conn.rb' AWS::S3::Bucket.delete('my-new-bucket1') [user@dev ~]USD chmod +x del_empty_bucket.rb [user@dev ~]USD ./del_empty_bucket.rb | echo USD? [user@dev ~]USD vim del_non_empty_bucket.rb #!/usr/bin/env ruby load 'conn.rb' AWS::S3::Bucket.delete('my-new-bucket1', :force => true) [user@dev ~]USD chmod +x del_non_empty_bucket.rb [user@dev ~]USD ./del_non_empty_bucket.rb | echo USD? [user@dev ~]USD vim delete_object.rb #!/usr/bin/env ruby load 'conn.rb' AWS::S3::S3Object.delete('hello.txt', 'my-new-bucket1') [user@dev ~]USD chmod +x delete_object.rb [user@dev ~]USD ./delete_object.rb yum install ruby gem install aws-sdk [user@dev ~]USD mkdir ruby_aws_sdk [user@dev ~]USD cd ruby_aws_sdk [user@dev ~]USD vim conn.rb #!/usr/bin/env ruby require 'aws-sdk' require 'resolv-replace' Aws.config.update( endpoint: 'http:// FQDN_OF_GATEWAY_NODE :8080', access_key_id: ' MY_ACCESS_KEY ', secret_access_key: ' MY_SECRET_KEY ', force_path_style: true, region: 'us-east-1' ) #!/usr/bin/env ruby require 'aws-sdk' require 'resolv-replace' Aws.config.update( endpoint: 'http://testclient.englab.pnq.redhat.com:8080', access_key_id: '98J4R9P22P5CDL65HKP8', secret_access_key: '6C+jcaP0dp0+FZfrRNgyGA9EzRy25pURldwje049', force_path_style: true, region: 'us-east-1' ) [user@dev ~]USD chmod +x conn.rb [user@dev ~]USD ./conn.rb | echo USD? [user@dev ~]USD vim create_bucket.rb #!/usr/bin/env ruby load 'conn.rb' s3_client = Aws::S3::Client.new s3_client.create_bucket(bucket: 'my-new-bucket2') [user@dev ~]USD chmod +x create_bucket.rb [user@dev ~]USD ./create_bucket.rb [user@dev ~]USD vim list_owned_buckets.rb #!/usr/bin/env ruby load 'conn.rb' s3_client = Aws::S3::Client.new s3_client.list_buckets.buckets.each do |bucket| puts "{bucket.name}\t{bucket.creation_date}" end [user@dev ~]USD chmod +x list_owned_buckets.rb [user@dev ~]USD ./list_owned_buckets.rb my-new-bucket2 2020-01-21 10:33:19 UTC [user@dev ~]USD vim create_object.rb #!/usr/bin/env ruby load 'conn.rb' s3_client = Aws::S3::Client.new s3_client.put_object( key: 'hello.txt', body: 'Hello World!', bucket: 'my-new-bucket2', content_type: 'text/plain' ) [user@dev ~]USD chmod +x create_object.rb [user@dev ~]USD ./create_object.rb [user@dev ~]USD vim list_bucket_content.rb #!/usr/bin/env ruby load 'conn.rb' s3_client = Aws::S3::Client.new s3_client.list_objects(bucket: 'my-new-bucket2').contents.each do |object| puts "{object.key}\t{object.size}" end [user@dev ~]USD chmod +x list_bucket_content.rb [user@dev ~]USD ./list_bucket_content.rb hello.txt 12 Fri, 22 Jan 2020 15:54:52 GMT [user@dev ~]USD vim del_empty_bucket.rb #!/usr/bin/env ruby load 'conn.rb' s3_client = Aws::S3::Client.new s3_client.delete_bucket(bucket: 'my-new-bucket2') [user@dev ~]USD chmod +x del_empty_bucket.rb [user@dev ~]USD ./del_empty_bucket.rb | echo USD? [user@dev ~]USD vim del_non_empty_bucket.rb #!/usr/bin/env ruby load 'conn.rb' s3_client = Aws::S3::Client.new Aws::S3::Bucket.new('my-new-bucket2', client: s3_client).clear! s3_client.delete_bucket(bucket: 'my-new-bucket2') [user@dev ~]USD chmod +x del_non_empty_bucket.rb [user@dev ~]USD ./del_non_empty_bucket.rb | echo USD? [user@dev ~]USD vim delete_object.rb #!/usr/bin/env ruby load 'conn.rb' s3_client = Aws::S3::Client.new s3_client.delete_object(key: 'hello.txt', bucket: 'my-new-bucket2') [user@dev ~]USD chmod +x delete_object.rb [user@dev ~]USD ./delete_object.rb yum install php [user@dev ~]USD mkdir php_s3 [user@dev ~]USD cd php_s3 [user@dev ~]USD cp -r ~/Downloads/aws/ ~/php_s3/ [user@dev ~]USD vim conn.php <?php define('AWS_KEY', ' MY_ACCESS_KEY '); define('AWS_SECRET_KEY', ' MY_SECRET_KEY '); define('HOST', ' FQDN_OF_GATEWAY_NODE '); define('PORT', '8080'); // require the AWS SDK for php library require '/ PATH_TO_AWS /aws-autoloader.php'; use Aws\S3\S3Client; // Establish connection with host using S3 Client client = S3Client::factory(array( 'base_url' => HOST , 'port' => PORT , 'key' => AWS_KEY , 'secret' => AWS_SECRET_KEY )); ?> [user@dev ~]USD php -f conn.php | echo USD? [user@dev ~]USD vim create_bucket.php <?php include 'conn.php'; client->createBucket(array('Bucket' => 'my-new-bucket3')); ?> [user@dev ~]USD php -f create_bucket.php [user@dev ~]USD vim list_owned_buckets.php <?php include 'conn.php'; blist = client->listBuckets(); echo "Buckets belonging to " . blist['Owner']['ID'] . ":\n"; foreach (blist['Buckets'] as b) { echo "{b['Name']}\t{b['CreationDate']}\n"; } ?> [user@dev ~]USD php -f list_owned_buckets.php my-new-bucket3 2020-01-21 10:33:19 UTC [user@dev ~]USD echo "Hello World!" > hello.txt [user@dev ~]USD vim create_object.php <?php include 'conn.php'; key = 'hello.txt'; source_file = './hello.txt'; acl = 'private'; bucket = 'my-new-bucket3'; client->upload(bucket, key, fopen(source_file, 'r'), acl); ?> [user@dev ~]USD php -f create_object.php [user@dev ~]USD vim list_bucket_content.php <?php include 'conn.php'; o_iter = client->getIterator('ListObjects', array( 'Bucket' => 'my-new-bucket3' )); foreach (o_iter as o) { echo "{o['Key']}\t{o['Size']}\t{o['LastModified']}\n"; } ?> [user@dev ~]USD php -f list_bucket_content.php hello.txt 12 Fri, 22 Jan 2020 15:54:52 GMT [user@dev ~]USD vim del_empty_bucket.php <?php include 'conn.php'; client->deleteBucket(array('Bucket' => 'my-new-bucket3')); ?> [user@dev ~]USD php -f del_empty_bucket.php | echo USD? [user@dev ~]USD vim delete_object.php <?php include 'conn.php'; client->deleteObject(array( 'Bucket' => 'my-new-bucket3', 'Key' => 'hello.txt', )); ?> [user@dev ~]USD php -f delete_object.php ceph config set RGW_CLIENT_NAME rgw_sts_key STS_KEY ceph config set RGW_CLIENT_NAME rgw_s3_auth_use_sts true ceph config set client.rgw rgw_sts_key 7f8fd8dd4700mnop ceph config set client.rgw rgw_s3_auth_use_sts true systemctl restart ceph- CLUSTER_ID @ SERVICE_TYPE . ID .service systemctl restart ceph-c4b34c6f-8365-11ba-dc31-529020a7702d@rgw.realm.zone.host01.gwasto.service ceph orch restart SERVICE_TYPE ceph orch restart rgw radosgw-admin --uid USER_NAME --display-name " DISPLAY_NAME " --access_key USER_NAME --secret SECRET user create [user@rgw ~]USD radosgw-admin --uid TESTER --display-name "TestUser" --access_key TESTER --secret test123 user create radosgw-admin caps add --uid=" USER_NAME " --caps="oidc-provider=*" [user@rgw ~]USD radosgw-admin caps add --uid="TESTER" --caps="oidc-provider=*" "{\"Version\":\"2020-01-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/ IDP_URL \"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\" IDP_URL :app_id\":\" AUD_FIELD \"\}\}\}\]\}" curl -k -v -X GET -H "Content-Type: application/x-www-form-urlencoded" " IDP_URL :8000/ CONTEXT /realms/ REALM /.well-known/openid-configuration" | jq . [user@client ~]USD curl -k -v -X GET -H "Content-Type: application/x-www-form-urlencoded" "http://www.example.com:8000/auth/realms/quickstart/.well-known/openid-configuration" | jq . curl -k -v -X GET -H "Content-Type: application/x-www-form-urlencoded" " IDP_URL / CONTEXT /realms/ REALM /protocol/openid-connect/certs" | jq . [user@client ~]USD curl -k -v -X GET -H "Content-Type: application/x-www-form-urlencoded" "http://www.example.com/auth/realms/quickstart/protocol/openid-connect/certs" | jq . -----BEGIN CERTIFICATE----- MIIDYjCCAkqgAwIBAgIEEEd2CDANBgkqhkiG9w0BAQsFADBzMQkwBwYDVQQGEwAxCTAHBgNVBAgTADEJMAcGA1UEBxMAMQkwBwYDVQQKEwAxCTAHBgNVBAsTADE6MDgGA1UEAxMxYXV0aHN2Yy1pbmxpbmVtZmEuZGV2LnZlcmlmeS5pYm1jbG91ZHNlY3VyaXR5LmNvbTAeFw0yMTA3MDUxMzU2MzZaFw0zMTA3MDMxMzU2MzZaMHMxCTAHBgNVBAYTADEJMAcGA1UECBMAMQkwBwYDVQQHEwAxCTAHBgNVBAoTADEJMAcGA1UECxMAMTowOAYDVQQDEzFhdXRoc3ZjLWlubGluZW1mYS5kZXYudmVyaWZ5LmlibWNsb3Vkc2VjdXJpdHkuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAphyu3HaAZ14JH/EXetZxtNnerNuqcnfxcmLhBz9SsTlFD59ta+BOVlRnK5SdYEqO3ws2iGEzTvC55rczF+hDVHFZEBJLVLQe8ABmi22RAtG1P0dA/Bq8ReFxpOFVWJUBc31QM+ummW0T4yw44wQJI51LZTMz7PznB0ScpObxKe+frFKd1TCMXPlWOSzmTeFYKzR83Fg9hsnz7Y8SKGxi+RoBbTLT+ektfWpR7O+oWZIf4INe1VYJRxZvn+qWcwI5uMRCtQkiMknc3Rj6Eupiqq6FlAjDs0p//EzsHAlW244jMYnHCGq0UP3oE7vViLJyiOmZw7J3rvs3m9mOQiPLoQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCeVqAzSh7Tp8LgaTIFUuRbdjBAKXC9Nw3+pRBHoiUTdhqO3ualyGih9m/js/clb8Vq/39zl0VPeaslWl2NNX9zaK7xo+ckVIOY3ucCaTC04ZUn1KzZu/7azlN0C5XSWg/CfXgU2P3BeMNzc1UNY1BASGyWn2lEplIVWKLaDZpNdSyyGyaoQAIBdzxeNCyzDfPCa2oSO8WH1czmFiNPqR5kdknHI96CmsQdi+DT4jwzVsYgrLfcHXmiWyIAb883hR3Pobp+Bsw7LUnxebQ5ewccjYmrJzOk5Wb5FpXBhaJH1B3AEd6RGalRUyc/zUKdvEy0nIRMDS9x2BP3NVvZSADD -----END CERTIFICATE----- openssl x509 -in CERT_FILE -fingerprint -noout [user@client ~]USD openssl x509 -in certificate.crt -fingerprint -noout SHA1 Fingerprint=F7:D7:B3:51:5D:D0:D3:19:DD:21:9A:43:A9:EA:72:7A:D6:06:52:87 bash check_token_isv.sh | jq .iss "https://keycloak-sso.apps.ocp.example.com/auth/realms/ceph" aws --endpoint https://cephproxy1.example.com:8443 iam create-open-id-connect-provider --url https://keycloak-sso.apps.ocp.example.com/auth/realms/ceph --thumbprint-list 00E9CFD697E0B16DD13C86B0FFDC29957E5D24DF aws --endpoint https://cephproxy1.example.com:8443 iam list-open-id-connect-providers { "OpenIDConnectProviderList": [ { "Arn": "arn:aws:iam:::oidc-provider/keycloak-sso.apps.ocp.example.com/auth/realms/ceph" } ] } curl -k -q -L -X POST "https://keycloak-sso.apps.example.com/auth/realms/ceph/protocol/openid-connect/ token" -H 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'client_id=ceph' --data-urlencode 'grant_type=password' --data-urlencode 'client_secret=XXXXXXXXXXXXXXXXXXXXXXX' --data-urlencode 'scope=openid' --data-urlencode "username=SSOUSERNAME" --data-urlencode "password=SSOPASSWORD" cat check_token.sh USERNAME=USD1 PASSWORD=USD2 KC_CLIENT="ceph" KC_CLIENT_SECRET="7sQXqyMSzHIeMcSALoKaljB6sNIBDRjU" KC_ACCESS_TOKEN="USD(./get_web_token.sh USDUSERNAME USDPASSWORD | jq -r '.access_token')" KC_SERVER="https://keycloak-sso.apps.ocp.stg.local" KC_CONTEXT="auth" KC_REALM="ceph" curl -k -s -q -X POST -u "USDKC_CLIENT:USDKC_CLIENT_SECRET" -d "token=USDKC_ACCESS_TOKEN" "USDKC_SERVER/USDKC_CONTEXT/realms/USDKC_REALM/protocol/openid-connect/token/introspect" | jq . ./check_token.sh s3admin passw0rd | jq .sub "ceph" cat role-rgwadmins.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Federated": [ "arn:aws:iam:::oidc-provider/keycloak-sso.apps.example.com/auth/realms/ceph" ] }, "Action": [ "sts:AssumeRoleWithWebIdentity" ], "Condition": { "StringLike": { "keycloak-sso.apps.example.com/auth/realms/ceph:sub":"ceph" } } } ] } radosgw-admin role create --role-name rgwadmins --assume-role-policy-doc=USD(jq -rc . /root/role-rgwadmins.json) cat test-assume-role.sh #!/bin/bash export AWS_CA_BUNDLE="/etc/pki/ca-trust/source/anchors/cert.pem" unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY unset AWS_SESSION_TOKEN KC_ACCESS_TOKEN=USD(curl -k -q -L -X POST "https://keycloak-sso.apps.ocp.example.com/auth/realms/ceph/protocol/openid-connect/ token" -H 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'client_id=ceph' --data-urlencode 'grant_type=password' --data-urlencode 'client_secret=XXXXXXXXXXXXXXXXXXXXXXX' --data-urlencode 'scope=openid' --data-urlencode "<varname>SSOUSERNAME</varname>" --data-urlencode "<varname>SSOPASSWORD</varname>" | jq -r .access_token) echo USD{KC_ACCESS_TOKEN} IDM_ASSUME_ROLE_CREDS=USD(aws sts assume-role-with-web-identity --role-arn "arn:aws:iam:::role/USD3" --role-session-name testbr --endpoint=https://cephproxy1.example.com:8443 --web-identity-token="USDKC_ACCESS_TOKEN") echo "aws sts assume-role-with-web-identity --role-arn "arn:aws:iam:::role/USD3" --role-session-name testb --endpoint=https://cephproxy1.example.com:8443 --web-identity-token="USDKC_ACCESS_TOKEN"" echo USDIDM_ASSUME_ROLE_CREDS export AWS_ACCESS_KEY_ID=USD(echo USDIDM_ASSUME_ROLE_CREDS | jq -r .Credentials.AccessKeyId) export AWS_SECRET_ACCESS_KEY=USD(echo USDIDM_ASSUME_ROLE_CREDS | jq -r .Credentials.SecretAccessKey) export AWS_SESSION_TOKEN=USD(echo USDIDM_ASSUME_ROLE_CREDS | jq -r .Credentials.SessionToken) source ./test-assume-role.sh s3admin passw0rd rgwadmins aws s3 mb s3://testbucket aws s3 ls ceph config set RGW_CLIENT_NAME rgw_sts_key STS_KEY ceph config set RGW_CLIENT_NAME rgw_s3_auth_use_sts true ceph config set client.rgw rgw_sts_key 7f8fd8dd4700mnop ceph config set client.rgw rgw_s3_auth_use_sts true [user@osp ~]USD openstack ec2 credentials create +------------+--------------------------------------------------------+ | Field | Value | +------------+--------------------------------------------------------+ | access | b924dfc87d454d15896691182fdeb0ef | | links | {u'self': u'http://192.168.0.15/identity/v3/users/ | | | 40a7140e424f493d8165abc652dc731c/credentials/ | | | OS-EC2/b924dfc87d454d15896691182fdeb0ef'} | | project_id | c703801dccaf4a0aaa39bec8c481e25a | | secret | 6a2142613c504c42a94ba2b82147dc28 | | trust_id | None | | user_id | 40a7140e424f493d8165abc652dc731c | +------------+--------------------------------------------------------+ import boto3 access_key = b924dfc87d454d15896691182fdeb0ef secret_key = 6a2142613c504c42a94ba2b82147dc28 client = boto3.client('sts', aws_access_key_id=access_key, aws_secret_access_key=secret_key, endpoint_url=https://www.example.com/rgw, region_name='', ) response = client.get_session_token( DurationSeconds=43200 ) s3client = boto3.client('s3', aws_access_key_id = response['Credentials']['AccessKeyId'], aws_secret_access_key = response['Credentials']['SecretAccessKey'], aws_session_token = response['Credentials']['SessionToken'], endpoint_url=https://www.example.com/s3, region_name='') bucket = s3client.create_bucket(Bucket='my-new-shiny-bucket') response = s3client.list_buckets() for bucket in response["Buckets"]: print "{name}\t{created}".format( name = bucket['Name'], created = bucket['CreationDate'], ) radosgw-admin caps add --uid=" USER " --caps="roles=*" radosgw-admin caps add --uid="gwadmin" --caps="roles=*" radosgw-admin role create --role-name= ROLE_NAME --path= PATH --assume-role-policy-doc= TRUST_POLICY_DOC radosgw-admin role create --role-name=S3Access --path=/application_abc/component_xyz/ --assume-role-policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\} radosgw-admin role-policy put --role-name= ROLE_NAME --policy-name= POLICY_NAME --policy-doc= PERMISSION_POLICY_DOC radosgw-admin role-policy put --role-name=S3Access --policy-name=Policy --policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Action\":\[\"s3:*\"\],\"Resource\":\"arn:aws:s3:::example_bucket\"\}\]\} radosgw-admin user info --uid=gwuser | grep -A1 access_key import boto3 access_key = 11BS02LGFB6AL6H1ADMW secret_key = vzCEkuryfn060dfee4fgQPqFrncKEIkh3ZcdOANY client = boto3.client('sts', aws_access_key_id=access_key, aws_secret_access_key=secret_key, endpoint_url=https://www.example.com/rgw, region_name='', ) response = client.assume_role( RoleArn='arn:aws:iam:::role/application_abc/component_xyz/S3Access', RoleSessionName='Bob', DurationSeconds=3600 ) class SigV4Auth(BaseSigner): """ Sign a request with Signature V4. """ REQUIRES_REGION = True def __init__(self, credentials, service_name, region_name): self.credentials = credentials # We initialize these value here so the unit tests can have # valid values. But these will get overriden in ``add_auth`` # later for real requests. self._region_name = region_name if service_name == 'sts': 1 self._service_name = 's3' 2 else: 3 self._service_name = service_name 4 def _modify_request_before_signing(self, request): if 'Authorization' in request.headers: del request.headers['Authorization'] self._set_necessary_date_headers(request) if self.credentials.token: if 'X-Amz-Security-Token' in request.headers: del request.headers['X-Amz-Security-Token'] request.headers['X-Amz-Security-Token'] = self.credentials.token if not request.context.get('payload_signing_enabled', True): if 'X-Amz-Content-SHA256' in request.headers: del request.headers['X-Amz-Content-SHA256'] request.headers['X-Amz-Content-SHA256'] = UNSIGNED_PAYLOAD 1 else: 2 request.headers['X-Amz-Content-SHA256'] = self.payload(request) client.put_bucket_notification_configuration( Bucket=bucket_name, NotificationConfiguration={ 'TopicConfigurations': [ { 'Id': notification_name, 'TopicArn': topic_arn, 'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*', 's3:ObjectLifecycle:Expiration:*'] }]}) Get / BUCKET ?notification= NOTIFICATION_ID HTTP/1.1 Host: cname.domain.com Date: date Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET Get /testbucket?notification=testnotificationID HTTP/1.1 Host: cname.domain.com Date: date Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET <NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TopicConfiguration> <Id></Id> <Topic></Topic> <Event></Event> <Filter> <S3Key> <FilterRule> <Name></Name> <Value></Value> </FilterRule> </S3Key> <S3Metadata> <FilterRule> <Name></Name> <Value></Value> </FilterRule> </S3Metadata> <S3Tags> <FilterRule> <Name></Name> <Value></Value> </FilterRule> </S3Tags> </Filter> </TopicConfiguration> </NotificationConfiguration> DELETE / BUCKET ?notification= NOTIFICATION_ID HTTP/1.1 DELETE /testbucket?notification=testnotificationID HTTP/1.1 GET /mybucket HTTP/1.1 Host: cname.domain.com GET / HTTP/1.1 Host: mybucket.cname.domain.com GET / HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET GET / BUCKET ?max-keys=25 HTTP/1.1 Host: cname.domain.com PUT / BUCKET HTTP/1.1 Host: cname.domain.com x-amz-acl: public-read-write Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET PUT / BUCKET ?website-configuration=HTTP/1.1 PUT /testbucket?website-configuration=HTTP/1.1 GET / BUCKET ?website-configuration=HTTP/1.1 GET /testbucket?website-configuration=HTTP/1.1 DELETE / BUCKET ?website-configuration=HTTP/1.1 DELETE /testbucket?website-configuration=HTTP/1.1 PUT / BUCKET ?replication HTTP/1.1 PUT /testbucket?replication HTTP/1.1 GET / BUCKET ?replication HTTP/1.1 GET /testbucket?replication HTTP/1.1 DELETE / BUCKET ?replication HTTP/1.1 DELETE /testbucket?replication HTTP/1.1 DELETE / BUCKET HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET <LifecycleConfiguration> <Rule> <Prefix/> <Status>Enabled</Status> <Expiration> <Days>10</Days> </Expiration> </Rule> </LifecycleConfiguration> <LifecycleConfiguration> <Rule> <Status>Enabled</Status> <Filter> <Prefix>keypre/</Prefix> </Filter> </Rule> </LifecycleConfiguration> <LifecycleConfiguration> <Rule> <Status>Enabled</Status> <Filter> <Prefix>keypre/</Prefix> </Filter> </Rule> <Rule> <Status>Enabled</Status> <Filter> <Prefix>mypre/</Prefix> </Filter> </Rule> </LifecycleConfiguration> <LifecycleConfiguration> <Rule> <Status>Enabled</Status> <Filter> <Tag> <Key>key</Key> <Value>value</Value> </Tag> </Filter> </Rule> </LifecycleConfiguration> <LifecycleConfiguration> <Rule> <Status>Enabled</Status> <Filter> <And> <Prefix>key-prefix</Prefix> <Tag> <Key>key1</Key> <Value>value1</Value> </Tag> <Tag> <Key>key2</Key> <Value>value2</Value> </Tag> </And> </Filter> </Rule> </LifecycleConfiguration> GET / BUCKET ?lifecycle HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET PUT / BUCKET ?lifecycle HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET <LifecycleConfiguration> <Rule> <Expiration> <Days>10</Days> </Expiration> </Rule> <Rule> </Rule> </LifecycleConfiguration> DELETE / BUCKET ?lifecycle HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET GET / BUCKET ?location HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET GET / BUCKET ?versioning HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET PUT / BUCKET ?versioning HTTP/1.1 PUT /testbucket?versioning HTTP/1.1 GET / BUCKET ?acl HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET PUT / BUCKET ?acl HTTP/1.1 GET / BUCKET ?cors HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET PUT / BUCKET ?cors HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET DELETE / BUCKET ?cors HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET GET / BUCKET ?versions HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET HEAD / BUCKET HTTP/1.1 Host: cname.domain.com Date: date Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET GET / BUCKET ?uploads HTTP/1.1 cat > examplepol { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": {"AWS": ["arn:aws:iam::usfolks:user/fred"]}, "Action": "s3:PutObjectAcl", "Resource": [ "arn:aws:s3:::happybucket/*" ] }] } s3cmd setpolicy examplepol s3://happybucket s3cmd delpolicy s3://happybucket GET / BUCKET ?requestPayment HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET PUT / BUCKET ?requestPayment HTTP/1.1 Host: cname.domain.com https://rgw.domain.com/tenant:bucket from boto.s3.connection import S3Connection, OrdinaryCallingFormat c = S3Connection( aws_access_key_id="TESTER", aws_secret_access_key="test123", host="rgw.domain.com", calling_format = OrdinaryCallingFormat() ) bucket = c.get_bucket("tenant:bucket") { "Principal": "*", "Resource": "*", "Action": "s3:PutObject", "Effect": "Allow", "Condition": { "StringLike": {"aws:SourceVpc": "vpc-*"}} } { "Principal": "*", "Resource": "*", "Action": "s3:PutObject", "Effect": "Allow", "Condition": {"StringEquals": {"aws:SourceVpc": "vpc-91237329"}} } GET /v20180820/configuration/publicAccessBlock HTTP/1.1 Host: cname.domain.com x-amz-account-id: _ACCOUNTID_ PUT /?publicAccessBlock HTTP/1.1 Host: Bucket.s3.amazonaws.com Content-MD5: ContentMD5 x-amz-sdk-checksum-algorithm: ChecksumAlgorithm x-amz-expected-bucket-owner: ExpectedBucketOwner <?xml version="1.0" encoding="UTF-8"?> <PublicAccessBlockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <BlockPublicAcls>boolean</BlockPublicAcls> <IgnorePublicAcls>boolean</IgnorePublicAcls> <BlockPublicPolicy>boolean</BlockPublicPolicy> <RestrictPublicBuckets>boolean</RestrictPublicBuckets> </PublicAccessBlockConfiguration> DELETE /v20180820/configuration/publicAccessBlock HTTP/1.1 Host: s3-control.amazonaws.com x-amz-account-id: AccountId GET / BUCKET / OBJECT HTTP/1.1 GET / BUCKET / OBJECT ?versionId= VERSION_ID HTTP/1.1 GET / BUCKET / OBJECT ?partNumber= PARTNUMBER &versionId= VersionId HTTP/1.1 Host: Bucket.s3.amazonaws.com If-Match: IfMatch If-Modified-Since: IfModifiedSince If-None-Match: IfNoneMatch If-Unmodified-Since: IfUnmodifiedSince Range: Range GET /BUCKET/OBJECT?attributes&versionId=VersionId GET /testbucket/testobject?attributes&versionId=testversionid Host: Bucket.s3.amazonaws.com x-amz-max-parts: MaxParts x-amz-part-number-marker: PartNumberMarker x-amz-server-side-encryption-customer-algorithm: SSECustomerAlgorithm x-amz-server-side-encryption-customer-key: SSECustomerKey x-amz-server-side-encryption-customer-key-MD5: SSECustomerKeyMD5 x-amz-request-payer: RequestPayer x-amz-expected-bucket-owner: ExpectedBucketOwner x-amz-object-attributes: ObjectAttributes GET /{Key+}?attributes&versionId=VersionId HTTP/1.1 Host: Bucket.s3.amazonaws.com x-amz-max-parts: MaxParts x-amz-part-number-marker: PartNumberMarker x-amz-server-side-encryption-customer-algorithm: SSECustomerAlgorithm x-amz-server-side-encryption-customer-key: SSECustomerKey x-amz-server-side-encryption-customer-key-MD5: SSECustomerKeyMD5 x-amz-request-payer: RequestPayer x-amz-expected-bucket-owner: ExpectedBucketOwner x-amz-object-attributes: ObjectAttributes HTTP/1.1 200 x-amz-delete-marker: DeleteMarker Last-Modified: LastModified x-amz-version-id: VersionId x-amz-request-charged: RequestCharged <?xml version="1.0" encoding="UTF-8"?> <GetObjectAttributesOutput> <ETag>string</ETag> <Checksum> <ChecksumCRC32>string</ChecksumCRC32> <ChecksumCRC32C>string</ChecksumCRC32C> <ChecksumSHA1>string</ChecksumSHA1> <ChecksumSHA256>string</ChecksumSHA256> </Checksum> <ObjectParts> <IsTruncated>boolean</IsTruncated> <MaxParts>integer</MaxParts> <NextPartNumberMarker>integer</NextPartNumberMarker> <PartNumberMarker>integer</PartNumberMarker> <Part> <ChecksumCRC32>string</ChecksumCRC32> <ChecksumCRC32C>string</ChecksumCRC32C> <ChecksumSHA1>string</ChecksumSHA1> <ChecksumSHA256>string</ChecksumSHA256> <PartNumber>integer</PartNumber> <Size>long</Size> </Part> <PartsCount>integer</PartsCount> </ObjectParts> <StorageClass>string</StorageClass> <ObjectSize>long</ObjectSize> </GetObjectAttributesOutput> HEAD / BUCKET / OBJECT HTTP/1.1 HEAD / BUCKET / OBJECT ?versionId= VERSION_ID HTTP/1.1 PUT / BUCKET ?object-lock HTTP/1.1 PUT /testbucket?object-lock HTTP/1.1 GET / BUCKET ?object-lock HTTP/1.1 GET /testbucket?object-lock HTTP/1.1 PUT / BUCKET / OBJECT ?legal-hold&versionId= HTTP/1.1 PUT /testbucket/testobject?legal-hold&versionId= HTTP/1.1 GET / BUCKET / OBJECT ?legal-hold&versionId= HTTP/1.1 GET /testbucket/testobject?legal-hold&versionId= HTTP/1.1 PUT / BUCKET / OBJECT ?retention&versionId= HTTP/1.1 PUT /testbucket/testobject?retention&versionId= HTTP/1.1 GET / BUCKET / OBJECT ?retention&versionId= HTTP/1.1 GET /testbucket/testobject?retention&versionId= HTTP/1.1 PUT / BUCKET / OBJECT ?tagging&versionId= HTTP/1.1 PUT /testbucket/testobject?tagging&versionId= HTTP/1.1 GET / BUCKET / OBJECT ?tagging&versionId= HTTP/1.1 GET /testbucket/testobject?tagging&versionId= HTTP/1.1 DELETE / BUCKET / OBJECT ?tagging&versionId= HTTP/1.1 DELETE /testbucket/testobject?tagging&versionId= HTTP/1.1 PUT / BUCKET / OBJECT HTTP/1.1 DELETE / BUCKET / OBJECT HTTP/1.1 DELETE / BUCKET / OBJECT ?versionId= VERSION_ID HTTP/1.1 POST / BUCKET / OBJECT ?delete HTTP/1.1 GET / BUCKET / OBJECT ?acl HTTP/1.1 GET / BUCKET / OBJECT ?versionId= VERSION_ID &acl HTTP/1.1 PUT / BUCKET / OBJECT ?acl PUT / DEST_BUCKET / DEST_OBJECT HTTP/1.1 x-amz-copy-source: SOURCE_BUCKET / SOURCE_OBJECT POST / BUCKET / OBJECT HTTP/1.1 OPTIONS / OBJECT HTTP/1.1 POST / BUCKET / OBJECT ?uploads PUT / BUCKET / OBJECT ?partNumber=&uploadId= UPLOAD_ID HTTP/1.1 GET / BUCKET / OBJECT ?uploadId= UPLOAD_ID HTTP/1.1 POST / BUCKET / OBJECT ?uploadId= UPLOAD_ID HTTP/1.1 PUT / BUCKET / OBJECT ?partNumber=PartNumber&uploadId= UPLOAD_ID HTTP/1.1 Host: cname.domain.com Authorization: AWS ACCESS_KEY : HASH_OF_HEADER_AND_SECRET DELETE / BUCKET / OBJECT ?uploadId= UPLOAD_ID HTTP/1.1 select customerid from s3Object where age>30 and age<65; POST / BUCKET / KEY ?select&select-type=2 HTTP/1.1\r\n POST /testbucket/sample1csv?select&select-type=2 HTTP/1.1\r\n POST /testbucket/sample1parquet?select&select-type=2 HTTP/1.1\r\n {:event-type,records} {:content-type,application/octet-stream} {:message-type,event} aws --endpoint- URL http://localhost:80 s3api select-object-content --bucket BUCKET_NAME --expression-type 'SQL' --input-serialization '{"CSV": {"FieldDelimiter": "," , "QuoteCharacter": "\"" , "RecordDelimiter" : "\n" , "QuoteEscapeCharacter" : "\\" , "FileHeaderInfo": "USE" }, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --key OBJECT_NAME .csv --expression "select count(0) from s3object where int(_1)<10;" output.csv aws --endpoint-url http://localhost:80 s3api select-object-content --bucket testbucket --expression-type 'SQL' --input-serialization '{"CSV": {"FieldDelimiter": "," , "QuoteCharacter": "\"" , "RecordDelimiter" : "\n" , "QuoteEscapeCharacter" : "\\" , "FileHeaderInfo": "USE" }, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --key testobject.csv --expression "select count(0) from s3object where int(_1)<10;" output.csv aws --endpoint-url http://localhost:80 s3api select-object-content --bucket BUCKET_NAME --expression-type 'SQL' --input-serialization '{"Parquet": {}, {"CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --key OBJECT_NAME .parquet --expression "select count(0) from s3object where int(_1)<10;" output.csv aws --endpoint-url http://localhost:80 s3api select-object-content --bucket testbucket --expression-type 'SQL' --input-serialization '{"Parquet": {}, {"CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --key testobject.parquet --expression "select count(0) from s3object where int(_1)<10;" output.csv aws --endpoint- URL http://localhost:80 s3api select-object-content --bucket BUCKET_NAME --expression-type 'SQL' --input-serialization '{"JSON": {"CompressionType": "NONE"}' --output-serialization '{"CSV": {}}}' --key OBJECT_NAME .json --expression "select count(0) from s3object where int(_1)<10;" output.csv aws --endpoint-url http://localhost:80 s3api select-object-content --bucket testbucket --expression-type 'SQL' --input-serialization '{"JSON": {"CompressionType": "NONE"}' --output-serialization '{"CSV": {}}}' --key testobject.json --expression "select count(0) from s3object where int(_1)<10;" output.csv import pprint import boto3 from botocore.exceptions import ClientError def run_s3select(bucket,key,query,column_delim=",",row_delim="\n",quot_char='"',esc_char='\\',csv_header_info="NONE"): s3 = boto3.client('s3', endpoint_url=endpoint, aws_access_key_id=access_key, region_name=region_name, aws_secret_access_key=secret_key) result = "" try: r = s3.select_object_content( Bucket=bucket, Key=key, ExpressionType='SQL', InputSerialization = {"CSV": {"RecordDelimiter" : row_delim, "FieldDelimiter" : column_delim,"QuoteEscapeCharacter": esc_char, "QuoteCharacter": quot_char, "FileHeaderInfo": csv_header_info}, "CompressionType": "NONE"}, OutputSerialization = {"CSV": {}}, Expression=query, RequestProgress = {"Enabled": progress}) except ClientError as c: result += str(c) return result for event in r['Payload']: if 'Records' in event: result = "" records = event['Records']['Payload'].decode('utf-8') result += records if 'Progress' in event: print("progress") pprint.pprint(event['Progress'],width=1) if 'Stats' in event: print("Stats") pprint.pprint(event['Stats'],width=1) if 'End' in event: print("End") pprint.pprint(event['End'],width=1) return result run_s3select( "my_bucket", "my_csv_object", "select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3 from s3object where a3>100 and a3<300;") select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3 from s3object where a3>100 and a3<300;") 4-byte magic number "PAR1" <Column 1 Chunk 1 + Column Metadata> <Column 2 Chunk 1 + Column Metadata> <Column N Chunk 1 + Column Metadata> <Column 1 Chunk 2 + Column Metadata> <Column 2 Chunk 2 + Column Metadata> <Column N Chunk 2 + Column Metadata> <Column 1 Chunk M + Column Metadata> <Column 2 Chunk M + Column Metadata> <Column N Chunk M + Column Metadata> File Metadata 4-byte length in bytes of file metadata 4-byte magic number "PAR1" { "firstName": "Joe", "lastName": "Jackson", "gender": "male", "age": "twenty" }, { "firstName": "Joe_2", "lastName": "Jackson_2", "gender": "male", "age": 21 }, "phoneNumbers": [ { "type": "home1", "number": "734928_1","addr": 11 }, { "type": "home2", "number": "734928_2","addr": 22 } ], "key_after_array": "XXX", "description" : { "main_desc" : "value_1", "second_desc" : "value_2" } the from-clause define a single row. _1 points to root object level. _1.age appears twice in Documnet-row, the last value is used for the operation. query = "select _1.firstname,_1.key_after_array,_1.age+4,_1.description.main_desc,_1.description.second_desc from s3object[*].aa.bb.cc;"; expected_result = Joe_2,XXX,25,value_1,value_2 [cephuser@host01 ~]USD git clone https://github.com/ceph/s3select.git [cephuser@host01 ~]USD cd s3select [cephuser@host01 s3select]USD cat container/trino/hms_trino.yaml version: '3' services: hms: image: galsl/hms:dev container_name: hms environment: # S3_ENDPOINT the CEPH/RGW end-point-url - S3_ENDPOINT=http://rgw_ip:port - S3_ACCESS_KEY=abc - S3_SECRET_KEY=abc # the container starts with booting the hive metastore command: sh -c '. ~/.bashrc; start_hive_metastore' ports: - 9083:9083 networks: - trino_hms trino: image: trinodb/trino:405 container_name: trino volumes: # the trino directory contains the necessary configuration - ./trino:/etc/trino ports: - 8080:8080 networks: - trino_hms networks: trino_hm [cephuser@host01 s3select]USD cat container/trino/trino/catalog/hive.properties connector.name=hive hive.metastore.uri=thrift://hms:9083 #hive.metastore.warehouse.dir=s3a://hive/ hive.allow-drop-table=true hive.allow-rename-table=true hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true hive.non-managed-table-writes-enabled=true hive.s3select-pushdown.enabled=true hive.s3.aws-access-key=abc hive.s3.aws-secret-key=abc should modify per s3-endpoint-url hive.s3.endpoint=http://rgw_ip:port #hive.s3.max-connections=1 #hive.s3select-pushdown.max-connections=1 hive.s3.connect-timeout=100s hive.s3.socket-timeout=100s hive.max-splits-per-second=10000 hive.max-split-size=128MB [cephuser@host01 s3select]USD sudo docker compose -f ./container/trino/hms_trino.yaml up -d [cephuser@host01 s3select]USD sudo docker exec -it trino /bin/bash trino@66f753905e82:/USD trino trino> create schema hive.csvbkt1schema; trino> create table hive.csvbkt1schema.polariondatacsv(c1 varchar,c2 varchar, c3 varchar, c4 varchar, c5 varchar, c6 varchar, c7 varchar, c8 varchar, c9 varchar) WITH ( external_location = 's3a://csvbkt1/',format = 'CSV'); trino> select * from hive.csvbkt1schema.polariondatacsv; radosgw-admin subuser create --uid= NAME --subuser= NAME :swift --access=full radosgw-admin subuser create --uid=testuser --subuser=testuser:swift --access=full { "user_id": "testuser", "display_name": "First User", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [ { "id": "testuser:swift", "permissions": "full-control" } ], "keys": [ { "user": "testuser", "access_key": "O8JDE41XMI74O185EHKD", "secret_key": "i4Au2yxG5wtr1JK01mI8kjJPM93HNAoVWOSTdJd6" } ], "swift_keys": [ { "user": "testuser:swift", "secret_key": "13TLtdEW7bCqgttQgPzxFxziu0AgabtOc6vM8DLA" } ], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw" } radosgw-admin key create --subuser= NAME :swift --key-type=swift --gen-secret radosgw-admin key create --subuser=testuser:swift --key-type=swift --gen-secret { "user_id": "testuser", "display_name": "First User", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [ { "id": "testuser:swift", "permissions": "full-control" } ], "keys": [ { "user": "testuser", "access_key": "O8JDE41XMI74O185EHKD", "secret_key": "i4Au2yxG5wtr1JK01mI8kjJPM93HNAoVWOSTdJd6" } ], "swift_keys": [ { "user": "testuser:swift", "secret_key": "a4ioT4jEP653CDcdU8p4OuhruwABBRZmyNUbnSSt" } ], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw" } GET /auth HTTP/1.1 Host: swift.example.com X-Auth-User: johndoe X-Auth-Key: R7UUOLFDI2ZI9PRCQ53K HTTP/1.1 204 No Content Date: Mon, 16 Jul 2012 11:05:33 GMT Server: swift X-Storage-Url: https://swift.example.com X-Storage-Token: UOlCCC8TahFKlWuv9DB09TWHF0nDjpPElha0kAa Content-Length: 0 Content-Type: text/plain; charset=UTF-8 POST / API_VERSION / ACCOUNT / TENANT : CONTAINER HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN X-Container-Read: * X-Container-Write: UID1 , UID2 , UID3 GET / API_VERSION / ACCOUNT HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN GET / API_VERSION / TENANT : CONTAINER HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN PUT / API_VERSION / ACCOUNT / TENANT : CONTAINER HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN X-Container-Read: COMMA_SEPARATED_UIDS X-Container-Write: COMMA_SEPARATED_UIDS X-Container-Meta- KEY : VALUE X-Storage-Policy: PLACEMENT_POOLS_KEY DELETE / API_VERSION / ACCOUNT / TENANT : CONTAINER HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN POST / API_VERSION / ACCOUNT / TENANT : CONTAINER HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN X-Container-Meta-Color: red X-Container-Meta-Taste: salty GET / API_VERSION / ACCOUNT / TENANT : CONTAINER / OBJECT HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN PUT / API_VERSION / ACCOUNT / TENANT : CONTAINER HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN DELETE / API_VERSION / ACCOUNT / TENANT : CONTAINER / OBJECT HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN PUT / API_VERSION / ACCOUNT / TENANT : CONTAINER HTTP/1.1 X-Copy-From: TENANT : SOURCE_CONTAINER / SOURCE_OBJECT Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN COPY / API_VERSION / ACCOUNT / TENANT : SOURCE_CONTAINER / SOURCE_OBJECT HTTP/1.1 Destination: TENANT : DEST_CONTAINER / DEST_OBJECT HEAD / API_VERSION / ACCOUNT / TENANT : CONTAINER / OBJECT HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN POST / API_VERSION / ACCOUNT / TENANT : CONTAINER / OBJECT HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN import hmac from hashlib import sha1 from time import time method = 'GET' host = 'https://objectstore.example.com' duration_in_seconds = 300 # Duration for which the url is valid expires = int(time() + duration_in_seconds) path = '/v1/your-bucket/your-object' key = 'secret' hmac_body = '%s\n%s\n%s' % (method, expires, path) hmac_body = hmac.new(key, hmac_body, sha1).hexdigest() sig = hmac.new(key, hmac_body, sha1).hexdigest() rest_uri = "{host}{path}?temp_url_sig={sig}&temp_url_expires={expires}".format( host=host, path=path, sig=sig, expires=expires) print rest_uri https://objectstore.example.com/v1/your-bucket/your-object?temp_url_sig=ff4657876227fc6025f04fcf1e82818266d022c6&temp_url_expires=1423200992 POST / API_VERSION / ACCOUNT HTTP/1.1 Host: FULLY_QUALIFIED_DOMAIN_NAME X-Auth-Token: AUTH_TOKEN https://rgw.domain.com/tenant:container create_container("tenant:container") GET /api/summary HTTP/1.1 Host: example.com curl -i -k --location -X POST 'https://192.168.0.44:8443/api/auth' -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' --data '{"password": "admin@123", "username": "admin"}' POST /api/auth HTTP/1.1 Host: example.com Content-Type: application/json { "password": " STRING ", "username": " STRING " } POST /api/auth/check?token= STRING HTTP/1.1 Host: example.com Content-Type: application/json { "token": " STRING " } GET /api/cephfs HTTP/1.1 Host: example.com GET /api/cephfs/ FS_ID HTTP/1.1 Host: example.com GET /api/cephfs/ FS_ID /clients HTTP/1.1 Host: example.com GET /api/cephfs/ FS_ID /get_root_directory HTTP/1.1 Host: example.com GET /api/cephfs/ FS_ID /ls_dir HTTP/1.1 Host: example.com GET /api/cephfs/ FS_ID /mds_counters HTTP/1.1 Host: example.com GET /api/cephfs/ FS_ID /quota?path= STRING HTTP/1.1 Host: example.com PUT /api/cephfs/ FS_ID /quota HTTP/1.1 Host: example.com Content-Type: application/json { "max_bytes": " STRING ", "max_files": " STRING ", "path": " STRING " } POST /api/cephfs/ FS_ID /snapshot HTTP/1.1 Host: example.com Content-Type: application/json { "name": " STRING ", "path": " STRING " } POST /api/cephfs/ FS_ID /tree HTTP/1.1 Host: example.com Content-Type: application/json { "path": " STRING " } GET /api/cluster_conf HTTP/1.1 Host: example.com POST /api/cluster_conf HTTP/1.1 Host: example.com Content-Type: application/json { "name": " STRING ", "value": " STRING " } PUT /api/cluster_conf HTTP/1.1 Host: example.com Content-Type: application/json { "options": " STRING " } GET /api/cluster_conf/filter HTTP/1.1 Host: example.com GET /api/cluster_conf/ NAME HTTP/1.1 Host: example.com GET /api/crush_rule HTTP/1.1 Host: example.com POST /api/crush_rule HTTP/1.1 Host: example.com Content-Type: application/json { "device_class": " STRING ", "failure_domain": " STRING ", "name": " STRING ", "root": " STRING " } GET /api/crush_rule/ NAME HTTP/1.1 Host: example.com GET /api/erasure_code_profile HTTP/1.1 Host: example.com POST /api/erasure_code_profile HTTP/1.1 Host: example.com Content-Type: application/json { "name": " STRING " } GET /api/erasure_code_profile/ NAME HTTP/1.1 Host: example.com GET /api/feature_toggles HTTP/1.1 Host: example.com GET /api/grafana/url HTTP/1.1 Host: example.com GET /api/grafana/validation/ PARAMS HTTP/1.1 Host: example.com GET /api/health/full HTTP/1.1 Host: example.com GET /api/health/minimal HTTP/1.1 Host: example.com GET /api/host HTTP/1.1 Host: example.com POST /api/host HTTP/1.1 Host: example.com Content-Type: application/json { "hostname": " STRING ", "status": " STRING " } GET /api/host/ HOST_NAME HTTP/1.1 Host: example.com PUT /api/host/ HOST_NAME HTTP/1.1 Host: example.com Content-Type: application/json { "force": true, "labels": [ " STRING " ], "maintenance": true, "update_labels": true } GET /api/host/ HOST_NAME /daemons HTTP/1.1 Host: example.com GET /api/host/ HOST_NAME /devices HTTP/1.1 Host: example.com POST /api/host/ HOST_NAME /identify_device HTTP/1.1 Host: example.com Content-Type: application/json { "device": " STRING ", "duration": " STRING " } GET /api/host/ HOST_NAME /inventory HTTP/1.1 Host: example.com GET /api/host/ HOST_NAME /smart HTTP/1.1 Host: example.com GET /api/logs/all HTTP/1.1 Host: example.com GET /api/mgr/module HTTP/1.1 Host: example.com GET /api/mgr/module/ MODULE_NAME HTTP/1.1 Host: example.com PUT /api/mgr/module/ MODULE_NAME HTTP/1.1 Host: example.com Content-Type: application/json { "config": " STRING " } GET /api/mgr/module/ MODULE_NAME /options HTTP/1.1 Host: example.com GET /api/monitor HTTP/1.1 Host: example.com GET /api/osd HTTP/1.1 Host: example.com POST /api/osd HTTP/1.1 Host: example.com Content-Type: application/json { "data": " STRING ", "method": " STRING ", "tracking_id": " STRING " } GET /api/osd/flags HTTP/1.1 Host: example.com PUT /api/osd/flags HTTP/1.1 Host: example.com Content-Type: application/json { "flags": [ " STRING " ] } GET /api/osd/flags/individual HTTP/1.1 Host: example.com PUT /api/osd/flags/individual HTTP/1.1 Host: example.com Content-Type: application/json { "flags": { "nodown": true, "noin": true, "noout": true, "noup": true }, "ids": [ 1 ] } GET /api/osd/safe_to_delete?svc_ids= STRING HTTP/1.1 Host: example.com GET /api/osd/safe_to_destroy?ids= STRING HTTP/1.1 Host: example.com GET /api/osd/ SVC_ID HTTP/1.1 Host: example.com PUT /api/osd/ SVC_ID HTTP/1.1 Host: example.com Content-Type: application/json { "device_class": " STRING " } GET /api/osd/ SVC_ID /devices HTTP/1.1 Host: example.com GET /api/osd/ SVC_ID /histogram HTTP/1.1 Host: example.com PUT /api/osd/ SVC_ID /mark HTTP/1.1 Host: example.com Content-Type: application/json { "action": " STRING " } POST /api/osd/ SVC_ID /reweight HTTP/1.1 Host: example.com Content-Type: application/json { "weight": " STRING " } POST /api/osd/ SVC_ID /scrub HTTP/1.1 Host: example.com Content-Type: application/json { "deep": true } GET /api/osd/ SVC_ID /smart HTTP/1.1 Host: example.com GET /api/rgw/status HTTP/1.1 Host: example.com GET /api/rgw/daemon HTTP/1.1 Host: example.com GET /api/rgw/daemon/ SVC_ID HTTP/1.1 Host: example.com GET /api/rgw/site HTTP/1.1 Host: example.com GET /api/rgw/bucket HTTP/1.1 Host: example.com POST /api/rgw/bucket HTTP/1.1 Host: example.com Content-Type: application/json { "bucket": " STRING ", "daemon_name": " STRING ", "lock_enabled": "false", "lock_mode": " STRING ", "lock_retention_period_days": " STRING ", "lock_retention_period_years": " STRING ", "placement_target": " STRING ", "uid": " STRING ", "zonegroup": " STRING " } GET /api/rgw/bucket/ BUCKET HTTP/1.1 Host: example.com PUT /api/rgw/bucket/ BUCKET HTTP/1.1 Host: example.com Content-Type: application/json { "bucket_id": " STRING ", "daemon_name": " STRING ", "lock_mode": " STRING ", "lock_retention_period_days": " STRING ", "lock_retention_period_years": " STRING ", "mfa_delete": " STRING ", "mfa_token_pin": " STRING ", "mfa_token_serial": " STRING ", "uid": " STRING ", "versioning_state": " STRING " } GET /api/rgw/user HTTP/1.1 Host: example.com POST /api/rgw/user HTTP/1.1 Host: example.com Content-Type: application/json { "access_key": " STRING ", "daemon_name": " STRING ", "display_name": " STRING ", "email": " STRING ", "generate_key": " STRING ", "max_buckets": " STRING ", "secret_key": " STRING ", "suspended": " STRING ", "uid": " STRING " } GET /api/rgw/user/get_emails HTTP/1.1 Host: example.com GET /api/rgw/user/ UID HTTP/1.1 Host: example.com PUT /api/rgw/user/ UID HTTP/1.1 Host: example.com Content-Type: application/json { "daemon_name": " STRING ", "display_name": " STRING ", "email": " STRING ", "max_buckets": " STRING ", "suspended": " STRING " } POST /api/rgw/user/ UID /capability HTTP/1.1 Host: example.com Content-Type: application/json { "daemon_name": " STRING ", "perm": " STRING ", "type": " STRING " } POST /api/rgw/user/ UID /key HTTP/1.1 Host: example.com Content-Type: application/json { "access_key": " STRING ", "daemon_name": " STRING ", "generate_key": "true", "key_type": "s3", "secret_key": " STRING ", "subuser": " STRING " } GET /api/rgw/user/ UID /quota HTTP/1.1 Host: example.com PUT /api/rgw/user/ UID /quota HTTP/1.1 Host: example.com Content-Type: application/json { "daemon_name": " STRING ", "enabled": " STRING ", "max_objects": " STRING ", "max_size_kb": 1, "quota_type": " STRING " } POST /api/rgw/user/ UID /subuser HTTP/1.1 Host: example.com Content-Type: application/json { "access": " STRING ", "access_key": " STRING ", "daemon_name": " STRING ", "generate_secret": "true", "key_type": "s3", "secret_key": " STRING ", "subuser": " STRING " } radosgw-admin --uid TESTER --display-name "TestUser" --access_key TESTER --secret test123 user create radosgw-admin caps add --uid="TESTER" --caps="roles=*" POST "<hostname>?Action=CreateRole&RoleName= ROLE_NAME &Path= PATH_TO_FILE &AssumeRolePolicyDocument= TRUST_RELATIONSHIP_POLICY_DOCUMENT " POST "<hostname>?Action=CreateRole&RoleName=S3Access&Path=/application_abc/component_xyz/&AssumeRolePolicyDocument={"Version":"2022-06-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}" <role> <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id> <name>S3Access</name> <path>/application_abc/component_xyz/</path> <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn> <create_date>2022-06-23T07:43:42.811Z</create_date> <max_session_duration>3600</max_session_duration> <assume_role_policy_document>{"Version":"2022-06-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document> </role> POST "<hostname>?Action=GetRole&RoleName= ROLE_NAME " POST "<hostname>?Action=GetRole&RoleName=S3Access" <role> <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id> <name>S3Access</name> <path>/application_abc/component_xyz/</path> <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn> <create_date>2022-06-23T07:43:42.811Z</create_date> <max_session_duration>3600</max_session_duration> <assume_role_policy_document>{"Version":"2022-06-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document> </role> POST "<hostname>?Action=GetRole&RoleName= ROLE_NAME &PathPrefix= PATH_PREFIX " POST "<hostname>?Action=ListRoles&RoleName=S3Access&PathPrefix=/application" <role> <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id> <name>S3Access</name> <path>/application_abc/component_xyz/</path> <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn> <create_date>2022-06-23T07:43:42.811Z</create_date> <max_session_duration>3600</max_session_duration> <assume_role_policy_document>{"Version":"2022-06-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document> </role> POST "<hostname>?Action=UpdateAssumeRolePolicy&RoleName= ROLE_NAME &PolicyDocument= TRUST_RELATIONSHIP_POLICY_DOCUMENT " POST "<hostname>?Action=UpdateAssumeRolePolicy&RoleName=S3Access&PolicyDocument={"Version":"2022-06-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER2"]},"Action":["sts:AssumeRole"]}]}" POST "<hostname>?Action=PutRolePolicy&RoleName= ROLE_NAME &PolicyName= POLICY_NAME &PolicyDocument= TRUST_RELATIONSHIP_POLICY_DOCUMENT " POST "<hostname>?Action=PutRolePolicy&RoleName=S3Access&PolicyName=Policy1&PolicyDocument={"Version":"2022-06-17","Statement":[{"Effect":"Allow","Action":["s3:CreateBucket"],"Resource":"arn:aws:s3:::example_bucket"}]}" POST "<hostname>?Action=ListRolePolicies&RoleName= ROLE_NAME " POST "<hostname>?Action=ListRolePolicies&RoleName=S3Access" <PolicyNames> <member>Policy1</member> </PolicyNames> POST "<hostname>?Action=GetRolePolicy&RoleName= ROLE_NAME &PolicyName= POLICY_NAME " POST "<hostname>?Action=GetRolePolicy&RoleName=S3Access&PolicyName=Policy1" <GetRolePolicyResult> <PolicyName>Policy1</PolicyName> <RoleName>S3Access</RoleName> <Permission_policy>{"Version":"2022-06-17","Statement":[{"Effect":"Allow","Action":["s3:CreateBucket"],"Resource":"arn:aws:s3:::example_bucket"}]}</Permission_policy> </GetRolePolicyResult> POST "hostname>?Action=DeleteRolePolicy&RoleName= ROLE_NAME &PolicyName= POLICY_NAME " POST "<hostname>?Action=DeleteRolePolicy&RoleName=S3Access&PolicyName=Policy1" POST "<hostname>?Action=DeleteRole&RoleName= ROLE_NAME " POST "<hostname>?Action=DeleteRole&RoleName=S3Access" GET /api/nfs-ganesha/daemon HTTP/1.1 Host: example.com GET /api/nfs-ganesha/export HTTP/1.1 Host: example.com POST /api/nfs-ganesha/export HTTP/1.1 Host: example.com Content-Type: application/json { "access_type": " STRING ", "clients": [ { "access_type": " STRING ", "addresses": [ " STRING " ], "squash": " STRING " } ], "cluster_id": " STRING ", "daemons": [ " STRING " ], "fsal": { "filesystem": " STRING ", "name": " STRING ", "rgw_user_id": " STRING ", "sec_label_xattr": " STRING ", "user_id": " STRING " }, "path": " STRING ", "protocols": [ 1 ], "pseudo": " STRING ", "reload_daemons": true, "security_label": " STRING ", "squash": " STRING ", "tag": " STRING ", "transports": [ " STRING " ] } GET /api/nfs-ganesha/export/ CLUSTER_ID / EXPORT_ID HTTP/1.1 Host: example.com PUT /api/nfs-ganesha/export/ CLUSTER_ID / EXPORT_ID HTTP/1.1 Host: example.com Content-Type: application/json { "access_type": " STRING ", "clients": [ { "access_type": " STRING ", "addresses": [ " STRING " ], "squash": " STRING " } ], "daemons": [ " STRING " ], "fsal": { "filesystem": " STRING ", "name": " STRING ", "rgw_user_id": " STRING ", "sec_label_xattr": " STRING ", "user_id": " STRING " }, "path": " STRING ", "protocols": [ 1 ], "pseudo": " STRING ", "reload_daemons": true, "security_label": " STRING ", "squash": " STRING ", "tag": " STRING ", "transports": [ " STRING " ] } GET /api/nfs-ganesha/status HTTP/1.1 Host: example.com GET /api/orchestrator/status HTTP/1.1 Host: example.com GET /api/pool HTTP/1.1 Host: example.com POST /api/pool HTTP/1.1 Host: example.com Content-Type: application/json { "application_metadata": " STRING ", "configuration": " STRING ", "erasure_code_profile": " STRING ", "flags": " STRING ", "pg_num": 1, "pool": " STRING ", "pool_type": " STRING ", "rule_name": " STRING " } GET /api/pool/ POOL_NAME HTTP/1.1 Host: example.com PUT /api/pool/ POOL_NAME HTTP/1.1 Host: example.com Content-Type: application/json { "application_metadata": " STRING ", "configuration": " STRING ", "flags": " STRING " } GET /api/pool/ POOL_NAME /configuration HTTP/1.1 Host: example.com GET /api/prometheus/rules HTTP/1.1 Host: example.com GET /api/prometheus/rules HTTP/1.1 Host: example.com GET /api/prometheus/silences HTTP/1.1 Host: example.com GET /api/prometheus/notifications HTTP/1.1 Host: example.com GET /api/block/image HTTP/1.1 Host: example.com POST /api/block/image HTTP/1.1 Host: example.com Content-Type: application/json { "configuration": " STRING ", "data_pool": " STRING ", "features": " STRING ", "name": " STRING ", "namespace": " STRING ", "obj_size": 1, "pool_name": " STRING ", "size": 1, "stripe_count": 1, "stripe_unit": " STRING " } GET /api/block/image/clone_format_version HTTP/1.1 Host: example.com GET /api/block/image/default_features HTTP/1.1 Host: example.com GET /api/block/image/default_features HTTP/1.1 Host: example.com GET /api/block/image/ IMAGE_SPEC HTTP/1.1 Host: example.com PUT /api/block/image/ IMAGE_SPEC HTTP/1.1 Host: example.com Content-Type: application/json { "configuration": " STRING ", "features": " STRING ", "name": " STRING ", "size": 1 } POST /api/block/image/ IMAGE_SPEC /copy HTTP/1.1 Host: example.com Content-Type: application/json { "configuration": " STRING ", "data_pool": " STRING ", "dest_image_name": " STRING ", "dest_namespace": " STRING ", "dest_pool_name": " STRING ", "features": " STRING ", "obj_size": 1, "snapshot_name": " STRING ", "stripe_count": 1, "stripe_unit": " STRING " } POST /api/block/image/ IMAGE_SPEC /move_trash HTTP/1.1 Host: example.com Content-Type: application/json { "delay": 1 } GET /api/block/mirroring/site_name HTTP/1.1 Host: example.com PUT /api/block/mirroring/site_name HTTP/1.1 Host: example.com Content-Type: application/json { "site_name": " STRING " } POST /api/block/mirroring/pool/ POOL_NAME /bootstrap/peer HTTP/1.1 Host: example.com Content-Type: application/json { "direction": " STRING ", "token": " STRING " } GET /api/block/mirroring/pool/ POOL_NAME HTTP/1.1 Host: example.com PUT /api/block/mirroring/pool/ POOL_NAME HTTP/1.1 Host: example.com Content-Type: application/json { "mirror_mode": " STRING " } GET /api/block/mirroring/pool/ POOL_NAME /peer HTTP/1.1 Host: example.com POST /api/block/mirroring/pool/ POOL_NAME /peer HTTP/1.1 Host: example.com Content-Type: application/json { "client_id": " STRING ", "cluster_name": " STRING ", "key": " STRING ", "mon_host": " STRING " } GET /api/block/mirroring/pool/ POOL_NAME /peer/ PEER_UUID HTTP/1.1 Host: example.com PUT /api/block/mirroring/pool/ POOL_NAME /peer/ PEER_UUID HTTP/1.1 Host: example.com Content-Type: application/json { "client_id": " STRING ", "cluster_name": " STRING ", "key": " STRING ", "mon_host": " STRING " } GET /api/block/mirroring/summary HTTP/1.1 Host: example.com GET /api/block/pool/ POOL_NAME /namespace HTTP/1.1 Host: example.com POST /api/block/pool/ POOL_NAME /namespace HTTP/1.1 Host: example.com Content-Type: application/json { "namespace": " STRING " } POST /api/block/image/ IMAGE_SPEC /snap HTTP/1.1 Host: example.com Content-Type: application/json { "snapshot_name": " STRING " } PUT /api/block/image/ IMAGE_SPEC /snap/ SNAPSHOT_NAME HTTP/1.1 Host: example.com Content-Type: application/json { "is_protected": true, "new_snap_name": " STRING " } POST /api/block/image/ IMAGE_SPEC /snap/ SNAPSHOT_NAME /clone HTTP/1.1 Host: example.com Content-Type: application/json { "child_image_name": " STRING ", "child_namespace": " STRING ", "child_pool_name": " STRING ", "configuration": " STRING ", "data_pool": " STRING ", "features": " STRING ", "obj_size": 1, "stripe_count": 1, "stripe_unit": " STRING " } GET /api/block/image/trash HTTP/1.1 Host: example.com POST /api/block/image/trash/purge HTTP/1.1 Host: example.com Content-Type: application/json { "pool_name": " STRING " } POST /api/block/image/trash/ IMAGE_ID_SPEC /restore HTTP/1.1 Host: example.com Content-Type: application/json { "new_image_name": " STRING " } GET /api/perf_counters HTTP/1.1 Host: example.com GET /api/perf_counters/mds/ SERVICE_ID HTTP/1.1 Host: example.com GET /api/perf_counters/mgr/ SERVICE_ID HTTP/1.1 Host: example.com GET /api/perf_counters/mon/ SERVICE_ID HTTP/1.1 Host: example.com GET /api/perf_counters/osd/ SERVICE_ID HTTP/1.1 Host: example.com GET /api/perf_counters/rbd-mirror/ SERVICE_ID HTTP/1.1 Host: example.com GET /api/perf_counters/rgw/ SERVICE_ID HTTP/1.1 Host: example.com GET /api/perf_counters/tcmu-runner/ SERVICE_ID HTTP/1.1 Host: example.com GET /api/role HTTP/1.1 Host: example.com POST /api/role HTTP/1.1 Host: example.com Content-Type: application/json { "description": " STRING ", "name": " STRING ", "scopes_permissions": " STRING " } GET /api/role/ NAME HTTP/1.1 Host: example.com PUT /api/role/ NAME HTTP/1.1 Host: example.com Content-Type: application/json { "description": " STRING ", "scopes_permissions": " STRING " } POST /api/role/ NAME /clone HTTP/1.1 Host: example.com Content-Type: application/json { "new_name": " STRING " } GET /api/service HTTP/1.1 Host: example.com POST /api/service HTTP/1.1 Host: example.com Content-Type: application/json { "service_name": " STRING ", "service_spec": " STRING " } GET /api/service/known_types HTTP/1.1 Host: example.com GET /api/service/ SERVICE_NAME HTTP/1.1 Host: example.com GET /api/service/ SERVICE_NAME /daemons HTTP/1.1 Host: example.com GET /api/settings HTTP/1.1 Host: example.com GET /api/settings/ NAME HTTP/1.1 Host: example.com PUT /api/settings/ NAME HTTP/1.1 Host: example.com Content-Type: application/json { "value": " STRING " } GET /api/task HTTP/1.1 Host: example.com PUT /api/telemetry HTTP/1.1 Host: example.com Content-Type: application/json { "enable": true, "license_name": " STRING " } GET /api/telemetry/report HTTP/1.1 Host: example.com GET /api/user HTTP/1.1 Host: example.com POST /api/user HTTP/1.1 Host: example.com Content-Type: application/json { "email": " STRING ", "enabled": true, "name": " STRING ", "password": " STRING ", "pwdExpirationDate": " STRING ", "pwdUpdateRequired": true, "roles": " STRING ", "username": " STRING " } GET /api/user/ USER_NAME HTTP/1.1 Host: example.com PUT /api/user/ USER_NAME HTTP/1.1 Host: example.com Content-Type: application/json { "email": " STRING ", "enabled": " STRING ", "name": " STRING ", "password": " STRING ", "pwdExpirationDate": " STRING ", "pwdUpdateRequired": true, "roles": " STRING " } POST /api/user/ USER_NAME /change_password HTTP/1.1 Host: example.com Content-Type: application/json { "new_password": " STRING ", "old_password": " STRING " } POST /api/user/validate_password HTTP/1.1 Host: example.com Content-Type: application/json { "old_password": " STRING ", "password": " STRING ", "username": " STRING " } import boto3 iam_client = boto3.client('iam', aws_access_key_id= ACCESS_KEY_OF_TESTER1 , aws_secret_access_key= SECRET_KEY_OF_TESTER1 , endpoint_url=<IAM URL>, region_name='' ) policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER1\"]},\"Action\":[\"sts:AssumeRole\"]}]}" role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, Path='/', RoleName='S3Access', ) role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}" response = iam_client.put_role_policy( RoleName='S3Access', PolicyName='Policy1', PolicyDocument=role_policy ) sts_client = boto3.client('sts', aws_access_key_id= ACCESS_KEY_OF_TESTER2 , aws_secret_access_key= SECRET_KEY_OF_TESTER2 , endpoint_url=<STS URL>, region_name='', ) response = sts_client.assume_role( RoleArn=role_response['Role']['Arn'], RoleSessionName='Bob', DurationSeconds=3600 ) s3client = boto3.client('s3', aws_access_key_id = response['Credentials']['AccessKeyId'], aws_secret_access_key = response['Credentials']['SecretAccessKey'], aws_session_token = response['Credentials']['SessionToken'], endpoint_url=<S3 URL>, region_name='',) bucket_name = 'my-bucket' s3bucket = s3client.create_bucket(Bucket=bucket_name) resp = s3client.list_buckets() import boto3 iam_client = boto3.client('iam', aws_access_key_id= ACCESS_KEY_OF_TESTER1 , aws_secret_access_key= SECRET_KEY_OF_TESTER1 , endpoint_url=<IAM URL>, region_name='' ) oidc_response = iam_client.create_open_id_connect_provider( Url=<URL of the OpenID Connect Provider>, ClientIDList=[ <Client id registered with the IDP> ], ThumbprintList=[ <IDP THUMBPRINT> ] ) policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"Federated\":\[\"arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/demo\"\]\},\"Action\":\[\"sts:AssumeRoleWithWebIdentity\"\],\"Condition\":\{\"StringEquals\":\{\"localhost:8080/auth/realms/demo:app_id\":\"customer-portal\"\}\}\}\]\}" role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, Path='/', RoleName='S3Access', ) role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}" response = iam_client.put_role_policy( RoleName='S3Access', PolicyName='Policy1', PolicyDocument=role_policy ) sts_client = boto3.client('sts', aws_access_key_id= ACCESS_KEY_OF_TESTER2 , aws_secret_access_key= SECRET_KEY_OF_TESTER2 , endpoint_url=<STS URL>, region_name='', ) response = sts_client.assume_role_with_web_identity( RoleArn=role_response['Role']['Arn'], RoleSessionName='Bob', DurationSeconds=3600, WebIdentityToken=<Web Token> ) s3client = boto3.client('s3', aws_access_key_id = response['Credentials']['AccessKeyId'], aws_secret_access_key = response['Credentials']['SecretAccessKey'], aws_session_token = response['Credentials']['SessionToken'], endpoint_url=<S3 URL>, region_name='',) bucket_name = 'my-bucket' s3bucket = s3client.create_bucket(Bucket=bucket_name) resp = s3client.list_buckets()