From 05b175ce3965e98b6b20fd4db53d85d328ebd391 Mon Sep 17 00:00:00 2001 From: Rajat Garg Date: Thu, 12 May 2016 17:08:54 +0530 Subject: [PATCH 01/60] Final API logging of incoming request and outgoing response --- src/rgw/rgw_civetweb_log.cc | 4 ++-- src/rgw/rgw_main.cc | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rgw/rgw_civetweb_log.cc b/src/rgw/rgw_civetweb_log.cc index e2d8dd9ad015d..a3bde833561f6 100644 --- a/src/rgw/rgw_civetweb_log.cc +++ b/src/rgw/rgw_civetweb_log.cc @@ -7,12 +7,12 @@ int rgw_civetweb_log_callback(const struct mg_connection *conn, const char *buf) { - dout(0) << "civetweb: " << (void *)conn << ": " << buf << dendl; + dout(0) << "DSS API LOGGING: " << (void *)conn << ": " << buf << dendl; return 0; } int rgw_civetweb_log_access_callback(const struct mg_connection *conn, const char *buf) { - dout(1) << "civetweb: " << (void *)conn << ": " << buf << dendl; + dout(1) << "DSS API LOGGING: " << (void *)conn << ": " << buf << dendl; return 0; } diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 487835cd56afd..4e8f7954bf237 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -560,7 +560,8 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC s->req_id = store->unique_id(req->id); s->trans_id = store->unique_trans_id(req->id); - dout(1) << "DSS API LOGGING: ====== starting new request trans=" << s->trans_id.c_str() << " =====" << dendl; + dout(1) << "DSS API LOGGING: Request-Id: "<< s->req_id.c_str() << dendl; + dout(1) << "DSS INFO: Transaction Id: " << s->trans_id.c_str() << dendl; //req->log_format(s, "initializing for trans_id = %s", s->trans_id.c_str()); /* Logic for checking whether the request is token based or signature based */ @@ -694,7 +695,6 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC handler->put_op(op); rest->put_handler(handler); utime_t req_serve_time = req->time_elapsed(); - dout(1) << "DSS API LOGGING: ====== req done trans=" << s->trans_id.c_str() << " http_status=" << http_ret << " req_serving_time= " << req_serve_time << " ======" << dendl; return (ret < 0 ? ret : s->err.ret); @@ -769,6 +769,7 @@ static int civetweb_callback(struct mg_connection *conn) { * method required is EC2 signature or tokens. * While there can be at most 100 header fields in a HTTP request, * http_headers is an array of size 64 elements inside civetweb */ + dout(1) << "DSS API LOGGING: ====== starting new request ======" << dendl; dout(1) << "DSS INFO: Num headers is: " << req_info->num_headers << dendl; for (int i = 0; i < req_info->num_headers; i++) { @@ -783,8 +784,7 @@ static int civetweb_callback(struct mg_connection *conn) { break; } - dout(1) << "DSS INFO: CIVETWEB HEADER NAME: " << name_str << dendl; - dout(1) << "DSS INFO: CIVETWEB HEADER VALUE: " << value_str << dendl; + dout(1) << "DSS API LOGGING: " << name_str << " : "<< value_str << dendl; /* if (name_str.compare("X-Auth-Token") == 0) { From a1ff95382713688a98d81ef8c147c664eb578679 Mon Sep 17 00:00:00 2001 From: Rajat Garg Date: Fri, 13 May 2016 11:23:47 +0530 Subject: [PATCH 02/60] Final API logging of incoming request and outgoing response --- src/rgw/rgw_civetweb.cc | 22 ++++++++++++++++++++++ src/rgw/rgw_civetweb_log.cc | 4 ++-- src/rgw/rgw_main.cc | 18 ++++++++++++++---- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/rgw/rgw_civetweb.cc b/src/rgw/rgw_civetweb.cc index 2ae3b799166ee..e9f500e5aebc9 100644 --- a/src/rgw/rgw_civetweb.cc +++ b/src/rgw/rgw_civetweb.cc @@ -204,6 +204,28 @@ int RGWMongoose::complete_header() header_data.append("\r\n"); sent_header = true; + string response_headers; + header_data.copy(0, header_data.length(), response_headers); + + vector vec_headers; + char *headers = strdup(response_headers.c_str()); + char *savedptr; + char *p = strtok_r(headers, "\r\n", &savedptr); + + while (p) { + string tok = p; + vec_headers.push_back(tok); + p = strtok_r(NULL, "\r\n", &savedptr); + } + + dout(1) << "DSS API LOGGING: ===Printing headers sent in response:===" << dendl; + for (int i = 0; i< vec_headers.size(); i++) { + dout(1) << "DSS API LOGGING: " <req_id = store->unique_id(req->id); s->trans_id = store->unique_trans_id(req->id); - dout(1) << "DSS API LOGGING: Request-Id: "<< s->req_id.c_str() << dendl; - dout(1) << "DSS INFO: Transaction Id: " << s->trans_id.c_str() << dendl; + dout(1) << "DSS API LOGGING: Request-Id: "<< s->trans_id.c_str() << dendl; //req->log_format(s, "initializing for trans_id = %s", s->trans_id.c_str()); /* Logic for checking whether the request is token based or signature based */ @@ -753,7 +752,6 @@ void RGWLoadGenProcess::handle_request(RGWRequest *r) static int civetweb_callback(struct mg_connection *conn) { struct mg_request_info *req_info = mg_get_request_info(conn); RGWProcessEnv *pe = static_cast(req_info->user_data); - RGWRados *store = pe->store; RGWREST *rest = pe->rest; OpsLogSocket *olog = pe->olog; @@ -771,7 +769,19 @@ static int civetweb_callback(struct mg_connection *conn) { * http_headers is an array of size 64 elements inside civetweb */ dout(1) << "DSS API LOGGING: ====== starting new request ======" << dendl; - dout(1) << "DSS INFO: Num headers is: " << req_info->num_headers << dendl; + dout(1) << "DSS INFO: Printing received headers: Total number of received headers are: " << req_info->num_headers << dendl; + + string req_str; + + req_str.append(" "); + req_str.append(req_info->request_method); + req_str.append(" "); + req_str.append(req_info->uri); + req_str.append(" HTTP/"); + req_str.append(req_info->http_version); + + dout(1) << "DSS API LOGGING:" << req_str.c_str() << dendl; + for (int i = 0; i < req_info->num_headers; i++) { if ((req_info->http_headers[i]).name != NULL) { string name_str((req_info->http_headers[i]).name); From a1fdca76feba3a5eb05cdc03e75057f1f099ac6e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 7 Jun 2016 12:40:45 +0530 Subject: [PATCH 03/60] Squashed commit of the following: commit eb075aae5ce6fd8b0170db00058cc38bb9bdc557 Author: root Date: Wed May 25 18:43:05 2016 +0530 Adding back comment commit ccbefbe6e5b9b8954ccb4eb16aa046e48eda90cc Author: root Date: Wed May 25 16:33:46 2016 +0530 JOS-98 : RGW crashes when DELETE is done concurrently - providing threading functions for openssl to support multithreading --- configure.ac | 8 +++++ src/rgw/rgw_main.cc | 73 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 969baed8357db..8c7a65a6d431b 100644 --- a/configure.ac +++ b/configure.ac @@ -315,6 +315,13 @@ AC_SUBST(KEYUTILS_LIB) AC_CHECK_LIB([m], [pow], [true], AC_MSG_FAILURE([libm not found])) AC_CHECK_FUNCS([syncfs], AC_DEFINE([HAVE_SYS_SYNCFS], [1], [we have syncfs]), []) + +# since we are now using openssl and need to provide the additional functions +# for thread safety, we also need to link to crypto library (-lcrypto) +AC_CHECK_LIB(crypto, main, [], + AC_MSG_FAILURE(["Crypto system library not found."])) + + # Find some crypto library for us to use, while letting user to decide which one to use. AC_ARG_WITH([cryptopp], [AS_HELP_STRING([--with-cryptopp], [Use cryptographic functions from cryptopp])], @@ -872,6 +879,7 @@ AC_CHECK_LIB(boost_system-mt, main, [], [AC_CHECK_LIB(boost_system, main, [], AC_MSG_NOTICE(["Boost system library not found."]))]) + # Find the right boost_thread library. BOOST_THREAD_LIBS="" saved_LIBS="${LIBS}" diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 3d095ede6ad48..20ee8734ff2d5 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -1,4 +1,4 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab #include @@ -71,8 +71,73 @@ #include "include/types.h" #include "common/BackTrace.h" +#include +#include +#include + + #define dout_subsys ceph_subsys_rgw + +#define MUTEX_TYPE pthread_mutex_t +#define MUTEX_SETUP(x) pthread_mutex_init(&(x), NULL) +#define MUTEX_CLEANUP(x) pthread_mutex_destroy(&(x)) +#define MUTEX_LOCK(x) pthread_mutex_lock(&(x)) +#define MUTEX_UNLOCK(x) pthread_mutex_unlock(&(x)) +#define THREAD_ID pthread_self() + + + + +/* This array will store all of the mutexes available to OpenSSL. */ +static MUTEX_TYPE *mutex_buf= NULL; + +static void openssl_locking_function(int mode, int n, const char * file, int line) +{ + if(mode & CRYPTO_LOCK) + MUTEX_LOCK(mutex_buf[n]); + else + MUTEX_UNLOCK(mutex_buf[n]); +} + +static unsigned long openssl_id_function(void) +{ + return ((unsigned long)THREAD_ID); +} + +int openssl_thread_setup(void) +{ + int i; + + mutex_buf = (MUTEX_TYPE *) malloc(CRYPTO_num_locks() * sizeof(MUTEX_TYPE)); + if(!mutex_buf) + return 0; + for(i = 0; i < CRYPTO_num_locks(); i++) + MUTEX_SETUP(mutex_buf[i]); + CRYPTO_set_id_callback(openssl_id_function); + CRYPTO_set_locking_callback(openssl_locking_function); + return 1; +} + +int openssl_thread_cleanup(void) +{ + int i; + + if(!mutex_buf) + return 0; + CRYPTO_set_id_callback(NULL); + CRYPTO_set_locking_callback(NULL); + for(i = 0; i < CRYPTO_num_locks(); i++) + MUTEX_CLEANUP(mutex_buf[i]); + free(mutex_buf); + mutex_buf = NULL; + return 1; +} + + + + + using namespace std; static sig_t sighandler_alrm; @@ -1174,6 +1239,9 @@ int main(int argc, const char **argv) rgw_init_resolver(); curl_global_init(CURL_GLOBAL_ALL); + + /* setup openssl multi-threading functions */ + openssl_thread_setup(); FCGX_Init(); @@ -1367,6 +1435,9 @@ int main(int argc, const char **argv) rgw_shutdown_resolver(); curl_global_cleanup(); + /* cleanup openssl multi-threading functions */ + openssl_thread_cleanup(); + rgw_perf_stop(g_ceph_context); dout(1) << "final shutdown" << dendl; From b0ce5a6e4956fdad9dc04cc77655eb960f8b809c Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Thu, 9 Jun 2016 16:30:40 +0000 Subject: [PATCH 04/60] Initial rename op functions with dummy returns --- src/rgw/rgw_op.h | 11 +++++++++++ src/rgw/rgw_rest.h | 6 ++++++ src/rgw/rgw_rest_s3.cc | 10 ++++++++-- src/rgw/rgw_rest_s3.h | 13 +++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index f0354ab36237d..a38c0190b4c5c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -565,6 +565,17 @@ class RGWDeleteObj : public RGWOp { virtual uint32_t op_mask() { return RGW_OP_TYPE_DELETE; } }; +class RGWRenameObj : public RGWOp { + //<<<< + public: + RGWRenameObj() {} + ~RGWRenameObj() {} + int verify_permission() { return 0; } + void pre_exec() { int x; } + void execute() { int x; } + const string name() { return "Rename_obj"; } +}; + class RGWCopyObj : public RGWOp { protected: RGWAccessControlPolicy dest_policy; diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 60acbf700bb09..532a8a182683f 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -188,6 +188,12 @@ class RGWCopyObj_ObjStore : public RGWCopyObj { ~RGWCopyObj_ObjStore() {} }; +class RGWRenameObj_ObjStore : public RGWRenameObj { +public: + RGWRenameObj_ObjStore() {} + ~RGWRenameObj_ObjStore() {} +}; + class RGWGetACLs_ObjStore : public RGWGetACLs { public: RGWGetACLs_ObjStore() {} diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 6b903388b02d4..b6cd5bd4e60b9 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1238,7 +1238,7 @@ int RGWPostObj_ObjStore_S3::get_policy() return -EACCES; } user_info.user_id = keystone_validator.response.token.tenant.id; - user_info.display_name = keystone_validator.response.token.tenant.id; //<<<<<< DSS needs tenant.name + user_info.display_name = keystone_validator.response.token.tenant.id; /* try to store user if it not already exists */ if (rgw_get_user_info_by_uid(store, keystone_validator.response.token.tenant.id, user_info) < 0) { int ret = rgw_store_user_info(store, user_info, NULL, NULL, 0, true); @@ -2120,6 +2120,12 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_put() if (is_acl_op()) { return new RGWPutACLs_ObjStore_S3; } + if (is_rename_op()) { + //<<<<<<<< + return new RGWRenameObj_ObjStore_S3; + //dout(0) << "-------------- The value of new name is " << s->info.args.get("newname") << dendl; + //dout(0) << "=========== I am getting rename op =========" << dendl; + } if (!s->copy_source) return new RGWPutObj_ObjStore_S3; else @@ -2397,7 +2403,7 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, || is_url_token || is_infini_url_token); - /* Infinite URLs should only be used for GET <<<<<< Needs discussion */ + /* Infinite URLs should only be used for GET */ if (is_infini_url_token && !( (localAction.compare("ListBucket") == 0) || (localAction.compare("GetObject") == 0) || diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index f6077cf81aa4c..c4e0a3bbbb504 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -187,6 +187,16 @@ class RGWCopyObj_ObjStore_S3 : public RGWCopyObj_ObjStore { void send_response(); }; +class RGWRenameObj_ObjStore_S3 : public RGWRenameObj_ObjStore { + //<<<< + public: + void send_response() { + int y; + } + RGWRenameObj_ObjStore_S3() {} + ~RGWRenameObj_ObjStore_S3() {} +}; + class RGWGetACLs_ObjStore_S3 : public RGWGetACLs_ObjStore { public: RGWGetACLs_ObjStore_S3() {} @@ -446,6 +456,9 @@ class RGWHandler_ObjStore_Obj_S3 : public RGWHandler_ObjStore_S3 { bool is_obj_update_op() { return is_acl_op(); } + bool is_rename_op() { + return s->info.args.exists("newname"); + } RGWOp *get_obj_op(bool get_data); RGWOp *op_get(); From 74c84616c86df8c20858d1919deeeaca56099160 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Tue, 14 Jun 2016 15:40:28 +0000 Subject: [PATCH 05/60] send response handled --- src/rgw/rgw_op.cc | 21 +++++++++++++++++++++ src/rgw/rgw_op.h | 12 ++++++------ src/rgw/rgw_rest_s3.cc | 11 ++++++++++- src/rgw/rgw_rest_s3.h | 6 ++---- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index b4b4c8a3c48e5..0603569b79226 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3514,3 +3514,24 @@ void RGWHandler::put_op(RGWOp *op) delete op; } + + +/* Object Rename operation */ + +int RGWRenameObj::verify_permission() +{ + // This function used to check ACLs in legacy code + // DSS does not require this. + return 0; +} + +void RGWRenameObj::pre_exec() +{ + rgw_bucket_object_pre_exec(s); +} + +void RGWRenameObj::execute() +{ + ret = 0; + return; +} diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index a38c0190b4c5c..54591a4971284 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -566,14 +566,14 @@ class RGWDeleteObj : public RGWOp { }; class RGWRenameObj : public RGWOp { - //<<<< public: - RGWRenameObj() {} + int ret; + RGWRenameObj() : ret(0) {} ~RGWRenameObj() {} - int verify_permission() { return 0; } - void pre_exec() { int x; } - void execute() { int x; } - const string name() { return "Rename_obj"; } + int verify_permission(); + void pre_exec(); + void execute(); + virtual const string name() { return "Rename_obj"; } }; class RGWCopyObj : public RGWOp { diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index b6cd5bd4e60b9..23bb9e9f929e1 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1619,6 +1619,16 @@ int RGWPutACLs_ObjStore_S3::get_policy_from_state(RGWRados *store, struct req_st return 0; } +void RGWRenameObj_ObjStore_S3::send_response() +{ + if (ret) + set_req_state_err(s, ret); + dump_errno(s); + end_header(s, this, "application/xml"); + dump_start(s); + return; +} + void RGWPutACLs_ObjStore_S3::send_response() { if (ret) @@ -2121,7 +2131,6 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_put() return new RGWPutACLs_ObjStore_S3; } if (is_rename_op()) { - //<<<<<<<< return new RGWRenameObj_ObjStore_S3; //dout(0) << "-------------- The value of new name is " << s->info.args.get("newname") << dendl; //dout(0) << "=========== I am getting rename op =========" << dendl; diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index c4e0a3bbbb504..756af24aefd94 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -188,13 +188,11 @@ class RGWCopyObj_ObjStore_S3 : public RGWCopyObj_ObjStore { }; class RGWRenameObj_ObjStore_S3 : public RGWRenameObj_ObjStore { - //<<<< public: - void send_response() { - int y; - } RGWRenameObj_ObjStore_S3() {} ~RGWRenameObj_ObjStore_S3() {} + + void send_response(); }; class RGWGetACLs_ObjStore_S3 : public RGWGetACLs_ObjStore { From 805f9904572a9bbe1a368056ffee157f286a169d Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Fri, 17 Jun 2016 11:16:59 +0000 Subject: [PATCH 06/60] Calling copy and delete --- src/rgw/rgw_op.cc | 35 ++++++++++++++++++++++++++++++++++- src/rgw/rgw_rest_s3.cc | 27 ++++++++++++++++++++++++--- src/rgw/rgw_rest_s3.h | 4 ++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 0603569b79226..c02c2488eaa08 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -24,7 +24,7 @@ #include "rgw_multi_del.h" #include "rgw_cors.h" #include "rgw_cors_s3.h" - +#include "rgw_rest_s3.h" #include "rgw_client_io.h" #define dout_subsys ceph_subsys_rgw @@ -3533,5 +3533,38 @@ void RGWRenameObj::pre_exec() void RGWRenameObj::execute() { ret = 0; + string orig_object = (s->object).name; + (s->object).name = s->info.args.get("newname"); + string copysource = s->bucket_name_str; + copysource.append("/"); + copysource.append(orig_object); + ldout(s->cct, 0) << "DSS INFO: Converting to copy request. s->object: " + << (s->object).name << ". Copy source: " << copysource << dendl; + s->info.env->set("HTTP_X_JCS_COPY_SOURCE", copysource.c_str()); + s->info.env->set("HTTP_X_JCS_METADATA_DIRECTIVE", "COPY"); + + RGWCopyObj_ObjStore_S3* copy_op = new RGWCopyObj_ObjStore_S3; + copy_op->dss_set_req_state(s); + copy_op->dss_set_store(store); + copy_op->init_processing(); + s->system_request = true; + copy_op->pre_exec(); + copy_op->execute(); + copy_op->complete(); + ldout(s->cct, 0) << "DSS INFO: Rename op copy done" << dendl; + +/* s->system_request = false; + (s->object).name = orig_object; + ldout(s->cct, 0) << "DSS INFO: Rearraging things to continue with delete. s->object: " + << (s->object).name << dendl; + RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; + del_op->dss_set_req_state(s); + del_op->dss_set_store(store); + del_op->init_processing(); + s->system_request = true; + del_op->pre_exec(); + del_op->execute(); + ldout(s->cct, 0) << "DSS INFO: Rename op delete performed" << dendl;*/ + s->system_request = false; return; } diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 23bb9e9f929e1..a19b79ceb89fc 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1468,6 +1468,15 @@ void RGWPostObj_ObjStore_S3::send_response() rgw_flush_formatter_and_reset(s, s->formatter); } +void RGWDeleteObj_ObjStore_S3::dss_set_req_state(req_state* orig_s) +{ + s = orig_s; +} + +void RGWDeleteObj_ObjStore_S3::dss_set_store(RGWRados* orig_store) +{ + store = orig_store; +} void RGWDeleteObj_ObjStore_S3::send_response() { @@ -1502,6 +1511,16 @@ int RGWCopyObj_ObjStore_S3::init_dest_policy() return 0; } +void RGWCopyObj_ObjStore_S3::dss_set_req_state(req_state* orig_s) +{ + s = orig_s; +} + +void RGWCopyObj_ObjStore_S3::dss_set_store(RGWRados* orig_store) +{ + store = orig_store; +} + int RGWCopyObj_ObjStore_S3::get_params() { if_mod = s->info.env->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE"); @@ -2132,8 +2151,6 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_put() } if (is_rename_op()) { return new RGWRenameObj_ObjStore_S3; - //dout(0) << "-------------- The value of new name is " << s->info.args.get("newname") << dendl; - //dout(0) << "=========== I am getting rename op =========" << dendl; } if (!s->copy_source) return new RGWPutObj_ObjStore_S3; @@ -2823,7 +2840,11 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) if (s != NULL) { resource_object_name = s->object.name; } - dout(1) << "DSS API LOGGING: Action="<< resource_info.getAction() <<" Resource="<< resource_info.getResourceName() << " Tenant=" << resource_info.getTenantName() << " Object=" << resource_object_name << dendl; + dout(1) << "DSS API LOGGING: Action=" + << resource_info.getAction() + << " Resource="<< resource_info.getResourceName() + << " Tenant=" << resource_info.getTenantName() + << " Object=" << resource_object_name << dendl; if (isTokenBasedAuth) { keystone_result = keystone_validator.validate_request(resource_info.getAction(), diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 756af24aefd94..1f1729ce12738 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -173,6 +173,8 @@ class RGWDeleteObj_ObjStore_S3 : public RGWDeleteObj_ObjStore { ~RGWDeleteObj_ObjStore_S3() {} void send_response(); + void dss_set_req_state(req_state* orig_s); + void dss_set_store(RGWRados* orig_store); }; class RGWCopyObj_ObjStore_S3 : public RGWCopyObj_ObjStore { @@ -185,6 +187,8 @@ class RGWCopyObj_ObjStore_S3 : public RGWCopyObj_ObjStore { int get_params(); void send_partial_response(off_t ofs); void send_response(); + void dss_set_req_state(req_state* orig_s); + void dss_set_store(RGWRados* orig_store); }; class RGWRenameObj_ObjStore_S3 : public RGWRenameObj_ObjStore { From e4725b64a10604569bcdeae381ed5a56b8a0d492 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 15 Jan 2016 15:05:22 -0800 Subject: [PATCH 07/60] Revert "common/*Formatters: Split Formatters" This reverts commit 2b21e3c595c23f409485b8c391d1bd579b13b8e7. Conflicts: src/common/Formatter.cc src/common/JSONFormatter.h src/common/Makefile.am src/common/TableFormatter.h src/common/XMLFormatter.cc src/common/XMLFormatter.h src/rgw/rgw_rest.cc src/test/formatter.cc Conflicts: ceph-object-corpus src/mds/MDSDaemon.cc src/mds/MDSRank.cc src/rocksdb --- src/common/Formatter.cc | 753 ++++++++++++++++++++++++- src/common/Formatter.h | 123 ++++ src/common/JSONFormatter.cc | 226 -------- src/common/Makefile.am | 6 - src/common/TableFormatter.cc | 376 ------------ src/common/admin_socket.cc | 1 - src/mon/ConfigKeyService.cc | 1 - src/mon/Monitor.cc | 2 - src/mon/Paxos.cc | 1 - src/mon/Paxos.h | 3 - src/os/FileJournal.cc | 2 - src/os/FileStore.cc | 2 - src/os/FileStore.h | 2 - src/os/KeyValueStore.cc | 2 - src/os/KeyValueStore.h | 1 - src/os/MemStore.cc | 2 - src/osd/OSD.cc | 2 - src/osd/OSDMap.cc | 1 - src/rbd.cc | 2 - src/rgw/rgw_admin.cc | 2 - src/rgw/rgw_cors_s3.h | 1 - src/rgw/rgw_jsonparser.cc | 1 - src/rgw/rgw_log.cc | 1 - src/rgw/rgw_rest.cc | 2 - src/rgw/rgw_rest_s3.cc | 1 - src/rgw/rgw_swift.cc | 1 - src/rgw/rgw_swift_auth.cc | 1 - src/test/bench/small_io_bench.cc | 1 - src/test/bench/small_io_bench_dumb.cc | 1 - src/test/bench/small_io_bench_fs.cc | 1 - src/test/bench/small_io_bench_rbd.cc | 1 - src/test/bench/tp_bench.cc | 1 - src/test/common/test_tableformatter.cc | 2 +- src/test/crush/crush.cc | 1 - src/test/encoding/ceph_dencoder.cc | 1 - src/test/formatter.cc | 3 +- src/test/kv_store_bench.h | 1 - src/test/mon/test_mon_workloadgen.cc | 1 - src/test/test_rgw_admin_meta.cc | 2 - src/tools/ceph-client-debug.cc | 2 +- src/tools/ceph_monstore_tool.cc | 2 +- src/tools/ceph_objectstore_tool.cc | 1 - src/tools/cephfs/EventOutput.cc | 1 - src/tools/cephfs/JournalTool.cc | 1 - src/tools/cephfs/TableTool.cc | 2 - src/tools/rados/rados.cc | 2 - 46 files changed, 875 insertions(+), 670 deletions(-) delete mode 100644 src/common/JSONFormatter.cc delete mode 100644 src/common/TableFormatter.cc diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index 8bbb25b6c4d89..345f95df66dca 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -18,10 +18,6 @@ #include "assert.h" #include "Formatter.h" -#include "JSONFormatter.h" -#include "TableFormatter.h" -#include "XMLFormatter.h" -#include "HTMLFormatter.h" #include "common/escape.h" #include @@ -122,4 +118,751 @@ void Formatter::dump_format_unquoted(const char *name, const char *fmt, ...) va_end(ap); } -} // namespace ceph +// ----------------------- + +JSONFormatter::JSONFormatter(bool p) +: m_pretty(p), m_is_pending_string(false) +{ + reset(); +} + +void JSONFormatter::flush(std::ostream& os) +{ + finish_pending_string(); + os << m_ss.str(); + if (m_pretty) + os << "\n"; + m_ss.clear(); + m_ss.str(""); +} + +void JSONFormatter::reset() +{ + m_stack.clear(); + m_ss.clear(); + m_ss.str(""); + m_pending_string.clear(); + m_pending_string.str(""); +} + +void JSONFormatter::print_comma(json_formatter_stack_entry_d& entry) +{ + if (entry.size) { + if (m_pretty) { + m_ss << ",\n"; + for (unsigned i = 1; i < m_stack.size(); i++) + m_ss << " "; + } else { + m_ss << ","; + } + } else if (m_pretty) { + m_ss << "\n"; + for (unsigned i = 1; i < m_stack.size(); i++) + m_ss << " "; + } + if (m_pretty && entry.is_array) + m_ss << " "; +} + +void JSONFormatter::print_quoted_string(const std::string& s) +{ + int len = escape_json_attr_len(s.c_str(), s.size()); + char escaped[len]; + escape_json_attr(s.c_str(), s.size(), escaped); + m_ss << '\"' << escaped << '\"'; +} + +void JSONFormatter::print_name(const char *name) +{ + finish_pending_string(); + if (m_stack.empty()) + return; + struct json_formatter_stack_entry_d& entry = m_stack.back(); + print_comma(entry); + if (!entry.is_array) { + if (m_pretty) { + m_ss << " "; + } + m_ss << "\"" << name << "\""; + if (m_pretty) + m_ss << ": "; + else + m_ss << ':'; + } + ++entry.size; +} + +void JSONFormatter::open_section(const char *name, bool is_array) +{ + print_name(name); + if (is_array) + m_ss << '['; + else + m_ss << '{'; + + json_formatter_stack_entry_d n; + n.is_array = is_array; + m_stack.push_back(n); +} + +void JSONFormatter::open_array_section(const char *name) +{ + open_section(name, true); +} + +void JSONFormatter::open_array_section_in_ns(const char *name, const char *ns) +{ + std::ostringstream oss; + oss << name << " " << ns; + open_section(oss.str().c_str(), true); +} + +void JSONFormatter::open_object_section(const char *name) +{ + open_section(name, false); +} + +void JSONFormatter::open_object_section_in_ns(const char *name, const char *ns) +{ + std::ostringstream oss; + oss << name << " " << ns; + open_section(oss.str().c_str(), false); +} + +void JSONFormatter::close_section() +{ + assert(!m_stack.empty()); + finish_pending_string(); + + struct json_formatter_stack_entry_d& entry = m_stack.back(); + if (m_pretty && entry.size) { + m_ss << "\n"; + for (unsigned i = 1; i < m_stack.size(); i++) + m_ss << " "; + } + m_ss << (entry.is_array ? ']' : '}'); + m_stack.pop_back(); +} + +void JSONFormatter::finish_pending_string() +{ + if (m_is_pending_string) { + print_quoted_string(m_pending_string.str()); + m_pending_string.str(std::string()); + m_is_pending_string = false; + } +} + +void JSONFormatter::dump_unsigned(const char *name, uint64_t u) +{ + print_name(name); + m_ss << u; +} + +void JSONFormatter::dump_int(const char *name, int64_t s) +{ + print_name(name); + m_ss << s; +} + +void JSONFormatter::dump_float(const char *name, double d) +{ + print_name(name); + char foo[30]; + snprintf(foo, sizeof(foo), "%lf", d); + m_ss << foo; +} + +void JSONFormatter::dump_string(const char *name, const std::string& s) +{ + print_name(name); + print_quoted_string(s); +} + +std::ostream& JSONFormatter::dump_stream(const char *name) +{ + print_name(name); + m_is_pending_string = true; + return m_pending_string; +} + +void JSONFormatter::dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap) +{ + char buf[LARGE_SIZE]; + vsnprintf(buf, LARGE_SIZE, fmt, ap); + + print_name(name); + if (quoted) { + print_quoted_string(std::string(buf)); + } else { + m_ss << std::string(buf); + } +} + +int JSONFormatter::get_len() const +{ + return m_ss.str().size(); +} + +void JSONFormatter::write_raw_data(const char *data) +{ + m_ss << data; +} + +const char *XMLFormatter::XML_1_DTD = + ""; + +XMLFormatter::XMLFormatter(bool pretty) +: m_pretty(pretty) +{ + reset(); +} + +void XMLFormatter::flush(std::ostream& os) +{ + finish_pending_string(); + os << m_ss.str(); + if (m_pretty) + os << "\n"; + m_ss.clear(); + m_ss.str(""); +} + +void XMLFormatter::reset() +{ + m_ss.clear(); + m_ss.str(""); + m_pending_string.clear(); + m_pending_string.str(""); + m_sections.clear(); + m_pending_string_name.clear(); +} + +void XMLFormatter::open_object_section(const char *name) +{ + open_section_in_ns(name, NULL, NULL); +} + +void XMLFormatter::open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs) +{ + open_section_in_ns(name, NULL, &attrs); +} + +void XMLFormatter::open_object_section_in_ns(const char *name, const char *ns) +{ + open_section_in_ns(name, ns, NULL); +} + +void XMLFormatter::open_array_section(const char *name) +{ + open_section_in_ns(name, NULL, NULL); +} + +void XMLFormatter::open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs) +{ + open_section_in_ns(name, NULL, &attrs); +} + +void XMLFormatter::open_array_section_in_ns(const char *name, const char *ns) +{ + open_section_in_ns(name, ns, NULL); +} + +void XMLFormatter::close_section() +{ + assert(!m_sections.empty()); + finish_pending_string(); + + std::string section = m_sections.back(); + m_sections.pop_back(); + print_spaces(); + m_ss << ""; + if (m_pretty) + m_ss << "\n"; +} + +void XMLFormatter::dump_unsigned(const char *name, uint64_t u) +{ + std::string e(name); + print_spaces(); + m_ss << "<" << e << ">" << u << ""; + if (m_pretty) + m_ss << "\n"; +} + +void XMLFormatter::dump_int(const char *name, int64_t u) +{ + std::string e(name); + print_spaces(); + m_ss << "<" << e << ">" << u << ""; + if (m_pretty) + m_ss << "\n"; +} + +void XMLFormatter::dump_float(const char *name, double d) +{ + std::string e(name); + print_spaces(); + m_ss << "<" << e << ">" << d << ""; + if (m_pretty) + m_ss << "\n"; +} + +void XMLFormatter::dump_string(const char *name, const std::string& s) +{ + std::string e(name); + print_spaces(); + m_ss << "<" << e << ">" << escape_xml_str(s.c_str()) << ""; + if (m_pretty) + m_ss << "\n"; +} + +void XMLFormatter::dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) +{ + std::string e(name); + std::string attrs_str; + get_attrs_str(&attrs, attrs_str); + print_spaces(); + m_ss << "<" << e << attrs_str << ">" << escape_xml_str(s.c_str()) << ""; + if (m_pretty) + m_ss << "\n"; +} + +std::ostream& XMLFormatter::dump_stream(const char *name) +{ + print_spaces(); + m_pending_string_name = name; + m_ss << "<" << m_pending_string_name << ">"; + return m_pending_string; +} + +void XMLFormatter::dump_format_va(const char* name, const char *ns, bool quoted, const char *fmt, va_list ap) +{ + char buf[LARGE_SIZE]; + vsnprintf(buf, LARGE_SIZE, fmt, ap); + + std::string e(name); + print_spaces(); + if (ns) { + m_ss << "<" << e << " xmlns=\"" << ns << "\">" << buf << ""; + } else { + m_ss << "<" << e << ">" << escape_xml_str(buf) << ""; + } + + if (m_pretty) + m_ss << "\n"; +} + +int XMLFormatter::get_len() const +{ + return m_ss.str().size(); +} + +void XMLFormatter::write_raw_data(const char *data) +{ + m_ss << data; +} + +void XMLFormatter::get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str) +{ + std::stringstream attrs_ss; + + for (std::list >::const_iterator iter = attrs->attrs.begin(); + iter != attrs->attrs.end(); ++iter) { + std::pair p = *iter; + attrs_ss << " " << p.first << "=" << "\"" << p.second << "\""; + } + + attrs_str = attrs_ss.str(); +} + +void XMLFormatter::open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs) +{ + print_spaces(); + std::string attrs_str; + + if (attrs) { + get_attrs_str(attrs, attrs_str); + } + + if (ns) { + m_ss << "<" << name << attrs_str << " xmlns=\"" << ns << "\">"; + } else { + m_ss << "<" << name << attrs_str << ">"; + } + if (m_pretty) + m_ss << "\n"; + m_sections.push_back(name); +} + +void XMLFormatter::finish_pending_string() +{ + if (!m_pending_string_name.empty()) { + m_ss << escape_xml_str(m_pending_string.str().c_str()) + << ""; + m_pending_string_name.clear(); + m_pending_string.str(std::string()); + if (m_pretty) { + m_ss << "\n"; + } + } +} + +void XMLFormatter::print_spaces() +{ + finish_pending_string(); + if (m_pretty) { + std::string spaces(m_sections.size(), ' '); + m_ss << spaces; + } +} + +std::string XMLFormatter::escape_xml_str(const char *str) +{ + int len = escape_xml_attr_len(str); + std::vector escaped(len, '\0'); + escape_xml_attr(str, &escaped[0]); + return std::string(&escaped[0]); +} + +TableFormatter::TableFormatter(bool keyval) : m_keyval(keyval) +{ + reset(); +} + +void TableFormatter::flush(std::ostream& os) +{ + finish_pending_string(); + std::vector column_size = m_column_size; + std::vector column_name = m_column_name; + + std::set need_header_set; + + // auto-sizing columns + for (size_t i = 0; i < m_vec.size(); i++) { + for (size_t j = 0; j < m_vec[i].size(); j++) { + column_size.resize(m_vec[i].size()); + column_name.resize(m_vec[i].size()); + if (i > 0) { + if (m_vec[i - 1][j] != m_vec[i][j]) { + // changing row labels require to show the header + need_header_set.insert(i); + column_name[i] = m_vec[i][j].first; + } + } else { + column_name[i] = m_vec[i][j].first; + } + + if (m_vec[i][j].second.length() > column_size[j]) + column_size[j] = m_vec[i][j].second.length(); + if (m_vec[i][j].first.length() > column_size[j]) + column_size[j] = m_vec[i][j].first.length(); + } + } + + bool need_header = false; + if ((column_size.size() == m_column_size.size())) { + for (size_t i = 0; i < column_size.size(); i++) { + if (column_size[i] != m_column_size[i]) { + need_header = true; + break; + } + } + } else { + need_header = true; + } + + if (need_header) { + // first row always needs a header if there wasn't one before + need_header_set.insert(0); + } + + m_column_size = column_size; + for (size_t i = 0; i < m_vec.size(); i++) { + if (i == 0) { + if (need_header_set.count(i)) { + // print the header + if (!m_keyval) { + os << "+"; + for (size_t j = 0; j < m_vec[i].size(); j++) { + for (size_t v = 0; v < m_column_size[j] + 3; v++) + os << "-"; + os << "+"; + } + os << "\n"; + os << "|"; + + for (size_t j = 0; j < m_vec[i].size(); j++) { + os << " "; + std::stringstream fs; + fs << boost::format("%%-%is") % (m_column_size[j] + 2); + os << boost::format(fs.str()) % m_vec[i][j].first; + os << "|"; + } + os << "\n"; + os << "+"; + for (size_t j = 0; j < m_vec[i].size(); j++) { + for (size_t v = 0; v < m_column_size[j] + 3; v++) + os << "-"; + os << "+"; + } + os << "\n"; + } + } + } + // print body + if (!m_keyval) + os << "|"; + for (size_t j = 0; j < m_vec[i].size(); j++) { + if (!m_keyval) + os << " "; + std::stringstream fs; + + if (m_keyval) { + os << "key::"; + os << m_vec[i][j].first; + os << "="; + os << "\""; + os << m_vec[i][j].second; + os << "\" "; + } else { + fs << boost::format("%%-%is") % (m_column_size[j] + 2); + os << boost::format(fs.str()) % m_vec[i][j].second; + os << "|"; + } + } + + os << "\n"; + if (!m_keyval) { + if (i == (m_vec.size() - 1)) { + // print trailer + os << "+"; + for (size_t j = 0; j < m_vec[i].size(); j++) { + for (size_t v = 0; v < m_column_size[j] + 3; v++) + os << "-"; + os << "+"; + } + os << "\n"; + } + } + m_vec[i].clear(); + } + m_vec.clear(); +} + +void TableFormatter::reset() +{ + m_ss.clear(); + m_ss.str(""); + m_section_cnt.clear(); + m_column_size.clear(); + m_section_open = 0; +} + +void TableFormatter::open_object_section(const char *name) +{ + open_section_in_ns(name, NULL, NULL); +} + +void TableFormatter::open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs) +{ + open_section_in_ns(name, NULL, NULL); +} + +void TableFormatter::open_object_section_in_ns(const char *name, const char *ns) +{ + open_section_in_ns(name, NULL, NULL); +} + +void TableFormatter::open_array_section(const char *name) +{ + open_section_in_ns(name, NULL, NULL); +} + +void TableFormatter::open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs) +{ + open_section_in_ns(name, NULL, NULL); +} + +void TableFormatter::open_array_section_in_ns(const char *name, const char *ns) +{ + open_section_in_ns(name, NULL, NULL); +} + +void TableFormatter::open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs) +{ + m_section.push_back(name); + m_section_open++; +} + +void TableFormatter::close_section() +{ + // + m_section_open--; + if (m_section.size()) { + m_section_cnt[m_section.back()] = 0; + m_section.pop_back(); + } +} + +size_t TableFormatter::m_vec_index(const char *name) +{ + std::string key(name); + + size_t i = m_vec.size(); + if (i) + i--; + + // make sure there are vectors to push back key/val pairs + if (!m_vec.size()) + m_vec.resize(1); + + if (m_vec.size()) { + if (m_vec[i].size()) { + if (m_vec[i][0].first == key) { + // start a new column if a key is repeated + m_vec.resize(m_vec.size() + 1); + i++; + } + } + } + + return i; +} + +std::string TableFormatter::get_section_name(const char* name) +{ + std::string t_name = name; + for (size_t i = 0; i < m_section.size(); i++) { + t_name.insert(0, ":"); + t_name.insert(0, m_section[i]); + } + if (m_section_open) { + std::stringstream lss; + lss << t_name; + lss << "["; + lss << m_section_cnt[t_name]++; + lss << "]"; + return lss.str(); + } else { + return t_name; + } +} + +void TableFormatter::dump_unsigned(const char *name, uint64_t u) +{ + finish_pending_string(); + size_t i = m_vec_index(name); + m_ss << u; + m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); + m_ss.clear(); + m_ss.str(""); +} + +void TableFormatter::dump_int(const char *name, int64_t u) +{ + finish_pending_string(); + size_t i = m_vec_index(name); + m_ss << u; + m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); + m_ss.clear(); + m_ss.str(""); +} + +void TableFormatter::dump_float(const char *name, double d) +{ + finish_pending_string(); + size_t i = m_vec_index(name); + m_ss << d; + + m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); + m_ss.clear(); + m_ss.str(""); +} + +void TableFormatter::dump_string(const char *name, const std::string& s) +{ + finish_pending_string(); + size_t i = m_vec_index(name); + m_ss << s; + + m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); + m_ss.clear(); + m_ss.str(""); +} + +void TableFormatter::dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) +{ + finish_pending_string(); + size_t i = m_vec_index(name); + + std::string attrs_str; + get_attrs_str(&attrs, attrs_str); + m_ss << attrs_str << s; + + m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); + m_ss.clear(); + m_ss.str(""); +} + +void TableFormatter::dump_format_va(const char* name, const char *ns, bool quoted, const char *fmt, va_list ap) +{ + finish_pending_string(); + char buf[LARGE_SIZE]; + vsnprintf(buf, LARGE_SIZE, fmt, ap); + + size_t i = m_vec_index(name); + if (ns) { + m_ss << ns << "." << buf; + } else + m_ss << buf; + + m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); + m_ss.clear(); + m_ss.str(""); +} + +std::ostream& TableFormatter::dump_stream(const char *name) +{ + finish_pending_string(); + // we don't support this + m_pending_name = name; + return m_ss; +} + +int TableFormatter::get_len() const +{ + // we don't know the size until flush is called + return 0; +} + +void TableFormatter::write_raw_data(const char *data) { + // not supported +} + +void TableFormatter::get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str) +{ + std::stringstream attrs_ss; + + for (std::list >::const_iterator iter = attrs->attrs.begin(); + iter != attrs->attrs.end(); ++iter) { + std::pair p = *iter; + attrs_ss << " " << p.first << "=" << "\"" << p.second << "\""; + } + + attrs_str = attrs_ss.str(); +} + +void TableFormatter::finish_pending_string() +{ + if (m_pending_name.length()) { + std::string ss = m_ss.str(); + m_ss.clear(); + m_ss.str(""); + std::string pending_name = m_pending_name; + m_pending_name = ""; + dump_string(pending_name.c_str(), ss); + } +} +} + diff --git a/src/common/Formatter.h b/src/common/Formatter.h index 43d521d10dc51..aead9c26edb33 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -89,5 +89,128 @@ namespace ceph { } }; + class JSONFormatter : public Formatter { + public: + JSONFormatter(bool p = false); + + void flush(std::ostream& os); + void reset(); + virtual void open_array_section(const char *name); + void open_array_section_in_ns(const char *name, const char *ns); + void open_object_section(const char *name); + void open_object_section_in_ns(const char *name, const char *ns); + void close_section(); + void dump_unsigned(const char *name, uint64_t u); + void dump_int(const char *name, int64_t u); + void dump_float(const char *name, double d); + void dump_string(const char *name, const std::string& s); + std::ostream& dump_stream(const char *name); + void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap); + int get_len() const; + void write_raw_data(const char *data); + + private: + + struct json_formatter_stack_entry_d { + int size; + bool is_array; + json_formatter_stack_entry_d() : size(0), is_array(false) { } + }; + + bool m_pretty; + void open_section(const char *name, bool is_array); + void print_quoted_string(const std::string& s); + void print_name(const char *name); + void print_comma(json_formatter_stack_entry_d& entry); + void finish_pending_string(); + + std::stringstream m_ss, m_pending_string; + std::list m_stack; + bool m_is_pending_string; + }; + + class XMLFormatter : public Formatter { + public: + static const char *XML_1_DTD; + XMLFormatter(bool pretty = false); + + void flush(std::ostream& os); + void reset(); + void open_array_section(const char *name); + void open_array_section_in_ns(const char *name, const char *ns); + void open_object_section(const char *name); + void open_object_section_in_ns(const char *name, const char *ns); + void close_section(); + void dump_unsigned(const char *name, uint64_t u); + void dump_int(const char *name, int64_t u); + void dump_float(const char *name, double d); + void dump_string(const char *name, const std::string& s); + std::ostream& dump_stream(const char *name); + void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap); + int get_len() const; + void write_raw_data(const char *data); + + /* with attrs */ + void open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs); + void open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs); + void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs); + private: + void open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs); + void finish_pending_string(); + void print_spaces(); + static std::string escape_xml_str(const char *str); + void get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str); + + std::stringstream m_ss, m_pending_string; + std::deque m_sections; + bool m_pretty; + std::string m_pending_string_name; + }; + + class TableFormatter : public Formatter { + public: + TableFormatter(bool keyval = false); + + void flush(std::ostream& os); + void reset(); + virtual void open_array_section(const char *name); + void open_array_section_in_ns(const char *name, const char *ns); + void open_object_section(const char *name); + void open_object_section_in_ns(const char *name, const char *ns); + + void open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs); + void open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs); + + void close_section(); + void dump_unsigned(const char *name, uint64_t u); + void dump_int(const char *name, int64_t u); + void dump_float(const char *name, double d); + void dump_string(const char *name, const std::string& s); + void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap); + void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs); + std::ostream& dump_stream(const char *name); + + int get_len() const; + void write_raw_data(const char *data); + void get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str); + + private: + void open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs); + std::vector< std::vector > > m_vec; + std::stringstream m_ss; + size_t m_vec_index(const char* name); + std::string get_section_name(const char* name); + void finish_pending_string(); + std::string m_pending_name; + bool m_keyval; + + int m_section_open; + std::vector< std::string > m_section; + std::map m_section_cnt; + std::vector m_column_size; + std::vector< std::string > m_column_name; + }; + + } #endif diff --git a/src/common/JSONFormatter.cc b/src/common/JSONFormatter.cc deleted file mode 100644 index 8bf5007ba7fbf..0000000000000 --- a/src/common/JSONFormatter.cc +++ /dev/null @@ -1,226 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2011 New Dream Network - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#define LARGE_SIZE 1024 - -#include "include/int_types.h" - -#include "assert.h" -#include "Formatter.h" -#include "JSONFormatter.h" -#include "common/escape.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// ----------------------- -namespace ceph { - -JSONFormatter::JSONFormatter(bool p) -: m_pretty(p), m_is_pending_string(false) -{ - reset(); -} - -void JSONFormatter::flush(std::ostream& os) -{ - finish_pending_string(); - os << m_ss.str(); - if (m_pretty) - os << "\n"; - m_ss.clear(); - m_ss.str(""); -} - -void JSONFormatter::reset() -{ - m_stack.clear(); - m_ss.clear(); - m_ss.str(""); - m_pending_string.clear(); - m_pending_string.str(""); -} - -void JSONFormatter::print_comma(json_formatter_stack_entry_d& entry) -{ - if (entry.size) { - if (m_pretty) { - m_ss << ",\n"; - for (unsigned i = 1; i < m_stack.size(); i++) - m_ss << " "; - } else { - m_ss << ","; - } - } else if (m_pretty) { - m_ss << "\n"; - for (unsigned i = 1; i < m_stack.size(); i++) - m_ss << " "; - } - if (m_pretty && entry.is_array) - m_ss << " "; -} - -void JSONFormatter::print_quoted_string(const std::string& s) -{ - int len = escape_json_attr_len(s.c_str(), s.size()); - char escaped[len]; - escape_json_attr(s.c_str(), s.size(), escaped); - m_ss << '\"' << escaped << '\"'; -} - -void JSONFormatter::print_name(const char *name) -{ - finish_pending_string(); - if (m_stack.empty()) - return; - struct json_formatter_stack_entry_d& entry = m_stack.back(); - print_comma(entry); - if (!entry.is_array) { - if (m_pretty) { - m_ss << " "; - } - m_ss << "\"" << name << "\""; - if (m_pretty) - m_ss << ": "; - else - m_ss << ':'; - } - ++entry.size; -} - -void JSONFormatter::open_section(const char *name, bool is_array) -{ - print_name(name); - if (is_array) - m_ss << '['; - else - m_ss << '{'; - - json_formatter_stack_entry_d n; - n.is_array = is_array; - m_stack.push_back(n); -} - -void JSONFormatter::open_array_section(const char *name) -{ - open_section(name, true); -} - -void JSONFormatter::open_array_section_in_ns(const char *name, const char *ns) -{ - std::ostringstream oss; - oss << name << " " << ns; - open_section(oss.str().c_str(), true); -} - -void JSONFormatter::open_object_section(const char *name) -{ - open_section(name, false); -} - -void JSONFormatter::open_object_section_in_ns(const char *name, const char *ns) -{ - std::ostringstream oss; - oss << name << " " << ns; - open_section(oss.str().c_str(), false); -} - -void JSONFormatter::close_section() -{ - assert(!m_stack.empty()); - finish_pending_string(); - - struct json_formatter_stack_entry_d& entry = m_stack.back(); - if (m_pretty && entry.size) { - m_ss << "\n"; - for (unsigned i = 1; i < m_stack.size(); i++) - m_ss << " "; - } - m_ss << (entry.is_array ? ']' : '}'); - m_stack.pop_back(); -} - -void JSONFormatter::finish_pending_string() -{ - if (m_is_pending_string) { - print_quoted_string(m_pending_string.str()); - m_pending_string.str(std::string()); - m_is_pending_string = false; - } -} - -void JSONFormatter::dump_unsigned(const char *name, uint64_t u) -{ - print_name(name); - m_ss << u; -} - -void JSONFormatter::dump_int(const char *name, int64_t s) -{ - print_name(name); - m_ss << s; -} - -void JSONFormatter::dump_float(const char *name, double d) -{ - print_name(name); - char foo[30]; - snprintf(foo, sizeof(foo), "%lf", d); - m_ss << foo; -} - -void JSONFormatter::dump_string(const char *name, const std::string& s) -{ - print_name(name); - print_quoted_string(s); -} - -std::ostream& JSONFormatter::dump_stream(const char *name) -{ - print_name(name); - m_is_pending_string = true; - return m_pending_string; -} - -void JSONFormatter::dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap) -{ - char buf[LARGE_SIZE]; - vsnprintf(buf, LARGE_SIZE, fmt, ap); - - print_name(name); - if (quoted) { - print_quoted_string(std::string(buf)); - } else { - m_ss << std::string(buf); - } -} - -int JSONFormatter::get_len() const -{ - return m_ss.str().size(); -} - -void JSONFormatter::write_raw_data(const char *data) -{ - m_ss << data; -} - -} // namespace ceph diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 6f1f11c3c7500..81daa4ab18e78 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -45,10 +45,7 @@ libcommon_internal_la_SOURCES = \ common/simple_spin.cc \ common/Thread.cc \ common/Formatter.cc \ - common/XMLFormatter.cc \ common/HTMLFormatter.cc \ - common/JSONFormatter.cc \ - common/TableFormatter.cc \ common/HeartbeatMap.cc \ common/config.cc \ common/utf8.c \ @@ -182,9 +179,6 @@ noinst_HEADERS += \ common/DecayCounter.h \ common/Finisher.h \ common/Formatter.h \ - common/JSONFormatter.h \ - common/TableFormatter.h \ - common/XMLFormatter.h \ common/HTMLFormatter.h \ common/perf_counters.h \ common/OutputDataSocket.h \ diff --git a/src/common/TableFormatter.cc b/src/common/TableFormatter.cc deleted file mode 100644 index 4be4e39077b78..0000000000000 --- a/src/common/TableFormatter.cc +++ /dev/null @@ -1,376 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2011 New Dream Network - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#define LARGE_SIZE 1024 - -#include "include/int_types.h" - -#include "assert.h" -#include "Formatter.h" -#include "TableFormatter.h" -#include "common/escape.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// ----------------------- -namespace ceph { -TableFormatter::TableFormatter(bool keyval) : m_keyval(keyval) -{ - reset(); -} - -void TableFormatter::flush(std::ostream& os) -{ - finish_pending_string(); - std::vector column_size = m_column_size; - std::vector column_name = m_column_name; - - std::set need_header_set; - - // auto-sizing columns - for (size_t i = 0; i < m_vec.size(); i++) { - for (size_t j = 0; j < m_vec[i].size(); j++) { - column_size.resize(m_vec[i].size()); - column_name.resize(m_vec[i].size()); - if (i > 0) { - if (m_vec[i - 1][j] != m_vec[i][j]) { - // changing row labels require to show the header - need_header_set.insert(i); - column_name[i] = m_vec[i][j].first; - } - } else { - column_name[i] = m_vec[i][j].first; - } - - if (m_vec[i][j].second.length() > column_size[j]) - column_size[j] = m_vec[i][j].second.length(); - if (m_vec[i][j].first.length() > column_size[j]) - column_size[j] = m_vec[i][j].first.length(); - } - } - - bool need_header = false; - if ((column_size.size() == m_column_size.size())) { - for (size_t i = 0; i < column_size.size(); i++) { - if (column_size[i] != m_column_size[i]) { - need_header = true; - break; - } - } - } else { - need_header = true; - } - - if (need_header) { - // first row always needs a header if there wasn't one before - need_header_set.insert(0); - } - - m_column_size = column_size; - for (size_t i = 0; i < m_vec.size(); i++) { - if (i == 0) { - if (need_header_set.count(i)) { - // print the header - if (!m_keyval) { - os << "+"; - for (size_t j = 0; j < m_vec[i].size(); j++) { - for (size_t v = 0; v < m_column_size[j] + 3; v++) - os << "-"; - os << "+"; - } - os << "\n"; - os << "|"; - - for (size_t j = 0; j < m_vec[i].size(); j++) { - os << " "; - std::stringstream fs; - fs << boost::format("%%-%is") % (m_column_size[j] + 2); - os << boost::format(fs.str()) % m_vec[i][j].first; - os << "|"; - } - os << "\n"; - os << "+"; - for (size_t j = 0; j < m_vec[i].size(); j++) { - for (size_t v = 0; v < m_column_size[j] + 3; v++) - os << "-"; - os << "+"; - } - os << "\n"; - } - } - } - // print body - if (!m_keyval) - os << "|"; - for (size_t j = 0; j < m_vec[i].size(); j++) { - if (!m_keyval) - os << " "; - std::stringstream fs; - - if (m_keyval) { - os << "key::"; - os << m_vec[i][j].first; - os << "="; - os << "\""; - os << m_vec[i][j].second; - os << "\" "; - } else { - fs << boost::format("%%-%is") % (m_column_size[j] + 2); - os << boost::format(fs.str()) % m_vec[i][j].second; - os << "|"; - } - } - - os << "\n"; - if (!m_keyval) { - if (i == (m_vec.size() - 1)) { - // print trailer - os << "+"; - for (size_t j = 0; j < m_vec[i].size(); j++) { - for (size_t v = 0; v < m_column_size[j] + 3; v++) - os << "-"; - os << "+"; - } - os << "\n"; - } - } - m_vec[i].clear(); - } - m_vec.clear(); -} - -void TableFormatter::reset() -{ - m_ss.clear(); - m_ss.str(""); - m_section_cnt.clear(); - m_column_size.clear(); - m_section_open = 0; -} - -void TableFormatter::open_object_section(const char *name) -{ - open_section_in_ns(name, NULL, NULL); -} - -void TableFormatter::open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs) -{ - open_section_in_ns(name, NULL, NULL); -} - -void TableFormatter::open_object_section_in_ns(const char *name, const char *ns) -{ - open_section_in_ns(name, NULL, NULL); -} - -void TableFormatter::open_array_section(const char *name) -{ - open_section_in_ns(name, NULL, NULL); -} - -void TableFormatter::open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs) -{ - open_section_in_ns(name, NULL, NULL); -} - -void TableFormatter::open_array_section_in_ns(const char *name, const char *ns) -{ - open_section_in_ns(name, NULL, NULL); -} - -void TableFormatter::open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs) -{ - m_section.push_back(name); - m_section_open++; -} - -void TableFormatter::close_section() -{ - // - m_section_open--; - if (m_section.size()) { - m_section_cnt[m_section.back()] = 0; - m_section.pop_back(); - } -} - -size_t TableFormatter::m_vec_index(const char *name) -{ - std::string key(name); - - size_t i = m_vec.size(); - if (i) - i--; - - // make sure there are vectors to push back key/val pairs - if (!m_vec.size()) - m_vec.resize(1); - - if (m_vec.size()) { - if (m_vec[i].size()) { - if (m_vec[i][0].first == key) { - // start a new column if a key is repeated - m_vec.resize(m_vec.size() + 1); - i++; - } - } - } - - return i; -} - -std::string TableFormatter::get_section_name(const char* name) -{ - std::string t_name = name; - for (size_t i = 0; i < m_section.size(); i++) { - t_name.insert(0, ":"); - t_name.insert(0, m_section[i]); - } - if (m_section_open) { - std::stringstream lss; - lss << t_name; - lss << "["; - lss << m_section_cnt[t_name]++; - lss << "]"; - return lss.str(); - } else { - return t_name; - } -} - -void TableFormatter::dump_unsigned(const char *name, uint64_t u) -{ - finish_pending_string(); - size_t i = m_vec_index(name); - m_ss << u; - m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); - m_ss.clear(); - m_ss.str(""); -} - -void TableFormatter::dump_int(const char *name, int64_t u) -{ - finish_pending_string(); - size_t i = m_vec_index(name); - m_ss << u; - m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); - m_ss.clear(); - m_ss.str(""); -} - -void TableFormatter::dump_float(const char *name, double d) -{ - finish_pending_string(); - size_t i = m_vec_index(name); - m_ss << d; - - m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); - m_ss.clear(); - m_ss.str(""); -} - -void TableFormatter::dump_string(const char *name, const std::string& s) -{ - finish_pending_string(); - size_t i = m_vec_index(name); - m_ss << s; - - m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); - m_ss.clear(); - m_ss.str(""); -} - -void TableFormatter::dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) -{ - finish_pending_string(); - size_t i = m_vec_index(name); - - std::string attrs_str; - get_attrs_str(&attrs, attrs_str); - m_ss << attrs_str << s; - - m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); - m_ss.clear(); - m_ss.str(""); -} - -void TableFormatter::dump_format_va(const char* name, const char *ns, bool quoted, const char *fmt, va_list ap) -{ - finish_pending_string(); - char buf[LARGE_SIZE]; - vsnprintf(buf, LARGE_SIZE, fmt, ap); - - size_t i = m_vec_index(name); - if (ns) { - m_ss << ns << "." << buf; - } else - m_ss << buf; - - m_vec[i].push_back(std::make_pair(get_section_name(name), m_ss.str())); - m_ss.clear(); - m_ss.str(""); -} - -std::ostream& TableFormatter::dump_stream(const char *name) -{ - finish_pending_string(); - // we don't support this - m_pending_name = name; - return m_ss; -} - -int TableFormatter::get_len() const -{ - // we don't know the size until flush is called - return 0; -} - -void TableFormatter::write_raw_data(const char *data) { - // not supported -} - -void TableFormatter::get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str) -{ - std::stringstream attrs_ss; - - for (std::list >::const_iterator iter = attrs->attrs.begin(); - iter != attrs->attrs.end(); ++iter) { - std::pair p = *iter; - attrs_ss << " " << p.first << "=" << "\"" << p.second << "\""; - } - - attrs_str = attrs_ss.str(); -} - -void TableFormatter::finish_pending_string() -{ - if (m_pending_name.length()) { - std::string ss = m_ss.str(); - m_ss.clear(); - m_ss.str(""); - std::string pending_name = m_pending_name; - m_pending_name = ""; - dump_string(pending_name.c_str(), ss); - } -} - -} // namespace ceph diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index c3425d8945225..95a48b3ec129b 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -26,7 +26,6 @@ #include "common/safe_io.h" #include "common/version.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include #include diff --git a/src/mon/ConfigKeyService.cc b/src/mon/ConfigKeyService.cc index 790102e0f97ba..97126ed0d1f0e 100644 --- a/src/mon/ConfigKeyService.cc +++ b/src/mon/ConfigKeyService.cc @@ -24,7 +24,6 @@ #include "common/config.h" #include "common/cmdparse.h" #include "common/errno.h" -#include "common/JSONFormatter.h" #define dout_subsys ceph_subsys_mon #undef dout_prefix diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 479804bc14adc..4a34283e0d9d6 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -61,8 +61,6 @@ #include "common/errno.h" #include "common/perf_counters.h" #include "common/admin_socket.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "include/color.h" #include "include/ceph_fs.h" diff --git a/src/mon/Paxos.cc b/src/mon/Paxos.cc index 1eddd6a2ccc97..297ea17ce075a 100644 --- a/src/mon/Paxos.cc +++ b/src/mon/Paxos.cc @@ -23,7 +23,6 @@ #include "include/assert.h" #include "include/stringify.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" #define dout_subsys ceph_subsys_paxos #undef dout_prefix diff --git a/src/mon/Paxos.h b/src/mon/Paxos.h index 0c69f2b204619..457c8af1d267f 100644 --- a/src/mon/Paxos.h +++ b/src/mon/Paxos.h @@ -121,9 +121,6 @@ e 12v #include "common/perf_counters.h" #include -#include "common/Formatter.h" -#include "common/JSONFormatter.h" - #include "MonitorDBStore.h" class Monitor; diff --git a/src/os/FileJournal.cc b/src/os/FileJournal.cc index 3c930819b0ce1..c6bb6f2c0755d 100644 --- a/src/os/FileJournal.cc +++ b/src/os/FileJournal.cc @@ -34,8 +34,6 @@ #include "common/blkdev.h" #include "common/linux_version.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #define dout_subsys ceph_subsys_journal #undef dout_prefix diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index b06ddea921f21..4f0772e9b646e 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -66,8 +66,6 @@ #include "common/perf_counters.h" #include "common/sync_filesystem.h" #include "common/fd.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "HashIndex.h" #include "DBObjectMap.h" #include "KeyValueDB.h" diff --git a/src/os/FileStore.h b/src/os/FileStore.h index 11a4401233194..a9291b743c9bc 100644 --- a/src/os/FileStore.h +++ b/src/os/FileStore.h @@ -33,8 +33,6 @@ using namespace std; #include "common/Timer.h" #include "common/WorkQueue.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "common/Mutex.h" #include "HashIndex.h" diff --git a/src/os/KeyValueStore.cc b/src/os/KeyValueStore.cc index 64fff67477ac0..1881f2dcc17ac 100644 --- a/src/os/KeyValueStore.cc +++ b/src/os/KeyValueStore.cc @@ -48,8 +48,6 @@ #include "common/safe_io.h" #include "common/perf_counters.h" #include "common/sync_filesystem.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #ifdef HAVE_KINETIC #include "KineticStore.h" diff --git a/src/os/KeyValueStore.h b/src/os/KeyValueStore.h index 2519fdf5dee15..ef3085fa0d95b 100644 --- a/src/os/KeyValueStore.h +++ b/src/os/KeyValueStore.h @@ -33,7 +33,6 @@ using namespace std; #include "common/WorkQueue.h" #include "common/Finisher.h" #include "common/fd.h" -#include "common/JSONFormatter.h" #include "common/Mutex.h" #include "GenericObjectMap.h" diff --git a/src/os/MemStore.cc b/src/os/MemStore.cc index 67a70ffa55284..a1e1b274ec0e9 100644 --- a/src/os/MemStore.cc +++ b/src/os/MemStore.cc @@ -26,8 +26,6 @@ #include "include/unordered_map.h" #include "include/memory.h" #include "common/errno.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "MemStore.h" #define dout_subsys ceph_subsys_filestore diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index d1e9cc8ec8f6c..e0c5b3d6b8b96 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -43,8 +43,6 @@ #include "common/ceph_argparse.h" #include "common/version.h" #include "common/io_priority.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "os/ObjectStore.h" diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 54cf9215c402d..173b468b5b0ed 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -19,7 +19,6 @@ #include "common/config.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "common/TextTable.h" #include "include/ceph_features.h" #include "include/str_map.h" diff --git a/src/rbd.cc b/src/rbd.cc index 1a96177c832e1..6f5457d1461f9 100644 --- a/src/rbd.cc +++ b/src/rbd.cc @@ -51,8 +51,6 @@ #include "include/util.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" -#include "common/XMLFormatter.h" #include "common/Throttle.h" #if defined(__linux__) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index d40700b1de9c9..45cb2e197eddc 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -15,8 +15,6 @@ using namespace std; #include "common/config.h" #include "common/ceph_argparse.h" #include "common/Formatter.h" -#include "common/XMLFormatter.h" -#include "common/JSONFormatter.h" #include "common/errno.h" #include "global/global_init.h" diff --git a/src/rgw/rgw_cors_s3.h b/src/rgw/rgw_cors_s3.h index 6cac6765a311d..0db03c3ea1420 100644 --- a/src/rgw/rgw_cors_s3.h +++ b/src/rgw/rgw_cors_s3.h @@ -22,7 +22,6 @@ #include #include -#include #include "rgw_xml.h" #include "rgw_cors.h" diff --git a/src/rgw/rgw_jsonparser.cc b/src/rgw/rgw_jsonparser.cc index 8531d866e7911..b4b709ce8c71f 100644 --- a/src/rgw/rgw_jsonparser.cc +++ b/src/rgw/rgw_jsonparser.cc @@ -10,7 +10,6 @@ #include "include/types.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "common/ceph_json.h" #include "rgw_common.h" diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index 4d362e26d5052..795d78787327e 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -6,7 +6,6 @@ #include "common/utf8.h" #include "common/OutputDataSocket.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "rgw_log.h" #include "rgw_acl.h" diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 9196a4fa6915e..2f328aad19dc1 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -5,8 +5,6 @@ #include #include "common/Formatter.h" -#include "common/JSONFormatter.h" -#include "common/XMLFormatter.h" #include "common/HTMLFormatter.h" #include "common/utf8.h" #include "include/str_list.h" diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index cd21df5831923..7da0c7f8b37ec 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -6,7 +6,6 @@ #include "common/ceph_crypto.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "common/utf8.h" #include "common/ceph_json.h" diff --git a/src/rgw/rgw_swift.cc b/src/rgw/rgw_swift.cc index 06662da2b6a07..09fb9b7b92845 100644 --- a/src/rgw/rgw_swift.cc +++ b/src/rgw/rgw_swift.cc @@ -6,7 +6,6 @@ #include #include "common/ceph_json.h" -#include "common/JSONFormatter.h" #include "rgw_common.h" #include "rgw_swift.h" #include "rgw_swift_auth.h" diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc index 5311002ff858b..3e94e63a162ba 100644 --- a/src/rgw/rgw_swift_auth.cc +++ b/src/rgw/rgw_swift_auth.cc @@ -6,7 +6,6 @@ #include "common/ceph_crypto.h" #include "common/Clock.h" -#include "common/JSONFormatter.h" #include "auth/Crypto.h" diff --git a/src/test/bench/small_io_bench.cc b/src/test/bench/small_io_bench.cc index 657d6d0624a1d..2b200279c7a4b 100644 --- a/src/test/bench/small_io_bench.cc +++ b/src/test/bench/small_io_bench.cc @@ -14,7 +14,6 @@ #include #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "bencher.h" #include "rados_backend.h" diff --git a/src/test/bench/small_io_bench_dumb.cc b/src/test/bench/small_io_bench_dumb.cc index 913e8c0a1a333..c9d9f51a9db57 100644 --- a/src/test/bench/small_io_bench_dumb.cc +++ b/src/test/bench/small_io_bench_dumb.cc @@ -16,7 +16,6 @@ #include #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "bencher.h" #include "rados_backend.h" diff --git a/src/test/bench/small_io_bench_fs.cc b/src/test/bench/small_io_bench_fs.cc index 31bb10e2470ec..f5845855a1d5f 100644 --- a/src/test/bench/small_io_bench_fs.cc +++ b/src/test/bench/small_io_bench_fs.cc @@ -14,7 +14,6 @@ #include #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "bencher.h" #include "rados_backend.h" diff --git a/src/test/bench/small_io_bench_rbd.cc b/src/test/bench/small_io_bench_rbd.cc index 95193adf9b610..ba7071ed3833a 100644 --- a/src/test/bench/small_io_bench_rbd.cc +++ b/src/test/bench/small_io_bench_rbd.cc @@ -13,7 +13,6 @@ #include #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "bencher.h" #include "rbd_backend.h" diff --git a/src/test/bench/tp_bench.cc b/src/test/bench/tp_bench.cc index a1221997ca204..b9d5ff17c8f85 100644 --- a/src/test/bench/tp_bench.cc +++ b/src/test/bench/tp_bench.cc @@ -14,7 +14,6 @@ #include #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "bencher.h" #include "rados_backend.h" diff --git a/src/test/common/test_tableformatter.cc b/src/test/common/test_tableformatter.cc index c53b69f53652d..88648f01d73b5 100644 --- a/src/test/common/test_tableformatter.cc +++ b/src/test/common/test_tableformatter.cc @@ -1,6 +1,6 @@ #include "gtest/gtest.h" -#include "common/TableFormatter.h" +#include "common/Formatter.h" #include #include #include diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc index 1fafbd337072e..c46fa87ab5409 100644 --- a/src/test/crush/crush.cc +++ b/src/test/crush/crush.cc @@ -13,7 +13,6 @@ #include "include/stringify.h" #include "common/ceph_argparse.h" -#include "common/JSONFormatter.h" #include "global/global_init.h" #include "global/global_context.h" diff --git a/src/test/encoding/ceph_dencoder.cc b/src/test/encoding/ceph_dencoder.cc index 7386a7d563010..7e1056503ce01 100644 --- a/src/test/encoding/ceph_dencoder.cc +++ b/src/test/encoding/ceph_dencoder.cc @@ -5,7 +5,6 @@ #include "include/ceph_features.h" #include "common/ceph_argparse.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "common/errno.h" #include "msg/Message.h" #include "include/assert.h" diff --git a/src/test/formatter.cc b/src/test/formatter.cc index cf1e412758a2b..3913cc6f15420 100644 --- a/src/test/formatter.cc +++ b/src/test/formatter.cc @@ -13,8 +13,7 @@ */ #include "test/unit.h" -#include "common/JSONFormatter.h" -#include "common/XMLFormatter.h" +#include "common/Formatter.h" #include "common/HTMLFormatter.h" #include diff --git a/src/test/kv_store_bench.h b/src/test/kv_store_bench.h index 9be1e31386b9c..d12c5e850c0b1 100644 --- a/src/test/kv_store_bench.h +++ b/src/test/kv_store_bench.h @@ -20,7 +20,6 @@ #include "global/global_context.h" #include "common/Mutex.h" #include "common/Cond.h" -#include "common/JSONFormatter.h" #include #include diff --git a/src/test/mon/test_mon_workloadgen.cc b/src/test/mon/test_mon_workloadgen.cc index 36e36ba6c95da..e18906d193148 100644 --- a/src/test/mon/test_mon_workloadgen.cc +++ b/src/test/mon/test_mon_workloadgen.cc @@ -51,7 +51,6 @@ #include "auth/AuthAuthorizeHandler.h" #include "include/uuid.h" #include "include/assert.h" -#include "common/JSONFormatter.h" #include "messages/MOSDBoot.h" #include "messages/MOSDAlive.h" diff --git a/src/test/test_rgw_admin_meta.cc b/src/test/test_rgw_admin_meta.cc index b2213fc2afa72..74bc8ed1ec867 100644 --- a/src/test/test_rgw_admin_meta.cc +++ b/src/test/test_rgw_admin_meta.cc @@ -30,8 +30,6 @@ extern "C"{ #include "common/code_environment.h" #include "common/ceph_argparse.h" #include "common/Finisher.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "global/global_init.h" #include "rgw/rgw_common.h" #include "rgw/rgw_rados.h" diff --git a/src/tools/ceph-client-debug.cc b/src/tools/ceph-client-debug.cc index 20948bc527c98..2ed93326b4433 100644 --- a/src/tools/ceph-client-debug.cc +++ b/src/tools/ceph-client-debug.cc @@ -15,7 +15,7 @@ #include "common/ceph_argparse.h" #include "global/global_init.h" -#include "common/JSONFormatter.h" +#include "common/Formatter.h" #include "common/debug.h" #include "common/errno.h" #include "client/Inode.h" diff --git a/src/tools/ceph_monstore_tool.cc b/src/tools/ceph_monstore_tool.cc index 9190d7c050504..744000eba872e 100644 --- a/src/tools/ceph_monstore_tool.cc +++ b/src/tools/ceph_monstore_tool.cc @@ -16,7 +16,7 @@ #include #include -#include "common/JSONFormatter.h" +#include "common/Formatter.h" #include "common/errno.h" #include "global/global_init.h" diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc index 9ad502bdef7ef..9e6894664ca8c 100644 --- a/src/tools/ceph_objectstore_tool.cc +++ b/src/tools/ceph_objectstore_tool.cc @@ -19,7 +19,6 @@ #include #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "common/errno.h" #include "common/ceph_argparse.h" diff --git a/src/tools/cephfs/EventOutput.cc b/src/tools/cephfs/EventOutput.cc index a273c511b5506..a93c099a918d1 100644 --- a/src/tools/cephfs/EventOutput.cc +++ b/src/tools/cephfs/EventOutput.cc @@ -16,7 +16,6 @@ #include #include "common/errno.h" -#include "common/JSONFormatter.h" #include "mds/mdstypes.h" #include "mds/events/EUpdate.h" #include "mds/LogEvent.h" diff --git a/src/tools/cephfs/JournalTool.cc b/src/tools/cephfs/JournalTool.cc index 360782c141eb4..6118320435948 100644 --- a/src/tools/cephfs/JournalTool.cc +++ b/src/tools/cephfs/JournalTool.cc @@ -16,7 +16,6 @@ #include "common/ceph_argparse.h" #include "common/errno.h" -#include "common/JSONFormatter.h" #include "osdc/Journaler.h" #include "mds/mdstypes.h" #include "mds/LogEvent.h" diff --git a/src/tools/cephfs/TableTool.cc b/src/tools/cephfs/TableTool.cc index 005d8b4c6000f..4b22de04f7231 100644 --- a/src/tools/cephfs/TableTool.cc +++ b/src/tools/cephfs/TableTool.cc @@ -14,8 +14,6 @@ #include "common/ceph_argparse.h" #include "common/errno.h" -#include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "mds/SessionMap.h" #include "mds/InoTable.h" diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc index a727c162ae37c..4b96ac5ae4a10 100644 --- a/src/tools/rados/rados.cc +++ b/src/tools/rados/rados.cc @@ -26,8 +26,6 @@ using namespace librados; #include "common/debug.h" #include "common/errno.h" #include "common/Formatter.h" -#include "common/JSONFormatter.h" -#include "common/XMLFormatter.h" #include "common/obj_bencher.h" #include "mds/inode_backtrace.h" #include "auth/Crypto.h" From f52881ebf7c56f934e613c8263005a5fcd4064aa Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Thu, 3 Dec 2015 15:13:18 -0800 Subject: [PATCH 08/60] Undo constant renaming: RGW_REST_*->RGW_PROTO_*. For clarity, RGW_REST_* should be renamed to RGW_PROTO_* in future. For now, undo it to ease the merging speed for static-sites. Signed-off-by: Robin H. Johnson Conflicts: src/rgw/rgw_common.h --- src/rgw/rgw_common.h | 7 ++----- src/rgw/rgw_op.cc | 2 +- src/rgw/rgw_rest.cc | 18 +++++++++--------- src/rgw/rgw_rest_s3.cc | 4 ++-- src/rgw/rgw_rest_swift.cc | 2 +- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index d07f1de4ebbc4..477079a7cecd9 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -98,11 +98,8 @@ using ceph::crypto::MD5; #define RGW_REST_SWIFT 0x1 #define RGW_REST_SWIFT_AUTH 0x2 - -#define RGW_PROTO_SWIFT 0x1 -#define RGW_PROTO_SWIFT_AUTH 0x2 -#define RGW_PROTO_S3 0x4 -#define RGW_PROTO_WEBSITE 0x8 +#define RGW_REST_S3 0x4 +#define RGW_REST_WEBSITE 0x8 #define RGW_SUSPENDED_USER_AUID (uint64_t)-2 diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 4110f145116ad..005490dc67415 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3390,7 +3390,7 @@ void RGWListBucketMultiparts::execute() if (ret < 0) return; - if (s->prot_flags & RGW_PROTO_SWIFT) { + if (s->prot_flags & RGW_REST_SWIFT) { string path_args; path_args = s->info.args.get("path"); if (!path_args.empty()) { diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 2f328aad19dc1..6491cc6440ebd 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -311,7 +311,7 @@ void set_req_state_err(struct req_state *s, int err_no) err_no = -err_no; s->err.ret = -err_no; - if (s->prot_flags & RGW_PROTO_SWIFT) { + if (s->prot_flags & RGW_REST_SWIFT) { r = search_err(err_no, RGW_HTTP_SWIFT_ERRORS, ARRAY_LEN(RGW_HTTP_SWIFT_ERRORS)); if (r) { s->err.http_ret = r->http_ret; @@ -325,7 +325,7 @@ void set_req_state_err(struct req_state *s, int err_no) r = search_err(err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS)); if (r) { - if (s->prot_flags & RGW_PROTO_WEBSITE && err_no == ERR_WEBSITE_REDIRECT && !s->err.is_clear()) { + if (s->prot_flags & RGW_REST_WEBSITE && err_no == ERR_WEBSITE_REDIRECT && !s->err.is_clear()) { // http_ret was custom set, so don't change it! } else { s->err.http_ret = r->http_ret; @@ -381,7 +381,7 @@ void dump_content_length(struct req_state *s, uint64_t len) void dump_etag(struct req_state *s, const char *etag) { int r; - if (s->prot_flags & RGW_PROTO_SWIFT) { + if (s->prot_flags & RGW_REST_SWIFT) { r = s->cio->print("etag: %s\r\n", etag); } else @@ -546,7 +546,7 @@ void dump_start(struct req_state *s) void dump_trans_id(req_state *s) { - if (s->prot_flags & RGW_PROTO_SWIFT) { + if (s->prot_flags & RGW_REST_SWIFT) { s->cio->print("X-Trans-Id: %s\r\n", s->trans_id.c_str()); } else { @@ -624,7 +624,7 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const delete [] allowed_origins; - if (s->prot_flags & RGW_PROTO_SWIFT && !content_type) { + if (s->prot_flags & RGW_REST_SWIFT && !content_type) { force_content_type = true; } @@ -645,7 +645,7 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const ctype = "text/plain"; break; } - if (s->prot_flags & RGW_PROTO_SWIFT) + if (s->prot_flags & RGW_REST_SWIFT) ctype.append("; charset=utf-8"); content_type = ctype.c_str(); } @@ -1299,7 +1299,7 @@ int RGWHandler_ObjStore::allocate_formatter(struct req_state *s, int default_typ s->formatter = new JSONFormatter(false); break; case RGW_FORMAT_HTML: - s->formatter = new HTMLFormatter(s->prot_flags & RGW_PROTO_WEBSITE); + s->formatter = new HTMLFormatter(s->prot_flags & RGW_REST_WEBSITE); break; default: return -EINVAL; @@ -1527,7 +1527,7 @@ int RGWREST::preprocess(struct req_state *s, RGWClientIO *cio) in_hosted_domain = true; // TODO: should hostnames be a strict superset of hostnames_s3website? domain = s3website_domain; subdomain = s3website_subdomain; - s->prot_flags |= RGW_PROTO_WEBSITE; + s->prot_flags |= RGW_REST_WEBSITE; } } @@ -1557,7 +1557,7 @@ int RGWREST::preprocess(struct req_state *s, RGWClientIO *cio) in_hosted_domain = true; // TODO: should hostnames be a strict superset of hostnames_s3website? domain = s3website_domain; subdomain = s3website_subdomain; - s->prot_flags |= RGW_PROTO_WEBSITE; + s->prot_flags |= RGW_REST_WEBSITE; } } diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 7da0c7f8b37ec..41cd33f2e654a 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3092,7 +3092,7 @@ int RGWHandler_Auth_S3::init(RGWRados *store, struct req_state *state, RGWClient RGWHandler *RGWRESTMgr_S3::get_handler(struct req_state *s) { - bool is_s3website = enable_s3website && (s->prot_flags & RGW_PROTO_WEBSITE); + bool is_s3website = enable_s3website && (s->prot_flags & RGW_REST_WEBSITE); int ret = RGWHandler_ObjStore_S3::init_from_header(s, is_s3website ? RGW_FORMAT_HTML : RGW_FORMAT_XML, false); if (ret < 0) return NULL; @@ -3125,7 +3125,7 @@ int RGWHandler_ObjStore_S3Website::retarget(RGWOp *op, RGWOp **new_op) { *new_op = op; ldout(s->cct, 10) << __func__ << "Starting retarget" << dendl; - if (!(s->prot_flags & RGW_PROTO_WEBSITE)) + if (!(s->prot_flags & RGW_REST_WEBSITE)) return 0; RGWObjectCtx& obj_ctx = *static_cast(s->obj_ctx); diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 9d700d28fe671..bcbd45002c896 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -955,7 +955,7 @@ int RGWHandler_ObjStore_SWIFT::init_from_header(struct req_state *s) string req; string first; - s->prot_flags |= RGW_PROTO_SWIFT; + s->prot_flags |= RGW_REST_SWIFT; const char *req_name = s->decoded_uri.c_str(); const char *p; From 97675a1ffda38831e7d50592d346c15981e19e7f Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Thu, 3 Dec 2015 15:15:10 -0800 Subject: [PATCH 09/60] undo: clean up dead enum RGWEndpointType. Signed-off-by: Robin H. Johnson --- src/rgw/rgw_op.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 8337754d255cc..45acb270a2b28 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -64,14 +64,6 @@ enum RGWOpType { RGW_OP_DELETE_MULTI_OBJ, }; -enum RGWEndpointType { - RGW_ENDPOINT_REST, - RGW_ENDPOINT_REST_SWIFT, - RGW_ENDPOINT_REST_SWIFT_AUTH, - RGW_ENDPOINT_REST_S3, - RGW_ENDPOINT_WEBSITE, -}; - /** * Provide the base class for all ops. */ From 9bcc07d3cd02b255d6ca1603451d740069d3f2f4 Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Thu, 3 Dec 2015 15:43:01 -0800 Subject: [PATCH 10/60] cleanup: remove dead function. Signed-off-by: Robin H. Johnson --- src/rgw/rgw_op.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 45acb270a2b28..6753b6c6abdeb 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -112,9 +112,6 @@ class RGWOp { virtual RGWOpType get_type() { return RGW_OP_UNKNOWN; } virtual uint32_t op_mask() { return 0; } - virtual bool supports_website() { - return false; - } virtual int error_handler(int err_no, string *error_content); }; @@ -184,9 +181,6 @@ class RGWGetObj : public RGWOp { virtual RGWOpType get_type() { return RGW_OP_GET_OBJ; } virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; } virtual bool need_object_expiration() { return false; } - virtual bool supports_website() { - return true; - } }; class RGWGetObj_CB : public RGWGetDataCB @@ -230,9 +224,6 @@ class RGWListBuckets : public RGWOp { virtual const string name() { return "list_buckets"; } virtual RGWOpType get_type() { return RGW_OP_LIST_BUCKETS; } virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; } - virtual bool supports_website() { - return true; - } }; class RGWStatAccount : public RGWOp { From 9f063caad7ff319fd9d8a783d6d7ccea4a32fdc3 Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Thu, 3 Dec 2015 16:46:37 -0800 Subject: [PATCH 11/60] Copyright: add copyright blocks to new files, and update files with existing ones for website changes. Signed-off-by: Robin H. Johnson --- src/rgw/rgw_common.h | 1 + src/rgw/rgw_rest_s3website.h | 13 +++++++++++++ src/rgw/rgw_website.cc | 14 ++++++++++++++ src/rgw/rgw_website.h | 14 ++++++++++++++ src/rgw/rgw_xml_enc.cc | 13 ++++++++++++- 5 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 477079a7cecd9..312251c6763ec 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -4,6 +4,7 @@ * Ceph - scalable distributed file system * * Copyright (C) 2004-2009 Sage Weil + * Copyright (C) 2015 Yehuda Sadeh * * This is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/src/rgw/rgw_rest_s3website.h b/src/rgw/rgw_rest_s3website.h index 24292ca819170..18f00ee99e55d 100644 --- a/src/rgw/rgw_rest_s3website.h +++ b/src/rgw/rgw_rest_s3website.h @@ -1,3 +1,16 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Robin H. Johnson + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ #ifndef CEPH_RGW_REST_S3WEBSITE_H #define CEPH_RGW_REST_S3WEBSITE_H diff --git a/src/rgw/rgw_website.cc b/src/rgw/rgw_website.cc index 14c2af926fd79..eee8977a86f94 100644 --- a/src/rgw/rgw_website.cc +++ b/src/rgw/rgw_website.cc @@ -1,3 +1,17 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Yehuda Sadeh + * Copyright (C) 2015 Robin H. Johnson + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ #include "common/debug.h" #include "common/ceph_json.h" diff --git a/src/rgw/rgw_website.h b/src/rgw/rgw_website.h index 348412d001e06..6c1a92bb47603 100644 --- a/src/rgw/rgw_website.h +++ b/src/rgw/rgw_website.h @@ -1,3 +1,17 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Yehuda Sadeh + * Copyright (C) 2015 Robin H. Johnson + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ #ifndef RGW_WEBSITE_H #define RGW_WEBSITE_H diff --git a/src/rgw/rgw_xml_enc.cc b/src/rgw/rgw_xml_enc.cc index 7577826fcffb1..ff64efca7223b 100644 --- a/src/rgw/rgw_xml_enc.cc +++ b/src/rgw/rgw_xml_enc.cc @@ -1,6 +1,17 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab - +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Yehuda Sadeh + * Copyright (C) 2015 Robin H. Johnson + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ #include "rgw_common.h" #include "rgw_xml.h" From 66345a23e04bb1264a91d067be154d90b455438e Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Thu, 3 Dec 2015 17:18:29 -0800 Subject: [PATCH 12/60] s3website: update cmake. Signed-off-by: Robin H. Johnson Conflicts: src/CMakeLists.txt --- src/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 45b646e8da825..43da5e082d945 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -214,6 +214,7 @@ set(libcommon_files common/simple_spin.cc common/Thread.cc common/Formatter.cc + common/HTMLFormatter.cc common/HeartbeatMap.cc common/ceph_fs.cc common/ceph_hash.cc @@ -732,7 +733,9 @@ if(${WITH_RADOSGW}) rgw/rgw_replica_log.cc rgw/rgw_keystone.cc rgw/rgw_quota.cc - rgw/rgw_dencoder.cc) + rgw/rgw_dencoder.cc + rgw/rgw_website.cc + rgw/rgw_xml_enc.cc) add_library(rgw_a STATIC ${rgw_a_srcs}) From d682f10838f09adc6195d5fa7ec560ab54324b9d Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 19 Jan 2016 09:46:32 -0800 Subject: [PATCH 13/60] rgw: remove split formatter code Signed-off-by: Yehuda Sadeh --- src/common/JSONFormatter.h | 66 --------- src/common/TableFormatter.h | 71 ---------- src/common/XMLFormatter.cc | 274 ------------------------------------ src/common/XMLFormatter.h | 66 --------- 4 files changed, 477 deletions(-) delete mode 100644 src/common/JSONFormatter.h delete mode 100644 src/common/TableFormatter.h delete mode 100644 src/common/XMLFormatter.cc delete mode 100644 src/common/XMLFormatter.h diff --git a/src/common/JSONFormatter.h b/src/common/JSONFormatter.h deleted file mode 100644 index a6a50928be446..0000000000000 --- a/src/common/JSONFormatter.h +++ /dev/null @@ -1,66 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -#ifndef CEPH_JSON_FORMATTER_H -#define CEPH_JSON_FORMATTER_H - -#include "include/int_types.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "include/buffer.h" -#include "Formatter.h" - -namespace ceph { - class JSONFormatter : public Formatter { - public: - JSONFormatter(bool p = false); - - virtual void set_status(const char* status, const char* status_name) {}; - virtual void output_header() {}; - virtual void output_footer() {}; - void flush(std::ostream& os); - void reset(); - virtual void open_array_section(const char *name); - void open_array_section_in_ns(const char *name, const char *ns); - void open_object_section(const char *name); - void open_object_section_in_ns(const char *name, const char *ns); - void close_section(); - void dump_unsigned(const char *name, uint64_t u); - void dump_int(const char *name, int64_t u); - void dump_float(const char *name, double d); - void dump_string(const char *name, const std::string& s); - std::ostream& dump_stream(const char *name); - void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap); - int get_len() const; - void write_raw_data(const char *data); - - private: - - struct json_formatter_stack_entry_d { - int size; - bool is_array; - json_formatter_stack_entry_d() : size(0), is_array(false) { } - }; - - bool m_pretty; - void open_section(const char *name, bool is_array); - void print_quoted_string(const std::string& s); - void print_name(const char *name); - void print_comma(json_formatter_stack_entry_d& entry); - void finish_pending_string(); - - std::stringstream m_ss, m_pending_string; - std::list m_stack; - bool m_is_pending_string; - }; - -} - -#endif diff --git a/src/common/TableFormatter.h b/src/common/TableFormatter.h deleted file mode 100644 index ba8f66823ec7e..0000000000000 --- a/src/common/TableFormatter.h +++ /dev/null @@ -1,71 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -#ifndef CEPH_TABLE_FORMATTER_H -#define CEPH_TABLE_FORMATTER_H - -#include "include/int_types.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "include/buffer.h" -#include "Formatter.h" - -namespace ceph { - class TableFormatter : public Formatter { - public: - TableFormatter(bool keyval = false); - - virtual void set_status(const char* status, const char* status_name) {}; - virtual void output_header() {}; - virtual void output_footer() {}; - void flush(std::ostream& os); - void reset(); - virtual void open_array_section(const char *name); - void open_array_section_in_ns(const char *name, const char *ns); - void open_object_section(const char *name); - void open_object_section_in_ns(const char *name, const char *ns); - - void open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs); - void open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs); - - void close_section(); - void dump_unsigned(const char *name, uint64_t u); - void dump_int(const char *name, int64_t u); - void dump_float(const char *name, double d); - void dump_string(const char *name, const std::string& s); - void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap); - void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs); - std::ostream& dump_stream(const char *name); - - int get_len() const; - void write_raw_data(const char *data); - void get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str); - - private: - void open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs); - std::vector< std::vector > > m_vec; - std::stringstream m_ss; - size_t m_vec_index(const char* name); - std::string get_section_name(const char* name); - void finish_pending_string(); - std::string m_pending_name; - bool m_keyval; - - int m_section_open; - std::vector< std::string > m_section; - std::map m_section_cnt; - std::vector m_column_size; - std::vector< std::string > m_column_name; - }; - - -} - -#endif diff --git a/src/common/XMLFormatter.cc b/src/common/XMLFormatter.cc deleted file mode 100644 index 084fd40eaf85d..0000000000000 --- a/src/common/XMLFormatter.cc +++ /dev/null @@ -1,274 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2011 New Dream Network - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#define LARGE_SIZE 1024 - -#include "include/int_types.h" - -#include "assert.h" -#include "Formatter.h" -#include "XMLFormatter.h" -#include "common/escape.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// ----------------------- -namespace ceph { - -const char *XMLFormatter::XML_1_DTD = - ""; - -XMLFormatter::XMLFormatter(bool pretty) -: m_pretty(pretty) -{ - reset(); -} - -void XMLFormatter::flush(std::ostream& os) -{ - finish_pending_string(); - std::string m_ss_str = m_ss.str(); - os << m_ss_str; - /* There is a small catch here. If the rest of the formatter had NO output, - * we should NOT output a newline. This primarily triggers on HTTP redirects */ - if (m_pretty && !m_ss_str.empty()) - os << "\n"; - m_ss.clear(); - m_ss.str(""); -} - -void XMLFormatter::reset() -{ - m_ss.clear(); - m_ss.str(""); - m_pending_string.clear(); - m_pending_string.str(""); - m_sections.clear(); - m_pending_string_name.clear(); - m_header_done = false; -} - -void XMLFormatter::output_header() -{ - if(!m_header_done) { - m_header_done = true; - write_raw_data(XMLFormatter::XML_1_DTD);; - if (m_pretty) - m_ss << "\n"; - } -} - -void XMLFormatter::output_footer() -{ - while(!m_sections.empty()) { - close_section(); - } -} - -void XMLFormatter::open_object_section(const char *name) -{ - open_section_in_ns(name, NULL, NULL); -} - -void XMLFormatter::open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs) -{ - open_section_in_ns(name, NULL, &attrs); -} - -void XMLFormatter::open_object_section_in_ns(const char *name, const char *ns) -{ - open_section_in_ns(name, ns, NULL); -} - -void XMLFormatter::open_array_section(const char *name) -{ - open_section_in_ns(name, NULL, NULL); -} - -void XMLFormatter::open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs) -{ - open_section_in_ns(name, NULL, &attrs); -} - -void XMLFormatter::open_array_section_in_ns(const char *name, const char *ns) -{ - open_section_in_ns(name, ns, NULL); -} - -void XMLFormatter::close_section() -{ - assert(!m_sections.empty()); - finish_pending_string(); - - std::string section = m_sections.back(); - m_sections.pop_back(); - print_spaces(); - m_ss << ""; - if (m_pretty) - m_ss << "\n"; -} - -void XMLFormatter::dump_unsigned(const char *name, uint64_t u) -{ - std::string e(name); - print_spaces(); - m_ss << "<" << e << ">" << u << ""; - if (m_pretty) - m_ss << "\n"; -} - -void XMLFormatter::dump_int(const char *name, int64_t u) -{ - std::string e(name); - print_spaces(); - m_ss << "<" << e << ">" << u << ""; - if (m_pretty) - m_ss << "\n"; -} - -void XMLFormatter::dump_float(const char *name, double d) -{ - std::string e(name); - print_spaces(); - m_ss << "<" << e << ">" << d << ""; - if (m_pretty) - m_ss << "\n"; -} - -void XMLFormatter::dump_string(const char *name, const std::string& s) -{ - std::string e(name); - print_spaces(); - m_ss << "<" << e << ">" << escape_xml_str(s.c_str()) << ""; - if (m_pretty) - m_ss << "\n"; -} - -void XMLFormatter::dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) -{ - std::string e(name); - std::string attrs_str; - get_attrs_str(&attrs, attrs_str); - print_spaces(); - m_ss << "<" << e << attrs_str << ">" << escape_xml_str(s.c_str()) << ""; - if (m_pretty) - m_ss << "\n"; -} - -std::ostream& XMLFormatter::dump_stream(const char *name) -{ - print_spaces(); - m_pending_string_name = name; - m_ss << "<" << m_pending_string_name << ">"; - return m_pending_string; -} - -void XMLFormatter::dump_format_va(const char* name, const char *ns, bool quoted, const char *fmt, va_list ap) -{ - char buf[LARGE_SIZE]; - vsnprintf(buf, LARGE_SIZE, fmt, ap); - - std::string e(name); - print_spaces(); - if (ns) { - m_ss << "<" << e << " xmlns=\"" << ns << "\">" << buf << ""; - } else { - m_ss << "<" << e << ">" << escape_xml_str(buf) << ""; - } - - if (m_pretty) - m_ss << "\n"; -} - -int XMLFormatter::get_len() const -{ - return m_ss.str().size(); -} - -void XMLFormatter::write_raw_data(const char *data) -{ - m_ss << data; -} - -void XMLFormatter::get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str) -{ - std::stringstream attrs_ss; - - for (std::list >::const_iterator iter = attrs->attrs.begin(); - iter != attrs->attrs.end(); ++iter) { - std::pair p = *iter; - attrs_ss << " " << p.first << "=" << "\"" << p.second << "\""; - } - - attrs_str = attrs_ss.str(); -} - -void XMLFormatter::open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs) -{ - print_spaces(); - std::string attrs_str; - - if (attrs) { - get_attrs_str(attrs, attrs_str); - } - - if (ns) { - m_ss << "<" << name << attrs_str << " xmlns=\"" << ns << "\">"; - } else { - m_ss << "<" << name << attrs_str << ">"; - } - if (m_pretty) - m_ss << "\n"; - m_sections.push_back(name); -} - -void XMLFormatter::finish_pending_string() -{ - if (!m_pending_string_name.empty()) { - m_ss << escape_xml_str(m_pending_string.str().c_str()) - << ""; - m_pending_string_name.clear(); - m_pending_string.str(std::string()); - if (m_pretty) { - m_ss << "\n"; - } - } -} - -void XMLFormatter::print_spaces() -{ - finish_pending_string(); - if (m_pretty) { - std::string spaces(m_sections.size(), ' '); - m_ss << spaces; - } -} - -std::string XMLFormatter::escape_xml_str(const char *str) -{ - int len = escape_xml_attr_len(str); - std::vector escaped(len, '\0'); - escape_xml_attr(str, &escaped[0]); - return std::string(&escaped[0]); -} - -} // namespace ceph diff --git a/src/common/XMLFormatter.h b/src/common/XMLFormatter.h deleted file mode 100644 index f455f5dc8654f..0000000000000 --- a/src/common/XMLFormatter.h +++ /dev/null @@ -1,66 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -#ifndef CEPH_XML_FORMATTER_H -#define CEPH_XML_FORMATTER_H - -#include "include/int_types.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "include/buffer.h" -#include "Formatter.h" - -namespace ceph { - class XMLFormatter : public Formatter { - public: - static const char *XML_1_DTD; - XMLFormatter(bool pretty = false); - - virtual void set_status(const char* status, const char* status_name) {}; - virtual void output_header(); - virtual void output_footer(); - - void flush(std::ostream& os); - void reset(); - void open_array_section(const char *name); - void open_array_section_in_ns(const char *name, const char *ns); - void open_object_section(const char *name); - void open_object_section_in_ns(const char *name, const char *ns); - void close_section(); - void dump_unsigned(const char *name, uint64_t u); - void dump_int(const char *name, int64_t u); - void dump_float(const char *name, double d); - void dump_string(const char *name, const std::string& s); - std::ostream& dump_stream(const char *name); - void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap); - int get_len() const; - void write_raw_data(const char *data); - - /* with attrs */ - void open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs); - void open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs); - void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs); - protected: - void open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs); - void finish_pending_string(); - void print_spaces(); - static std::string escape_xml_str(const char *str); - void get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str); - - std::stringstream m_ss, m_pending_string; - std::deque m_sections; - bool m_pretty; - std::string m_pending_string_name; - bool m_header_done; - }; - -} - -#endif From a5abe3bcb5c2af69b1700e5bdf0c92217fd26fbc Mon Sep 17 00:00:00 2001 From: Gaurav Bafna Date: Mon, 20 Jun 2016 22:37:57 +1100 Subject: [PATCH 14/60] DSS Changes for merging formatters. Instead of big PR 7381 on upstream --- src/common/Formatter.cc | 19 +++++++++++++++++++ src/common/Formatter.h | 16 +++++++++++++--- src/common/HTMLFormatter.cc | 1 - src/common/HTMLFormatter.h | 2 +- src/mds/MDS.cc | 2 +- src/mon/MonitorDBStore.h | 2 +- src/rbd_replay/actions.hpp | 1 - 7 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index 345f95df66dca..34dcc9cf6b530 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -18,6 +18,7 @@ #include "assert.h" #include "Formatter.h" +#include "HTMLFormatter.h" #include "common/escape.h" #include @@ -336,6 +337,24 @@ void XMLFormatter::reset() m_pending_string.str(""); m_sections.clear(); m_pending_string_name.clear(); + m_header_done = false; +} + +void XMLFormatter::output_header() +{ + if(!m_header_done) { + m_header_done = true; + write_raw_data(XMLFormatter::XML_1_DTD);; + if (m_pretty) + m_ss << "\n"; + } +} + +void XMLFormatter::output_footer() +{ + while(!m_sections.empty()) { + close_section(); + } } void XMLFormatter::open_object_section(const char *name) diff --git a/src/common/Formatter.h b/src/common/Formatter.h index aead9c26edb33..d8eef64b730cd 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -92,7 +92,9 @@ namespace ceph { class JSONFormatter : public Formatter { public: JSONFormatter(bool p = false); - + virtual void set_status(const char* status, const char* status_name) {}; + virtual void output_header() {}; + virtual void output_footer() {}; void flush(std::ostream& os); void reset(); virtual void open_array_section(const char *name); @@ -134,6 +136,10 @@ namespace ceph { static const char *XML_1_DTD; XMLFormatter(bool pretty = false); + virtual void set_status(const char* status, const char* status_name) {}; + virtual void output_header(); + virtual void output_footer(); + void flush(std::ostream& os); void reset(); void open_array_section(const char *name); @@ -154,7 +160,7 @@ namespace ceph { void open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs); void open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs); void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs); - private: + protected: void open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs); void finish_pending_string(); void print_spaces(); @@ -165,12 +171,16 @@ namespace ceph { std::deque m_sections; bool m_pretty; std::string m_pending_string_name; + bool m_header_done; }; class TableFormatter : public Formatter { public: TableFormatter(bool keyval = false); - + + virtual void set_status(const char* status, const char* status_name) {}; + virtual void output_header() {}; + virtual void output_footer() {}; void flush(std::ostream& os); void reset(); virtual void open_array_section(const char *name); diff --git a/src/common/HTMLFormatter.cc b/src/common/HTMLFormatter.cc index 35e36a2340798..60d7e5d8415b9 100644 --- a/src/common/HTMLFormatter.cc +++ b/src/common/HTMLFormatter.cc @@ -19,7 +19,6 @@ #include "assert.h" #include "Formatter.h" #include "HTMLFormatter.h" -#include "XMLFormatter.h" #include "common/escape.h" #include diff --git a/src/common/HTMLFormatter.h b/src/common/HTMLFormatter.h index a8c6b20a78b21..de154c7b0d188 100644 --- a/src/common/HTMLFormatter.h +++ b/src/common/HTMLFormatter.h @@ -15,7 +15,7 @@ #include #include "include/buffer.h" -#include "XMLFormatter.h" +#include "Formatter.h" namespace ceph { class HTMLFormatter : public XMLFormatter { diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc index 0dbe97e3e1a63..8481a7017dd20 100644 --- a/src/mds/MDS.cc +++ b/src/mds/MDS.cc @@ -23,7 +23,7 @@ #include "common/signal.h" #include "common/ceph_argparse.h" #include "common/errno.h" -#include "common/JSONFormatter.h" +#include "common/Formatter.h" #include "msg/Messenger.h" #include "mon/MonClient.h" diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h index 58cff24e4156c..2f6f8d8ba8749 100644 --- a/src/mon/MonitorDBStore.h +++ b/src/mon/MonitorDBStore.h @@ -23,7 +23,7 @@ #include "os/KeyValueDB.h" #include "include/assert.h" -#include "common/JSONFormatter.h" +#include "common/Formatter.h" #include "common/Finisher.h" #include "common/errno.h" diff --git a/src/rbd_replay/actions.hpp b/src/rbd_replay/actions.hpp index a80ef95064eab..9a05285ca4f26 100644 --- a/src/rbd_replay/actions.hpp +++ b/src/rbd_replay/actions.hpp @@ -18,7 +18,6 @@ #include #include "include/rbd/librbd.hpp" #include "common/Formatter.h" -#include "common/JSONFormatter.h" #include "rbd_replay/ActionTypes.h" #include "rbd_loc.hpp" #include From 539506db37f5668bc26e0382625476d6be6c97a4 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Mon, 20 Jun 2016 11:54:51 +0000 Subject: [PATCH 15/60] First working code --- src/rgw/rgw_common.h | 5 +++++ src/rgw/rgw_op.cc | 44 +++++++++++++++++++++++++++++++----------- src/rgw/rgw_rest_s3.cc | 20 ------------------- src/rgw/rgw_rest_s3.h | 4 ---- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index aa348c53901b5..8177a455c40d2 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -945,6 +945,11 @@ struct rgw_obj_key { k->instance = instance; } + void dss_duplicate(rgw_obj_key* k) { + name = k->name; + instance = k->instance; + } + void set(const string& n) { name = n; instance.clear(); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index c02c2488eaa08..c407ebed34d26 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3533,38 +3533,60 @@ void RGWRenameObj::pre_exec() void RGWRenameObj::execute() { ret = 0; - string orig_object = (s->object).name; + rgw_obj_key orig_object; + orig_object.dss_duplicate(&(s->object)); (s->object).name = s->info.args.get("newname"); string copysource = s->bucket_name_str; copysource.append("/"); - copysource.append(orig_object); + copysource.append(orig_object.name); ldout(s->cct, 0) << "DSS INFO: Converting to copy request. s->object: " << (s->object).name << ". Copy source: " << copysource << dendl; s->info.env->set("HTTP_X_JCS_COPY_SOURCE", copysource.c_str()); s->info.env->set("HTTP_X_JCS_METADATA_DIRECTIVE", "COPY"); - + s->copy_source = s->info.env->get("HTTP_X_JCS_COPY_SOURCE"); RGWCopyObj_ObjStore_S3* copy_op = new RGWCopyObj_ObjStore_S3; - copy_op->dss_set_req_state(s); - copy_op->dss_set_store(store); + if (s->copy_source) { + ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object); + if (!ret) { + ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; + return; + } + /*ret = validate_bucket_name(s->src_bucket_name, bucket_name_strictness_value); + if (ret) { + ldout(s->cct, 0) << "DSS INFO: Rename op bucket name not valid" << dendl; + return; + } + ret = validate_object_name(s->src_object.name); + if (ret) { + ldout(s->cct, 0) << "DSS INFO: Rename op object name not valid" << dendl; + return; + }*/ + } + + copy_op->init(store, s, dialect_handler); + copy_op->verify_op_mask(); + copy_op->verify_permission(); + copy_op->verify_params(); copy_op->init_processing(); s->system_request = true; copy_op->pre_exec(); copy_op->execute(); - copy_op->complete(); ldout(s->cct, 0) << "DSS INFO: Rename op copy done" << dendl; -/* s->system_request = false; - (s->object).name = orig_object; + s->system_request = false; + (s->object).dss_duplicate(&orig_object); ldout(s->cct, 0) << "DSS INFO: Rearraging things to continue with delete. s->object: " << (s->object).name << dendl; RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; - del_op->dss_set_req_state(s); - del_op->dss_set_store(store); + del_op->init(store, s, dialect_handler); + del_op->verify_permission(); + del_op->verify_params(); + del_op->verify_op_mask(); del_op->init_processing(); s->system_request = true; del_op->pre_exec(); del_op->execute(); - ldout(s->cct, 0) << "DSS INFO: Rename op delete performed" << dendl;*/ + ldout(s->cct, 0) << "DSS INFO: Rename op delete performed" << dendl; s->system_request = false; return; } diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index a19b79ceb89fc..a3d8d77183ac1 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1468,16 +1468,6 @@ void RGWPostObj_ObjStore_S3::send_response() rgw_flush_formatter_and_reset(s, s->formatter); } -void RGWDeleteObj_ObjStore_S3::dss_set_req_state(req_state* orig_s) -{ - s = orig_s; -} - -void RGWDeleteObj_ObjStore_S3::dss_set_store(RGWRados* orig_store) -{ - store = orig_store; -} - void RGWDeleteObj_ObjStore_S3::send_response() { int r = ret; @@ -1511,16 +1501,6 @@ int RGWCopyObj_ObjStore_S3::init_dest_policy() return 0; } -void RGWCopyObj_ObjStore_S3::dss_set_req_state(req_state* orig_s) -{ - s = orig_s; -} - -void RGWCopyObj_ObjStore_S3::dss_set_store(RGWRados* orig_store) -{ - store = orig_store; -} - int RGWCopyObj_ObjStore_S3::get_params() { if_mod = s->info.env->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE"); diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 1f1729ce12738..756af24aefd94 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -173,8 +173,6 @@ class RGWDeleteObj_ObjStore_S3 : public RGWDeleteObj_ObjStore { ~RGWDeleteObj_ObjStore_S3() {} void send_response(); - void dss_set_req_state(req_state* orig_s); - void dss_set_store(RGWRados* orig_store); }; class RGWCopyObj_ObjStore_S3 : public RGWCopyObj_ObjStore { @@ -187,8 +185,6 @@ class RGWCopyObj_ObjStore_S3 : public RGWCopyObj_ObjStore { int get_params(); void send_partial_response(off_t ofs); void send_response(); - void dss_set_req_state(req_state* orig_s); - void dss_set_store(RGWRados* orig_store); }; class RGWRenameObj_ObjStore_S3 : public RGWRenameObj_ObjStore { From 5aa0b04f41078236ee24403b207e6bd5be99953c Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 19 Jan 2016 15:33:48 -0800 Subject: [PATCH 16/60] rgw: init_permissions() shouldn't return ENOENT in a specific case If op is bucket creation it should ignore ENOENT. Otherwise we should make sure we return the appropriate error. Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_main.cc | 2 +- src/rgw/rgw_op.cc | 2 +- src/rgw/rgw_op.h | 2 +- src/rgw/rgw_rest.cc | 5 ++++- src/rgw/rgw_rest.h | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 2749e68ad61e9..fed09653a77e4 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -635,7 +635,7 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC } req->log(s, "init permissions"); - ret = handler->init_permissions(); + ret = handler->init_permissions(op); if (ret < 0) { abort_early(s, op, ret, handler); goto done; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 005490dc67415..c9da0c0e25ab1 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -423,7 +423,7 @@ static int rgw_build_bucket_policies(RGWRados *store, struct req_state *s) } } - return 0; + return ret; } /** diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 6753b6c6abdeb..3ec58011f5f0c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -1107,7 +1107,7 @@ class RGWHandler { virtual int validate_bucket_name(const string& bucket, int name_strictness) { return 0; } virtual RGWOp *get_op(RGWRados *store); virtual void put_op(RGWOp *op); - virtual int init_permissions() { + virtual int init_permissions(RGWOp *op) { return 0; } virtual int retarget(RGWOp *op, RGWOp **new_op) { diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 6491cc6440ebd..f9c00421bf68b 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -1372,8 +1372,11 @@ static http_op op_from_method(const char *method) return OP_UNKNOWN; } -int RGWHandler_ObjStore::init_permissions() +int RGWHandler_ObjStore::init_permissions(RGWOp *op) { + if (op->get_type() == RGW_OP_CREATE_BUCKET) + return 0; + return do_init_permissions(); } diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index d3287474b9bcc..ff2af52a90d6f 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -303,7 +303,7 @@ class RGWHandler_ObjStore : public RGWHandler { public: RGWHandler_ObjStore() {} virtual ~RGWHandler_ObjStore() {} - int init_permissions(); + int init_permissions(RGWOp *op); int read_permissions(RGWOp *op); virtual int retarget(RGWOp *op, RGWOp **new_op) { *new_op = op; From 867afa4d054bb2099f1a8cba49c600bc11e9b330 Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Thu, 7 Jan 2016 01:35:16 +0000 Subject: [PATCH 17/60] S3Website: AmazonS3 changed RedirectAll HTTP response. Signed-off-by: Robin H. Johnson --- src/rgw/rgw_website.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_website.cc b/src/rgw/rgw_website.cc index eee8977a86f94..a69ffe16d2981 100644 --- a/src/rgw/rgw_website.cc +++ b/src/rgw/rgw_website.cc @@ -94,7 +94,7 @@ bool RGWBucketWebsiteConf::should_redirect(const string& key, const int http_err if(!redirect_all.hostname.empty()) { RGWBWRoutingRule redirect_all_rule; redirect_all_rule.redirect_info.redirect = redirect_all; - redirect_all.http_redirect_code = 302; + redirect_all.http_redirect_code = 301; *redirect = redirect_all_rule; return true; } else if (!routing_rules.check_key_and_error_code_condition(key, http_error_code, &rule)) { From a9145e0e4ac048948c8d1a5343e3453b27f7e760 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Mon, 20 Jun 2016 15:34:54 +0000 Subject: [PATCH 18/60] Minor fixes for better error handling --- src/common/config_opts.h | 1 + src/rgw/rgw_common.h | 1 + src/rgw/rgw_http_errors.h | 1 + src/rgw/rgw_rest_s3.cc | 22 ++++++++++++++++------ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 5c445c5e99725..2cf919b0ddbbe 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1077,6 +1077,7 @@ OPTION(rgw_keystone_token_api, OPT_STR, "v3/token-auth") // api to validate tok OPTION(rgw_keystone_url_token_api, OPT_STR, "url-auth") // api to validate presigned token URL based authentication requests OPTION(rgw_keystone_infinite_url_token_api, OPT_STR, "preauth-token-auth") // api to validate infinite time presigned token URL OPTION(dss_regional_url, OPT_STR, "https://dss.ind-west-1.staging.jiocloudservices.com") // URL to be returned in XMLNS during anonymous list all buckets calls +OPTION(rgw_enable_rename_op, OPT_BOOL, true) // enable the atomic rename op OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf counter diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 8177a455c40d2..e8a5893e248d7 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -146,6 +146,7 @@ using ceph::crypto::MD5; #define ERR_SIGNATURE_NO_MATCH 2027 #define ERR_INVALID_ACCESS_KEY 2028 #define ERR_BUCKET_ALREADY_OWNED 2029 +#define ERR_BAD_RENAME_REQ 2030 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index 34b5f0b33dbc7..f80ef13815539 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -45,6 +45,7 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_USER_SUSPENDED, 403, "UserSuspended" }, { ERR_REQUEST_TIME_SKEWED, 403, "RequestTimeTooSkewed" ,"The difference between the request time and the server's time is too large."}, { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" }, + { ERR_BAD_RENAME_REQ, 403, "Rename request must have object name, new object name and the HTTP method should be PUT." }, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index a3d8d77183ac1..1022fdd03d9d8 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -2129,7 +2129,7 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_put() if (is_acl_op()) { return new RGWPutACLs_ObjStore_S3; } - if (is_rename_op()) { + if (is_rename_op() && store->ctx()->_conf->rgw_enable_rename_op) { return new RGWRenameObj_ObjStore_S3; } if (!s->copy_source) @@ -2740,6 +2740,17 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) return -EPERM; } + // Block rename op for illegal cases + if ((store->ctx()->_conf->rgw_enable_rename_op) && (s->info.args.exists("newname"))) { + if (s->op == OP_PUT) { + if ((s->object).name.empty()) { + return -ERR_BAD_RENAME_REQ; + } + } else { + return -ERR_BAD_RENAME_REQ; + } + } + /* neither keystone and rados enabled; warn and exit! */ if (!store->ctx()->_conf->rgw_s3_auth_use_rados && !store->ctx()->_conf->rgw_s3_auth_use_keystone) { @@ -2769,10 +2780,9 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) qsr = true; } else { /* anonymous access */ - //<<<<<< You will hit here for sign based req - //<<<<<< Add changes for anonymous access. Call a func from here. - init_anon_user(s); - return 0; + return -EPERM; + //init_anon_user(s); + //return 0; } } else { // strncmp returns 0 on match. If even one of AWS or JCS match, dont return -EINVAL. @@ -2842,7 +2852,7 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) "", /* Received signature */ resource_object_name, iamerror); - + } else { keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), From 9242aa45d09ef3ee96c214a3cde1b8972e3d534f Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Mon, 20 Jun 2016 16:15:21 +0000 Subject: [PATCH 19/60] Fixing return code --- src/rgw/rgw_rest_s3.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 1022fdd03d9d8..09e6adba7c4a3 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1620,12 +1620,12 @@ int RGWPutACLs_ObjStore_S3::get_policy_from_state(RGWRados *store, struct req_st void RGWRenameObj_ObjStore_S3::send_response() { + ret = 0; if (ret) set_req_state_err(s, ret); dump_errno(s); end_header(s, this, "application/xml"); dump_start(s); - return; } void RGWPutACLs_ObjStore_S3::send_response() From 999e8f93cce71bc884ea7def6fc9583637c23cba Mon Sep 17 00:00:00 2001 From: "Javier M. Mellid" Date: Wed, 21 Oct 2015 16:04:14 +0200 Subject: [PATCH 20/60] rgw: add explicit success/error paths in RGWGetObj::execute() Fixes: #12352 Signed-off-by: Javier M. Mellid Conflicts: src/rgw/rgw_rest_swift.cc src/rgw/rgw_rest_swift.h --- src/rgw/rgw_op.cc | 5 ++++- src/rgw/rgw_op.h | 1 + src/rgw/rgw_rest_s3.cc | 6 ++++++ src/rgw/rgw_rest_s3.h | 1 + src/rgw/rgw_rest_swift.cc | 13 +++++++++++++ src/rgw/rgw_rest_swift.h | 2 ++ 6 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index c9da0c0e25ab1..96f1c525adf79 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -958,8 +958,11 @@ void RGWGetObj::execute() goto done_err; } -done_err: send_response_data(bl, 0, 0); + return; + +done_err: + send_response_data_error(); } int RGWGetObj::init_common() diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 3ec58011f5f0c..4b3df0a8d211c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -175,6 +175,7 @@ class RGWGetObj : public RGWOp { int get_data_cb(bufferlist& bl, off_t ofs, off_t len); virtual int get_params() = 0; + virtual int send_response_data_error() = 0; virtual int send_response_data(bufferlist& bl, off_t ofs, off_t len) = 0; virtual const string name() { return "get_obj"; } diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 41cd33f2e654a..0a9150091808b 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -75,6 +75,12 @@ static struct response_attr_param resp_attr_params[] = { {NULL, NULL}, }; +int RGWGetObj_ObjStore_S3::send_response_data_error() +{ + bufferlist bl; + return send_response_data(bl, 0 , 0); +} + int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { const char *content_type = NULL; diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 135fa39704290..d0ab281c42a73 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -24,6 +24,7 @@ class RGWGetObj_ObjStore_S3 : public RGWGetObj_ObjStore RGWGetObj_ObjStore_S3() {} ~RGWGetObj_ObjStore_S3() {} + int send_response_data_error(); int send_response_data(bufferlist& bl, off_t ofs, off_t len); }; diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index bcbd45002c896..e5b5412035796 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -692,6 +692,19 @@ void RGWCopyObj_ObjStore_SWIFT::send_response() } } +int RGWGetObj_ObjStore_SWIFT::get_params() +{ + const string& mm = s->info.args.get("multipart-manifest"); + + return RGWGetObj_ObjStore::get_params(); +} + +int RGWGetObj_ObjStore_SWIFT::send_response_data_error() +{ + bufferlist bl; + return send_response_data(bl, 0, 0); +} + int RGWGetObj_ObjStore_SWIFT::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { string content_type; diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h index 19028ab337b43..ebd0d33d01a5c 100644 --- a/src/rgw/rgw_rest_swift.h +++ b/src/rgw/rgw_rest_swift.h @@ -13,6 +13,8 @@ class RGWGetObj_ObjStore_SWIFT : public RGWGetObj_ObjStore { RGWGetObj_ObjStore_SWIFT() {} ~RGWGetObj_ObjStore_SWIFT() {} + int get_params(); + int send_response_data_error(); int send_response_data(bufferlist& bl, off_t ofs, off_t len); }; From bcf8ae6281ea55ee5eb085133dc141a1631a736e Mon Sep 17 00:00:00 2001 From: Gaurav Bafna Date: Tue, 21 Jun 2016 18:51:30 +1100 Subject: [PATCH 21/60] WIP x-amz-website-redirect-location . Backported from 11913b964220e7435f8ce17ef19469ec519cca97 --- src/rgw/rgw_main.cc | 17 ++++++++- src/rgw/rgw_op.cc | 29 +++++++++++++- src/rgw/rgw_op.h | 73 ++++++++++-------------------------- src/rgw/rgw_rest.cc | 5 +++ src/rgw/rgw_rest_s3.cc | 20 ++++++++++ src/rgw/rgw_rest_s3website.h | 4 +- 6 files changed, 91 insertions(+), 57 deletions(-) diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index fed09653a77e4..d75911e6a7021 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -694,9 +694,24 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC goto done; } - req->log(s, "executing"); + req->log(s, "pre-executing"); op->pre_exec(); + ret = op->get_ret(); + if (ret < 0) { + dout(2) << "pre_exec ret=" << ret << dendl; + abort_early(s, op, ret, handler); + goto done; + } + + req->log(s, "executing"); op->execute(); + ret = op->get_ret(); + if (ret < 0) { + dout(2) << "execute ret=" << ret << dendl; + abort_early(s, op, ret, handler); + goto done; + } + req->log(s, "completing"); op->complete(); done: int r = client_io->complete_request(); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 96f1c525adf79..70e80de1213e1 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -887,6 +887,7 @@ int RGWGetObj::get_data_cb(bufferlist& bl, off_t bl_ofs, off_t bl_len) void RGWGetObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWGetObj::execute() @@ -953,12 +954,15 @@ void RGWGetObj::execute() ret = read_op.iterate(ofs, end, &cb); perfcounter->tinc(l_rgw_get_lat, - (ceph_clock_now(s->cct) - start_time)); + (ceph_clock_now(s->cct) - start_time)); if (ret < 0) { goto done_err; } - send_response_data(bl, 0, 0); + ret = send_response_data(bl, 0, 0); + if (ret < 0) { + goto done_err; + } return; done_err: @@ -1096,6 +1100,7 @@ int RGWGetBucketVersioning::verify_permission() void RGWGetBucketVersioning::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWGetBucketVersioning::execute() @@ -1115,6 +1120,7 @@ int RGWSetBucketVersioning::verify_permission() void RGWSetBucketVersioning::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWSetBucketVersioning::execute() @@ -1149,6 +1155,7 @@ int RGWGetBucketWebsite::verify_permission() void RGWGetBucketWebsite::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWGetBucketWebsite::execute() @@ -1169,6 +1176,7 @@ int RGWSetBucketWebsite::verify_permission() void RGWSetBucketWebsite::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWSetBucketWebsite::execute() @@ -1199,6 +1207,7 @@ int RGWDeleteBucketWebsite::verify_permission() void RGWDeleteBucketWebsite::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWDeleteBucketWebsite::execute() @@ -1224,6 +1233,7 @@ int RGWStatBucket::verify_permission() void RGWStatBucket::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWStatBucket::execute() @@ -1276,6 +1286,7 @@ int RGWListBucket::parse_max_keys() void RGWListBucket::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWListBucket::execute() @@ -1381,6 +1392,7 @@ void RGWCreateBucket::pre_exec() ret = dialect_handler->validate_bucket_name(s->bucket_name_str, s->cct->_conf->rgw_s3_bucket_name_create_strictness); rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWCreateBucket::execute() @@ -1528,6 +1540,7 @@ int RGWDeleteBucket::verify_permission() void RGWDeleteBucket::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWDeleteBucket::execute() @@ -1761,6 +1774,7 @@ void RGWPutObj::dispose_processor(RGWPutObjProcessor *processor) void RGWPutObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } static int put_data_and_throttle(RGWPutObjProcessor *processor, bufferlist& data, off_t ofs, @@ -2052,6 +2066,7 @@ void RGWPostObj::dispose_processor(RGWPutObjProcessor *processor) void RGWPostObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWPostObj::execute() @@ -2295,6 +2310,7 @@ int RGWDeleteObj::verify_permission() void RGWDeleteObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWDeleteObj::execute() @@ -2500,6 +2516,7 @@ void RGWCopyObj::progress_cb(off_t ofs) void RGWCopyObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWCopyObj::execute() @@ -2558,6 +2575,7 @@ int RGWGetACLs::verify_permission() void RGWGetACLs::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWGetACLs::execute() @@ -2588,6 +2606,7 @@ int RGWPutACLs::verify_permission() void RGWPutACLs::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWPutACLs::execute() @@ -2853,6 +2872,7 @@ int RGWInitMultipart::verify_permission() void RGWInitMultipart::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWInitMultipart::execute() @@ -3049,6 +3069,7 @@ int RGWCompleteMultipart::verify_permission() void RGWCompleteMultipart::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWCompleteMultipart::execute() @@ -3248,6 +3269,7 @@ int RGWAbortMultipart::verify_permission() void RGWAbortMultipart::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWAbortMultipart::execute() @@ -3349,6 +3371,7 @@ int RGWListMultipart::verify_permission() void RGWListMultipart::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWListMultipart::execute() @@ -3382,6 +3405,7 @@ int RGWListBucketMultiparts::verify_permission() void RGWListBucketMultiparts::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWListBucketMultiparts::execute() @@ -3442,6 +3466,7 @@ int RGWDeleteMultiObj::verify_permission() void RGWDeleteMultiObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWDeleteMultiObj::execute() diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 4b3df0a8d211c..58e6b6faf39d8 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -76,6 +76,7 @@ class RGWOp { bool cors_exist; RGWQuotaInfo bucket_quota; RGWQuotaInfo user_quota; + int ret; virtual int init_quota(); public: @@ -114,6 +115,7 @@ class RGWOp { virtual uint32_t op_mask() { return 0; } virtual int error_handler(int err_no, string *error_content); + int get_ret() { return ret; }; }; class RGWGetObj : public RGWOp { @@ -133,7 +135,6 @@ class RGWGetObj : public RGWOp { time_t *mod_ptr; time_t *unmod_ptr; map attrs; - int ret; bool get_data; bool partial_content; rgw_obj obj; @@ -158,7 +159,6 @@ class RGWGetObj : public RGWOp { unmod_ptr = NULL; get_data = false; partial_content = false; - ret = 0; } virtual bool prefetch_data() { return get_data; } @@ -200,14 +200,13 @@ class RGWGetObj_CB : public RGWGetDataCB class RGWListBuckets : public RGWOp { protected: - int ret; bool sent_data; string marker; uint64_t limit; uint64_t limit_max; public: - RGWListBuckets() : ret(0), sent_data(false) { + RGWListBuckets() : sent_data(false) { limit = limit_max = RGW_LIST_BUCKETS_LIMIT_MAX; } @@ -229,7 +228,6 @@ class RGWListBuckets : public RGWOp { class RGWStatAccount : public RGWOp { protected: - int ret; uint32_t buckets_count; uint64_t buckets_objcount; uint64_t buckets_size; @@ -264,7 +262,6 @@ class RGWListBucket : public RGWOp { string encoding_type; bool list_versions; int max; - int ret; vector objs; map common_prefixes; @@ -274,7 +271,7 @@ class RGWListBucket : public RGWOp { int parse_max_keys(); public: - RGWListBucket() : list_versions(false), max(0), ret(0), + RGWListBucket() : list_versions(false), max(0), default_max(0), is_truncated(false) {} int verify_permission(); void pre_exec(); @@ -292,7 +289,7 @@ class RGWGetBucketLogging : public RGWOp { public: RGWGetBucketLogging() {} int verify_permission(); - void execute() {} + void execute() { ret = 0; } virtual void send_response() = 0; virtual const string name() { return "get_bucket_logging"; } @@ -305,7 +302,7 @@ class RGWGetBucketLocation : public RGWOp { RGWGetBucketLocation() {} ~RGWGetBucketLocation() {} int verify_permission(); - void execute() {} + void execute() { ret = 0; } virtual void send_response() = 0; virtual const string name() { return "get_bucket_location"; } @@ -332,9 +329,8 @@ class RGWGetBucketVersioning : public RGWOp { class RGWSetBucketVersioning : public RGWOp { protected: bool enable_versioning; - int ret; public: - RGWSetBucketVersioning() : enable_versioning(false), ret(0) {} + RGWSetBucketVersioning() : enable_versioning(false) {} int verify_permission(); void pre_exec(); @@ -349,10 +345,8 @@ class RGWSetBucketVersioning : public RGWOp { }; class RGWGetBucketWebsite : public RGWOp { -protected: - int ret; public: - RGWGetBucketWebsite() : ret(0) {} + RGWGetBucketWebsite() {} int verify_permission(); void pre_exec(); @@ -366,10 +360,9 @@ class RGWGetBucketWebsite : public RGWOp { class RGWSetBucketWebsite : public RGWOp { protected: - int ret; RGWBucketWebsiteConf website_conf; public: - RGWSetBucketWebsite() : ret(0) {} + RGWSetBucketWebsite() {} int verify_permission(); void pre_exec(); @@ -384,10 +377,8 @@ class RGWSetBucketWebsite : public RGWOp { }; class RGWDeleteBucketWebsite : public RGWOp { -protected: - int ret; public: - RGWDeleteBucketWebsite() : ret(0) {} + RGWDeleteBucketWebsite() {} int verify_permission(); void pre_exec(); @@ -401,11 +392,10 @@ class RGWDeleteBucketWebsite : public RGWOp { class RGWStatBucket : public RGWOp { protected: - int ret; RGWBucketEnt bucket; public: - RGWStatBucket() : ret(0) {} + RGWStatBucket() {} ~RGWStatBucket() {} int verify_permission(); @@ -420,7 +410,6 @@ class RGWStatBucket : public RGWOp { class RGWCreateBucket : public RGWOp { protected: - int ret; RGWAccessControlPolicy policy; string location_constraint; string placement_rule; @@ -432,7 +421,7 @@ class RGWCreateBucket : public RGWOp { bufferlist in_data; public: - RGWCreateBucket() : ret(0), has_cors(false) {} + RGWCreateBucket() : has_cors(false) {} int verify_permission(); void pre_exec(); @@ -450,12 +439,10 @@ class RGWCreateBucket : public RGWOp { class RGWDeleteBucket : public RGWOp { protected: - int ret; - RGWObjVersionTracker objv_tracker; public: - RGWDeleteBucket() : ret(0) {} + RGWDeleteBucket() {} int verify_permission(); void pre_exec(); @@ -472,7 +459,6 @@ class RGWPutObj : public RGWOp { friend class RGWPutObjProcessor; protected: - int ret; off_t ofs; const char *supplied_md5_b64; const char *supplied_etag; @@ -491,7 +477,6 @@ class RGWPutObj : public RGWOp { public: RGWPutObj() { - ret = 0; ofs = 0; supplied_md5_b64 = NULL; supplied_etag = NULL; @@ -531,7 +516,6 @@ class RGWPostObj : public RGWOp { protected: off_t min_len; off_t max_len; - int ret; int len; off_t ofs; const char *supplied_md5_b64; @@ -544,7 +528,7 @@ class RGWPostObj : public RGWOp { map attrs; public: - RGWPostObj() : min_len(0), max_len(LLONG_MAX), ret(0), len(0), ofs(0), + RGWPostObj() : min_len(0), max_len(LLONG_MAX), len(0), ofs(0), supplied_md5_b64(NULL), supplied_etag(NULL), data_pending(false) {} @@ -570,7 +554,6 @@ class RGWPostObj : public RGWOp { class RGWPutMetadata : public RGWOp { protected: - int ret; set rmattr_names; bool has_policy, has_cors; RGWAccessControlPolicy policy; @@ -605,7 +588,6 @@ class RGWSetTempUrl : public RGWOp { map temp_url_keys; public: RGWSetTempUrl() : ret(0) {} - int verify_permission(); void execute(); @@ -617,12 +599,11 @@ class RGWSetTempUrl : public RGWOp { class RGWDeleteObj : public RGWOp { protected: - int ret; bool delete_marker; string version_id; public: - RGWDeleteObj() : ret(0), delete_marker(false) {} + RGWDeleteObj() : delete_marker(false) {} int verify_permission(); void pre_exec(); @@ -648,7 +629,6 @@ class RGWCopyObj : public RGWOp { time_t unmod_time; time_t *mod_ptr; time_t *unmod_ptr; - int ret; map attrs; string src_bucket_name; rgw_bucket src_bucket; @@ -717,11 +697,10 @@ class RGWCopyObj : public RGWOp { class RGWGetACLs : public RGWOp { protected: - int ret; string acls; public: - RGWGetACLs() : ret(0) {} + RGWGetACLs() {} int verify_permission(); void pre_exec(); @@ -735,7 +714,6 @@ class RGWGetACLs : public RGWOp { class RGWPutACLs : public RGWOp { protected: - int ret; size_t len; char *data; ACLOwner owner; @@ -764,10 +742,9 @@ class RGWPutACLs : public RGWOp { class RGWGetCORS : public RGWOp { protected: - int ret; public: - RGWGetCORS() : ret(0) {} + RGWGetCORS() {} int verify_permission(); void execute(); @@ -780,7 +757,6 @@ class RGWGetCORS : public RGWOp { class RGWPutCORS : public RGWOp { protected: - int ret; bufferlist cors_bl; public: @@ -801,10 +777,9 @@ class RGWPutCORS : public RGWOp { class RGWDeleteCORS : public RGWOp { protected: - int ret; public: - RGWDeleteCORS() : ret(0) {} + RGWDeleteCORS() {} int verify_permission(); void execute(); @@ -817,12 +792,11 @@ class RGWDeleteCORS : public RGWOp { class RGWOptionsCORS : public RGWOp { protected: - int ret; RGWCORSRule *rule; const char *origin, *req_hdrs, *req_meth; public: - RGWOptionsCORS() : ret(0), rule(NULL), origin(NULL), + RGWOptionsCORS() : rule(NULL), origin(NULL), req_hdrs(NULL), req_meth(NULL) { } @@ -838,7 +812,6 @@ class RGWOptionsCORS : public RGWOp { class RGWInitMultipart : public RGWOp { protected: - int ret; string upload_id; RGWAccessControlPolicy policy; @@ -864,7 +837,6 @@ class RGWInitMultipart : public RGWOp { class RGWCompleteMultipart : public RGWOp { protected: - int ret; string upload_id; string etag; char *data; @@ -893,10 +865,8 @@ class RGWCompleteMultipart : public RGWOp { class RGWAbortMultipart : public RGWOp { protected: - int ret; - public: - RGWAbortMultipart() : ret(0) {} + RGWAbortMultipart() {} int verify_permission(); void pre_exec(); @@ -910,7 +880,6 @@ class RGWAbortMultipart : public RGWOp { class RGWListMultipart : public RGWOp { protected: - int ret; string upload_id; map parts; int max_parts; @@ -1019,7 +988,6 @@ class RGWListBucketMultiparts : public RGWOp { RGWMultipartUploadEntry next_marker; int max_uploads; string delimiter; - int ret; vector uploads; map common_prefixes; bool is_truncated; @@ -1051,7 +1019,6 @@ class RGWListBucketMultiparts : public RGWOp { class RGWDeleteMultiObj : public RGWOp { protected: - int ret; int max_to_delete; size_t len; char *data; diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index f9c00421bf68b..ef5d9e2a4c2ea 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -44,6 +44,11 @@ static struct rgw_http_attr rgw_to_http_attr_list[] = { { RGW_ATTR_CONTENT_ENC, "Content-Encoding"}, { RGW_ATTR_USER_MANIFEST, "X-Object-Manifest"}, { RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION, "Location"}, + /* RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION header depends on access mode: + * S3 endpoint: x-amz-website-redirect-location + * S3Website endpoint: Location + */ + { RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION, "x-amz-website-redirect-location" }, { NULL, NULL}, }; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 0a9150091808b..d83fe069d34e8 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -75,6 +75,26 @@ static struct response_attr_param resp_attr_params[] = { {NULL, NULL}, }; +int RGWGetObj_ObjStore_S3Website::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { + map::iterator iter; + iter = attrs.find(RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION); + if (iter != attrs.end()) { + bufferlist &bl = iter->second; + s->redirect = string(bl.c_str(), bl.length()); + s->err.http_ret = 301; + ldout(s->cct, 20) << __CEPH_ASSERT_FUNCTION << " redirectng per x-dss-website-redirect-location=" << s->redirect << dendl; + ret = -ERR_WEBSITE_REDIRECT; + return ret; + } else { + return RGWGetObj_ObjStore_S3::send_response_data(bl, bl_ofs, bl_len); + } +} + +int RGWGetObj_ObjStore_S3Website::send_response_data_error() +{ + return RGWGetObj_ObjStore_S3::send_response_data_error(); +} + int RGWGetObj_ObjStore_S3::send_response_data_error() { bufferlist bl; diff --git a/src/rgw/rgw_rest_s3website.h b/src/rgw/rgw_rest_s3website.h index 18f00ee99e55d..06a22ccda22d7 100644 --- a/src/rgw/rgw_rest_s3website.h +++ b/src/rgw/rgw_rest_s3website.h @@ -75,6 +75,8 @@ class RGWGetObj_ObjStore_S3Website : public RGWGetObj_ObjStore_S3 RGWGetObj_ObjStore_S3Website() : is_errordoc_request(false) {} RGWGetObj_ObjStore_S3Website(bool is_errordoc_request) : is_errordoc_request(false) { this->is_errordoc_request = is_errordoc_request; } ~RGWGetObj_ObjStore_S3Website() {} + int send_response_data_error(); + int send_response_data(bufferlist& bl, off_t ofs, off_t len); // We override RGWGetObj_ObjStore::get_params here, to allow ignoring all // conditional params for error pages. int get_params() { @@ -84,7 +86,7 @@ class RGWGetObj_ObjStore_S3Website : public RGWGetObj_ObjStore_S3 if_unmod = NULL; if_match = NULL; if_nomatch = NULL; - return 0; + return 0; } else { return RGWGetObj_ObjStore_S3::get_params(); } From 18e83c119133b9fa162f5fa488c5fcd141d015ed Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 19 Jan 2016 16:25:59 -0800 Subject: [PATCH 22/60] rgw: a few minor cleanups delete getop, remove #warnings, add comments Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_rest.cc | 9 ++++++++- src/rgw/rgw_rest_s3.cc | 23 ++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index ef5d9e2a4c2ea..99b5fbf201e60 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -742,7 +742,14 @@ void abort_early(struct req_state *s, RGWOp *op, int err_no, RGWHandler* handler } if (!error_content.empty()) { ldout(s->cct, 20) << "error_content is set, we need to serve it INSTEAD of firing the formatter" << dendl; -#warning TODO we must add all error entries as headers here + /* + * FIXME we must add all error entries as headers here: + * when having a working errordoc, then the s3 error fields are rendered as HTTP headers, e.g.: + * + * x-amz-error-code: NoSuchKey + * x-amz-error-message: The specified key does not exist. + * x-amz-error-detail-Key: foo + */ end_header(s, op, NULL, NO_CONTENT_LENGTH, false, true); s->cio->write(error_content.c_str(), error_content.size()); s->formatter->reset(); diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index d83fe069d34e8..39dab8ce0f163 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3184,14 +3184,10 @@ int RGWHandler_ObjStore_S3Website::retarget(RGWOp *op, RGWOp **new_op) { return -ERR_WEBSITE_REDIRECT; } -#warning FIXME -#if 0 - if (s->object.empty() != new_obj.empty()) { - op->put(); - s->object = new_obj; - *new_op = get_op(); - } -#endif + /* + * FIXME: if s->object != new_obj, drop op and create a new op to handle operation. Or + * remove this comment if it's not applicable anymore + */ s->object = new_obj; @@ -3214,7 +3210,9 @@ int RGWHandler_ObjStore_S3Website::get_errordoc(const string errordoc_key, strin // 1. Check if errordoc exists // 2. Check if errordoc is public // 3. Fetch errordoc content -#warning 2015119: FIXME need to clear all + /* + * FIXME maybe: need to make sure all of the fields for conditional requests are cleared + */ RGWGetObj_ObjStore_S3Website *getop = new RGWGetObj_ObjStore_S3Website(true); getop->set_get_data(true); getop->init(store, s, this); @@ -3230,10 +3228,13 @@ int RGWHandler_ObjStore_S3Website::get_errordoc(const string errordoc_key, strin int64_t ofs = 0; int64_t end = -1; ret = read_op.prepare(&ofs, &end); - if (ret < 0) - return ret; + if (ret < 0) { + goto done; + } ret = read_op.iterate(ofs, end, &cb); // FIXME: need to know the final size? +done: + delete getop; return ret; } From 14962134d5e6dcd62b5b7c435463edc341c1293e Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 20 Jan 2016 10:32:15 -0800 Subject: [PATCH 23/60] rgw: client io shouldn't try to write zero length buffer Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_client_io.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rgw/rgw_client_io.cc b/src/rgw/rgw_client_io.cc index 1f8b803a3d405..b6ef745c4f275 100644 --- a/src/rgw/rgw_client_io.cc +++ b/src/rgw/rgw_client_io.cc @@ -51,6 +51,10 @@ int RGWClientIO::print(const char *format, ...) int RGWClientIO::write(const char *buf, int len) { + if (len == 0) { + return 0; + } + int ret = write_data(buf, len); if (ret < 0) return ret; From 9eadab8b31be8f56e5187a356e1c654ad2157844 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 20 Jan 2016 11:52:48 -0800 Subject: [PATCH 24/60] rgw: fix create bucket error handling Signed-off-by: Yehuda Sadeh Conflicts: src/rgw/rgw_op.cc --- src/rgw/rgw_op.cc | 3 ++- src/rgw/rgw_op.h | 4 +++- src/rgw/rgw_rest_swift.cc | 7 ++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 70e80de1213e1..af85c6137924e 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1525,7 +1525,8 @@ void RGWCreateBucket::execute() ldout(s->cct, 0) << "WARNING: failed to unlink bucket: ret=" << ret << dendl; } } else if (ret == -EEXIST || (ret == 0 && existed)) { - ret = -ERR_BUCKET_EXISTS; + ret = 0; + exist_ret = -ERR_BUCKET_EXISTS; } } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 58e6b6faf39d8..ddd3967697f70 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -420,8 +420,10 @@ class RGWCreateBucket : public RGWOp { bufferlist in_data; + int exist_ret; + public: - RGWCreateBucket() : has_cors(false) {} + RGWCreateBucket() : has_cors(false), exist_ret(0) {} int verify_permission(); void pre_exec(); diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index e5b5412035796..d9e9190ca5308 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -399,10 +399,11 @@ int RGWCreateBucket_ObjStore_SWIFT::get_params() void RGWCreateBucket_ObjStore_SWIFT::send_response() { - if (!ret) - ret = STATUS_CREATED; - else if (ret == -ERR_BUCKET_EXISTS) + if (exist_ret == -ERR_BUCKET_EXISTS) { ret = STATUS_ACCEPTED; + } else if (!ret) { + ret = STATUS_CREATED; + } set_req_state_err(s, ret); dump_errno(s); /* Propose ending HTTP header with 0 Content-Length header. */ From b62310695854decfc944773d73dddb6b1f659831 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 21 Jan 2016 16:35:29 -0800 Subject: [PATCH 25/60] rgw: add a configurable to enable/disable static website rgw_enable_static_website Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 2 ++ src/rgw/rgw_rest_s3.cc | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index db2ebb57eb89e..8086c4ffe6c1a 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1079,6 +1079,8 @@ OPTION(rgw_keystone_url_token_api, OPT_STR, "url-auth") // api to validate pre OPTION(rgw_keystone_infinite_url_token_api, OPT_STR, "preauth-token-auth") // api to validate infinite time presigned token URL OPTION(dss_regional_url, OPT_STR, "https://dss.ind-west-1.staging.jiocloudservices.com") // URL to be returned in XMLNS during anonymous list all buckets calls +OPTION(rgw_enable_static_website, OPT_BOOL, false) // enable static website feature + OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf counter diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 39dab8ce0f163..1907f0b796f28 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -2146,8 +2146,12 @@ RGWOp *RGWHandler_ObjStore_Bucket_S3::op_get() if (s->info.args.sub_resource_exists("versioning")) return new RGWGetBucketVersioning_ObjStore_S3; - if (s->info.args.sub_resource_exists("website")) + if (s->info.args.sub_resource_exists("website")) { + if (!s->cct->_conf->rgw_enable_static_website) { + return NULL; + } return new RGWGetBucketWebsite_ObjStore_S3; + } if (is_acl_op()) { return new RGWGetACLs_ObjStore_S3; @@ -2175,8 +2179,12 @@ RGWOp *RGWHandler_ObjStore_Bucket_S3::op_put() return NULL; if (s->info.args.sub_resource_exists("versioning")) return new RGWSetBucketVersioning_ObjStore_S3; - if (s->info.args.sub_resource_exists("website")) + if (s->info.args.sub_resource_exists("website")) { + if (!s->cct->_conf->rgw_enable_static_website) { + return NULL; + } return new RGWSetBucketWebsite_ObjStore_S3; + } if (is_acl_op()) { return new RGWPutACLs_ObjStore_S3; } else if (is_cors_op()) { @@ -2191,8 +2199,12 @@ RGWOp *RGWHandler_ObjStore_Bucket_S3::op_delete() return new RGWDeleteCORS_ObjStore_S3; } - if (s->info.args.sub_resource_exists("website")) + if (s->info.args.sub_resource_exists("website")) { + if (!s->cct->_conf->rgw_enable_static_website) { + return NULL; + } return new RGWDeleteBucketWebsite_ObjStore_S3; + } return new RGWDeleteBucket_ObjStore_S3; } From 661c8b97f6b40eaf9698215dda0ef2908b63d1aa Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Sun, 17 Apr 2016 08:23:23 -0700 Subject: [PATCH 26/60] rgw/s3website: Fix x-amz-website-redirect-location support. Support for the x-amz-website-redirect-location header was broken in the feature merging of Jewel, as it previously depended on the error handler to redirect, which was overkill. Simplify it to work. Backport: jewel Fixes: http://tracker.ceph.com/issues/15531 Signed-off-by: Robin H. Johnson Conflicts: src/rgw/rgw_rest_s3.cc --- src/rgw/rgw_rest_s3.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 1907f0b796f28..181fa982774b5 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -84,6 +84,11 @@ int RGWGetObj_ObjStore_S3Website::send_response_data(bufferlist& bl, off_t bl_of s->err.http_ret = 301; ldout(s->cct, 20) << __CEPH_ASSERT_FUNCTION << " redirectng per x-dss-website-redirect-location=" << s->redirect << dendl; ret = -ERR_WEBSITE_REDIRECT; + set_req_state_err(s, ret); + dump_errno(s); + dump_content_length(s, 0); + dump_redirect(s, s->redirect); + end_header(s, this); return ret; } else { return RGWGetObj_ObjStore_S3::send_response_data(bl, bl_ofs, bl_len); From 7c8e0afa5c077ee0e6506a17c8d8b8c993acc6af Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Wed, 22 Jun 2016 02:49:06 +0000 Subject: [PATCH 27/60] Error handling and messages --- src/rgw/rgw_common.h | 4 ++ src/rgw/rgw_http_errors.h | 5 +++ src/rgw/rgw_main.cc | 5 +-- src/rgw/rgw_op.cc | 91 +++++++++++++++++++++++++-------------- src/rgw/rgw_op.h | 5 ++- src/rgw/rgw_rest_s3.cc | 9 ++-- 6 files changed, 80 insertions(+), 39 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index e8a5893e248d7..59964af154a1b 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -147,6 +147,10 @@ using ceph::crypto::MD5; #define ERR_INVALID_ACCESS_KEY 2028 #define ERR_BUCKET_ALREADY_OWNED 2029 #define ERR_BAD_RENAME_REQ 2030 +#define ERR_RENAME_COPY_FAILED 2031 +#define ERR_RENAME_PARSE_FAILED 2032 +#define ERR_RENAME_DEL_FAILED 2033 +#define ERR_RENAME_NOT_ENABLED 2034 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index f80ef13815539..5b4710699315f 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -46,6 +46,8 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_REQUEST_TIME_SKEWED, 403, "RequestTimeTooSkewed" ,"The difference between the request time and the server's time is too large."}, { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" }, { ERR_BAD_RENAME_REQ, 403, "Rename request must have object name, new object name and the HTTP method should be PUT." }, + // Why should we show the web based user how to enable the option? + { ERR_RENAME_NOT_ENABLED, 403, "Rename operation is not enabled. Enable the operation from the config file ceph.conf by setting \\\"rgw enable rename op\\\" to true."}, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, @@ -59,6 +61,9 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_UNPROCESSABLE_ENTITY, 422, "UnprocessableEntity" }, { ERR_LOCKED, 423, "Locked" }, { ERR_INTERNAL_ERROR, 500, "InternalError", "We encountered an internal error. Please try again." }, + { ERR_RENAME_COPY_FAILED, 500, "RenameFailed", "Object copy failed during rename" }, + { ERR_RENAME_PARSE_FAILED, 500, "RenameFailed", "Failed to parse copy source location" }, + { ERR_RENAME_DEL_FAILED, 500, "RenameDeleteFailed", "Failed to delete the source object. Please delete the resultant object(s) manually." }, }; const static struct rgw_http_errors RGW_HTTP_SWIFT_ERRORS[] = { diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 3d095ede6ad48..78c11e2f7d138 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -582,9 +582,9 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC if( ( - (amz_metadata_directive != NULL && !strcmp(amz_metadata_directive, "COPY")) + (amz_metadata_directive != NULL && !strcmp(amz_metadata_directive, "COPY")) || (jcs_metadata_directive != NULL && !strcmp(jcs_metadata_directive, "COPY")) - ) + ) && (amz_copy_source != NULL || jcs_copy_source != NULL) ) { @@ -597,7 +597,6 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC } } - RGWOp *op = NULL; int init_error = 0; bool should_log = false; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index c407ebed34d26..ec959891588a9 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3533,12 +3533,13 @@ void RGWRenameObj::pre_exec() void RGWRenameObj::execute() { ret = 0; - rgw_obj_key orig_object; - orig_object.dss_duplicate(&(s->object)); + s->err.ret = 0; + rgw_obj_key buffer_object; + buffer_object.dss_duplicate(&(s->object)); (s->object).name = s->info.args.get("newname"); string copysource = s->bucket_name_str; copysource.append("/"); - copysource.append(orig_object.name); + copysource.append(buffer_object.name); ldout(s->cct, 0) << "DSS INFO: Converting to copy request. s->object: " << (s->object).name << ". Copy source: " << copysource << dendl; s->info.env->set("HTTP_X_JCS_COPY_SOURCE", copysource.c_str()); @@ -3549,44 +3550,70 @@ void RGWRenameObj::execute() ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object); if (!ret) { ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; + s->err.ret = -ERR_RENAME_PARSE_FAILED; return; } - /*ret = validate_bucket_name(s->src_bucket_name, bucket_name_strictness_value); - if (ret) { - ldout(s->cct, 0) << "DSS INFO: Rename op bucket name not valid" << dendl; - return; - } - ret = validate_object_name(s->src_object.name); - if (ret) { - ldout(s->cct, 0) << "DSS INFO: Rename op object name not valid" << dendl; - return; - }*/ } - copy_op->init(store, s, dialect_handler); - copy_op->verify_op_mask(); - copy_op->verify_permission(); - copy_op->verify_params(); - copy_op->init_processing(); - s->system_request = true; - copy_op->pre_exec(); - copy_op->execute(); + perform_external_op(copy_op); + if ((s->err.http_ret != 200) || (s->err.ret != 0)) { + ldout(s->cct, 0) << "DSS ERROR: Copy object failed during rename op." + << " . Return status: " << s->err.ret + << " . Return HTTP code: " << s->err.http_ret + << " . Return Message: " << copy_op->get_request_state()->err.message + << dendl; + s->err.ret = -ERR_RENAME_COPY_FAILED; + return; + } ldout(s->cct, 0) << "DSS INFO: Rename op copy done" << dendl; - s->system_request = false; - (s->object).dss_duplicate(&orig_object); + (s->object).dss_duplicate(&buffer_object); ldout(s->cct, 0) << "DSS INFO: Rearraging things to continue with delete. s->object: " << (s->object).name << dendl; RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; - del_op->init(store, s, dialect_handler); - del_op->verify_permission(); - del_op->verify_params(); - del_op->verify_op_mask(); - del_op->init_processing(); - s->system_request = true; - del_op->pre_exec(); - del_op->execute(); + perform_external_op(del_op); + if ((s->err.http_ret != 200) || (s->err.ret != 0)) { + ldout(s->cct, 0) << "DSS ERROR: Delete object failed during rename op. Please" + << "manually delete the original object specified in the request." + << " . Return status: " << s->err.ret + << " . Return HTTP code: " << s->err.http_ret + << " . Return Message: " << del_op->get_request_state()->err.message + << dendl; + s->err.ret = -ERR_RENAME_DEL_FAILED; + return; + } ldout(s->cct, 0) << "DSS INFO: Rename op delete performed" << dendl; - s->system_request = false; return; } + +void RGWRenameObj::perform_external_op(RGWOp* bp) +{ + bool failure = false; + bp->init(store, s, dialect_handler); + ret = bp->init_processing(); + if ((ret < 0) || failure) { + failure = true; + } + ret = bp->verify_op_mask(); + if ((ret < 0) || failure) { + failure = true; + } + ret = bp->verify_permission(); + if ((ret < 0) || failure) { + failure = true; + } + ret = bp->verify_params(); + if ((ret < 0) || failure) { + failure = true; + } + if (!failure) { + s->system_request = true; + bp->pre_exec(); + bp->execute(); + s->system_request = false; + s->err.ret = bp->get_request_state()->err.ret; + } else { + s->err.ret = ret; + } + s->err.http_ret = bp->get_request_state()->err.http_ret; +} diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 54591a4971284..675fbbc2e5d22 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -87,7 +87,9 @@ class RGWOp { return 0; } - + req_state* get_request_state() { + return s; + } virtual void init(RGWRados *store, struct req_state *s, RGWHandler *dialect_handler) { this->store = store; this->s = s; @@ -573,6 +575,7 @@ class RGWRenameObj : public RGWOp { int verify_permission(); void pre_exec(); void execute(); + void perform_external_op(RGWOp*); virtual const string name() { return "Rename_obj"; } }; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 09e6adba7c4a3..14e9984a82a50 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1620,7 +1620,7 @@ int RGWPutACLs_ObjStore_S3::get_policy_from_state(RGWRados *store, struct req_st void RGWRenameObj_ObjStore_S3::send_response() { - ret = 0; + ret = s->err.ret; if (ret) set_req_state_err(s, ret); dump_errno(s); @@ -2129,7 +2129,7 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_put() if (is_acl_op()) { return new RGWPutACLs_ObjStore_S3; } - if (is_rename_op() && store->ctx()->_conf->rgw_enable_rename_op) { + if (store->ctx()->_conf->rgw_enable_rename_op && is_rename_op()) { return new RGWRenameObj_ObjStore_S3; } if (!s->copy_source) @@ -2741,7 +2741,10 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) } // Block rename op for illegal cases - if ((store->ctx()->_conf->rgw_enable_rename_op) && (s->info.args.exists("newname"))) { + if (s->info.args.exists("newname")) { + if (!(store->ctx()->_conf->rgw_enable_rename_op)) { + return -ERR_RENAME_NOT_ENABLED; + } if (s->op == OP_PUT) { if ((s->object).name.empty()) { return -ERR_BAD_RENAME_REQ; From 8b77bc078655fdaf45947dac727673af8e3eef7f Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Wed, 20 Apr 2016 15:52:51 -0700 Subject: [PATCH 28/60] rgw/s3website: Implement ErrorDoc & fix Double-Fault handler Fix more last minute breakage from merges, now has has a working ErrorDoc as well as working double-fault. Also moves some s3website-specific code out of the main S3 codepath. Fixes: #15532 Fixes: #15555 Signed-off-by: Robin H. Johnson Conflicts: src/rgw/rgw_rest.cc src/rgw/rgw_rest_s3.cc src/rgw/rgw_rest_s3.h src/rgw/rgw_rest_s3website.h --- src/rgw/rgw_rest.cc | 94 ++++++++++++------------- src/rgw/rgw_rest_s3.cc | 133 +++++++++++++++++++++++++---------- src/rgw/rgw_rest_s3.h | 5 ++ src/rgw/rgw_rest_s3website.h | 5 +- 4 files changed, 149 insertions(+), 88 deletions(-) diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 99b5fbf201e60..bafd3a9db5f49 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -330,17 +330,8 @@ void set_req_state_err(struct req_state *s, int err_no) r = search_err(err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS)); if (r) { - if (s->prot_flags & RGW_REST_WEBSITE && err_no == ERR_WEBSITE_REDIRECT && !s->err.is_clear()) { - // http_ret was custom set, so don't change it! - } else { - s->err.http_ret = r->http_ret; - } + s->err.http_ret = r->http_ret; s->err.s3_code = r->s3_code; - if (r->s3_err_message) { - if ((s->err.message).empty()) { - s->err.message = r->s3_err_message; - } - } return; } dout(0) << "WARNING: set_req_state_err err_no=" << err_no << " resorting to 500" << dendl; @@ -715,47 +706,56 @@ void abort_early(struct req_state *s, RGWOp *op, int err_no, RGWHandler* handler ldout(s->cct, 20) << "handler->ERRORHANDLER: err_no=" << err_no << " new_err_no=" << new_err_no << dendl; err_no = new_err_no; } - set_req_state_err(s, err_no); - dump_errno(s); - dump_bucket_from_state(s); - if (err_no == -ERR_PERMANENT_REDIRECT || err_no == -ERR_WEBSITE_REDIRECT) { - string dest_uri; - if (!s->redirect.empty()) { - dest_uri = s->redirect; - } else if (!s->region_endpoint.empty()) { - string dest_uri = s->region_endpoint; - /* - * reqest_uri is always start with slash, so we need to remove - * the unnecessary slash at the end of dest_uri. - */ - if (dest_uri[dest_uri.size() - 1] == '/') { - dest_uri = dest_uri.substr(0, dest_uri.size() - 1); + + // If the error handler(s) above dealt with it completely, they should have + // returned 0. If non-zero, we need to continue here. + if(err_no) { + // Watch out, we might have a custom error state already set! + if(s->err.http_ret && s->err.http_ret != 200) { + dump_errno(s); + } else { + set_req_state_err(s, err_no); + dump_errno(s); + } + dump_bucket_from_state(s); + if (err_no == -ERR_PERMANENT_REDIRECT || err_no == -ERR_WEBSITE_REDIRECT) { + string dest_uri; + if (!s->redirect.empty()) { + dest_uri = s->redirect; + } else if (!s->region_endpoint.empty()) { + string dest_uri = s->region_endpoint; + /* + * reqest_uri is always start with slash, so we need to remove + * the unnecessary slash at the end of dest_uri. + */ + if (dest_uri[dest_uri.size() - 1] == '/') { + dest_uri = dest_uri.substr(0, dest_uri.size() - 1); + } + dest_uri += s->info.request_uri; + dest_uri += "?"; + dest_uri += s->info.request_params; + } + + if (!dest_uri.empty()) { + dump_redirect(s, dest_uri); } - dest_uri += s->info.request_uri; - dest_uri += "?"; - dest_uri += s->info.request_params; } - if (!dest_uri.empty()) { - dump_redirect(s, dest_uri); + if (!error_content.empty()) { + /* + * TODO we must add all error entries as headers here: + * when having a working errordoc, then the s3 error fields are + * rendered as HTTP headers, e.g.: + * x-amz-error-code: NoSuchKey + * x-amz-error-message: The specified key does not exist. + * x-amz-error-detail-Key: foo + */ + end_header(s, op, NULL, error_content.size(), false, true); + s->cio->write(error_content.c_str(), error_content.size()); + } else { + end_header(s, op); } - } - if (!error_content.empty()) { - ldout(s->cct, 20) << "error_content is set, we need to serve it INSTEAD of firing the formatter" << dendl; - /* - * FIXME we must add all error entries as headers here: - * when having a working errordoc, then the s3 error fields are rendered as HTTP headers, e.g.: - * - * x-amz-error-code: NoSuchKey - * x-amz-error-message: The specified key does not exist. - * x-amz-error-detail-Key: foo - */ - end_header(s, op, NULL, NO_CONTENT_LENGTH, false, true); - s->cio->write(error_content.c_str(), error_content.size()); - s->formatter->reset(); - } else { - end_header(s, op); - rgw_flush_formatter_and_reset(s, s->formatter); + rgw_flush_formatter(s, s->formatter); } perfcounter->inc(l_rgw_failed_req); } diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 181fa982774b5..cc1f9ef7d857f 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -193,9 +193,14 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ } done: - set_req_state_err(s, (partial_content && !ret) ? STATUS_PARTIAL_CONTENT : ret); - - dump_errno(s); + if(custom_http_ret) { + set_req_state_err(s, 0); + dump_errno(s, custom_http_ret); + } else { + set_req_state_err(s, (partial_content && !ret) ? STATUS_PARTIAL_CONTENT + : ret); + dump_errno(s); + } for (riter = response_attrs.begin(); riter != response_attrs.end(); ++riter) { s->cio->print("%s: %s\r\n", riter->first.c_str(), riter->second.c_str()); @@ -3221,47 +3226,87 @@ RGWOp *RGWHandler_ObjStore_S3Website::op_head() return get_obj_op(false); } -int RGWHandler_ObjStore_S3Website::get_errordoc(const string errordoc_key, string *error_content) { - ldout(s->cct, 20) << "TODO Serve Custom error page here if bucket has " << dendl; - *error_content = errordoc_key; - // 1. Check if errordoc exists - // 2. Check if errordoc is public - // 3. Fetch errordoc content - /* - * FIXME maybe: need to make sure all of the fields for conditional requests are cleared - */ - RGWGetObj_ObjStore_S3Website *getop = new RGWGetObj_ObjStore_S3Website(true); - getop->set_get_data(true); - getop->init(store, s, this); - - RGWGetObj_CB cb(getop); - rgw_obj obj(s->bucket, errordoc_key); - RGWObjectCtx rctx(store); - //RGWRados::Object op_target(store, s->bucket_info, *static_cast(s->obj_ctx), obj); - RGWRados::Object op_target(store, s->bucket_info, rctx, obj); - RGWRados::Object::Read read_op(&op_target); - - int ret; - int64_t ofs = 0; - int64_t end = -1; - ret = read_op.prepare(&ofs, &end); - if (ret < 0) { - goto done; - } +int RGWHandler_ObjStore_S3Website::serve_errordoc(int http_ret, const string& errordoc_key) { + int ret = 0; + s->formatter->reset(); /* Try to throw it all away */ + + RGWGetObj_ObjStore_S3Website* getop = (RGWGetObj_ObjStore_S3Website*) op_get(); + if(!getop) { + return -1; // Trigger double error handler + } + getop->init(store, s, this); + /*getop->range_str = NULL; + getop->if_mod = NULL; + getop->if_unmod = NULL; + getop->if_match = NULL; + getop->if_nomatch = NULL;*/ + s->object = errordoc_key; + + ret = init_permissions(getop); + if (ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, init_permissions ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + ret = read_permissions(getop); + if (ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, read_permissions ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + if(http_ret) { + getop->set_custom_http_response(http_ret); + } + + ret = getop->init_processing(); + if(ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, init_processing ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + ret = getop->verify_op_mask(); + if(ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, verify_op_mask ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + ret = getop->verify_permission(); + if(ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, verify_permission ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + ret = getop->verify_params(); + if(ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, verify_params ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + // No going back now + getop->pre_exec(); + /* + * FIXME Missing headers: + * With a working errordoc, the s3 error fields are rendered as HTTP headers, + * x-amz-error-code: NoSuchKey + * x-amz-error-message: The specified key does not exist. + * x-amz-error-detail-Key: foo + */ + getop->execute(); + getop->complete(); + return 0; - ret = read_op.iterate(ofs, end, &cb); // FIXME: need to know the final size? -done: - delete getop; - return ret; } - -int RGWHandler_ObjStore_S3Website::error_handler(int err_no, string *error_content) { - const struct rgw_http_errors *r; + +int RGWHandler_ObjStore_S3Website::error_handler(int err_no, + string* error_content) { + int new_err_no = -1; + const struct rgw_http_errors* r; int http_error_code = -1; - r = search_err(err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS)); + r = search_err(err_no > 0 ? err_no : -err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS)); if (r) { http_error_code = r->http_ret; } + ldout(s->cct, 10) << "RGWHandler_ObjStore_S3Website::error_handler err_no=" << err_no << " http_ret=" << http_error_code << dendl; RGWBWRoutingRule rrule; bool should_redirect = s->bucket_info.website_conf.should_redirect(s->object.name, http_error_code, &rrule); @@ -3276,8 +3321,18 @@ int RGWHandler_ObjStore_S3Website::error_handler(int err_no, string *error_conte s->err.http_ret = redirect_code; // Apply a custom HTTP response code ldout(s->cct, 10) << "error handler redirect code=" << redirect_code << " proto+host:" << protocol << "://" << hostname << " -> " << s->redirect << dendl; return -ERR_WEBSITE_REDIRECT; + } else if (err_no == -ERR_WEBSITE_REDIRECT) { + // Do nothing here, this redirect will be handled in abort_early's ERR_WEBSITE_REDIRECT block + // Do NOT fire the ErrorDoc handler } else if (!s->bucket_info.website_conf.error_doc.empty()) { - RGWHandler_ObjStore_S3Website::get_errordoc(s->bucket_info.website_conf.error_doc, error_content); + /* This serves an entire page! + On success, it will return zero, and no further content should be sent to the socket + On failure, we need the double-error handler + */ + new_err_no = RGWHandler_ObjStore_S3Website::serve_errordoc(http_error_code, s->bucket_info.website_conf.error_doc); + if(new_err_no && new_err_no != -1) { + err_no = new_err_no; + } } else { ldout(s->cct, 20) << "No special error handling today!" << dendl; } diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index d0ab281c42a73..3efad1079722e 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -20,12 +20,17 @@ void rgw_get_errno_s3(struct rgw_http_errors *e, int err_no); class RGWGetObj_ObjStore_S3 : public RGWGetObj_ObjStore { +protected: + // Serving a custom error page from an object is really a 200 response with + // just the status line altered. + int custom_http_ret = 0; public: RGWGetObj_ObjStore_S3() {} ~RGWGetObj_ObjStore_S3() {} int send_response_data_error(); int send_response_data(bufferlist& bl, off_t ofs, off_t len); + void set_custom_http_response(int http_ret) { custom_http_ret = http_ret; } }; class RGWListBuckets_ObjStore_S3 : public RGWListBuckets_ObjStore { diff --git a/src/rgw/rgw_rest_s3website.h b/src/rgw/rgw_rest_s3website.h index 06a22ccda22d7..f8ead0ba7c61b 100644 --- a/src/rgw/rgw_rest_s3website.h +++ b/src/rgw/rgw_rest_s3website.h @@ -32,7 +32,7 @@ class RGWHandler_ObjStore_S3Website : public RGWHandler_ObjStore_S3 { RGWOp *op_copy() { return NULL; } RGWOp *op_options() { return NULL; } - int get_errordoc(const string errordoc_key, string *error_content); + int serve_errordoc(int http_ret, const string &errordoc_key); public: RGWHandler_ObjStore_S3Website() : RGWHandler_ObjStore_S3() {} virtual ~RGWHandler_ObjStore_S3Website() {} @@ -69,6 +69,7 @@ class RGWHandler_ObjStore_Bucket_S3Website : public RGWHandler_ObjStore_S3Websit // TODO: do we actually need this? class RGWGetObj_ObjStore_S3Website : public RGWGetObj_ObjStore_S3 { + friend class RGWHandler_REST_S3Website; private: bool is_errordoc_request; public: @@ -86,7 +87,7 @@ class RGWGetObj_ObjStore_S3Website : public RGWGetObj_ObjStore_S3 if_unmod = NULL; if_match = NULL; if_nomatch = NULL; - return 0; + return 0; } else { return RGWGetObj_ObjStore_S3::get_params(); } From 490a4f5cdec3e7a3609761ac6a5ecd6ba49f3e40 Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Tue, 17 May 2016 17:38:38 -0700 Subject: [PATCH 29/60] rgw/s3website: whitespace style fixes Signed-off-by: Robin H. Johnson Conflicts: src/rgw/rgw_rest_s3.cc --- src/rgw/rgw_rest.cc | 4 ++-- src/rgw/rgw_rest_s3.cc | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index bafd3a9db5f49..2f0f28b304a42 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -709,9 +709,9 @@ void abort_early(struct req_state *s, RGWOp *op, int err_no, RGWHandler* handler // If the error handler(s) above dealt with it completely, they should have // returned 0. If non-zero, we need to continue here. - if(err_no) { + if (err_no) { // Watch out, we might have a custom error state already set! - if(s->err.http_ret && s->err.http_ret != 200) { + if (s->err.http_ret && s->err.http_ret != 200) { dump_errno(s); } else { set_req_state_err(s, err_no); diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index cc1f9ef7d857f..19da7bf7acee6 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -193,7 +193,7 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ } done: - if(custom_http_ret) { + if (custom_http_ret) { set_req_state_err(s, 0); dump_errno(s, custom_http_ret); } else { @@ -3254,30 +3254,30 @@ int RGWHandler_ObjStore_S3Website::serve_errordoc(int http_ret, const string& er return -1; // Trigger double error handler } - if(http_ret) { + if (http_ret) { getop->set_custom_http_response(http_ret); } ret = getop->init_processing(); - if(ret < 0) { + if (ret < 0) { ldout(s->cct, 20) << "serve_errordoc failed, init_processing ret=" << ret << dendl; return -1; // Trigger double error handler } ret = getop->verify_op_mask(); - if(ret < 0) { + if (ret < 0) { ldout(s->cct, 20) << "serve_errordoc failed, verify_op_mask ret=" << ret << dendl; return -1; // Trigger double error handler } ret = getop->verify_permission(); - if(ret < 0) { + if (ret < 0) { ldout(s->cct, 20) << "serve_errordoc failed, verify_permission ret=" << ret << dendl; return -1; // Trigger double error handler } ret = getop->verify_params(); - if(ret < 0) { + if (ret < 0) { ldout(s->cct, 20) << "serve_errordoc failed, verify_params ret=" << ret << dendl; return -1; // Trigger double error handler } From e4aa51ef03f88b3db4a024b71ffd39f6637e6a8d Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Fri, 20 May 2016 16:00:33 -0700 Subject: [PATCH 30/60] rgw: fix manager selection when APIs customized When modifying rgw_enable_apis per RGW instance, such as for staticsites, you can end up with RESTManager instance being null in some cases, which returns a HTTP 405 MethodNotAllowed to all requests. Example configuration to trigger the bug: rgw_enable_apis = s3website Backport: jewel X-Note: Patch from Yehuda in private IRC discussion, 2016/05/20. Fixes: http://tracker.ceph.com/issues/15973 Fixes: http://tracker.ceph.com/issues/15974 Signed-off-by: Robin H. Johnson (cherry picked from commit 7c7a465b55f7100eab0f140bf54f9420abd1c776) --- src/rgw/rgw_rest.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 2f0f28b304a42..172a6691fbf9d 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -1474,9 +1474,6 @@ RGWRESTMgr *RGWRESTMgr::get_resource_mgr(struct req_state *s, const string& uri, { *out_uri = uri; - if (resources_by_size.empty()) - return this; - multimap::reverse_iterator iter; for (iter = resources_by_size.rbegin(); iter != resources_by_size.rend(); ++iter) { From 0f7248a932923d1365ebf75da97dd0ec2b9f3dc9 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Thu, 23 Jun 2016 11:20:49 +0000 Subject: [PATCH 31/60] Adding proper fault injection --- src/common/config_opts.h | 6 +- src/rgw/rgw_common.h | 9 +-- src/rgw/rgw_http_errors.h | 10 +-- src/rgw/rgw_op.cc | 146 +++++++++++++++++++++++++++++++------- src/rgw/rgw_op.h | 2 + 5 files changed, 137 insertions(+), 36 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 2cf919b0ddbbe..b1021df13b19c 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1077,7 +1077,11 @@ OPTION(rgw_keystone_token_api, OPT_STR, "v3/token-auth") // api to validate tok OPTION(rgw_keystone_url_token_api, OPT_STR, "url-auth") // api to validate presigned token URL based authentication requests OPTION(rgw_keystone_infinite_url_token_api, OPT_STR, "preauth-token-auth") // api to validate infinite time presigned token URL OPTION(dss_regional_url, OPT_STR, "https://dss.ind-west-1.staging.jiocloudservices.com") // URL to be returned in XMLNS during anonymous list all buckets calls -OPTION(rgw_enable_rename_op, OPT_BOOL, true) // enable the atomic rename op +OPTION(rgw_enable_rename_op, OPT_BOOL, true) // Enable the atomic rename op +OPTION(fault_inj_rename_op_copy_fail, OPT_BOOL, false) // Injects fault in rename op during copy operation +OPTION(fault_inj_rename_op_delete_fail, OPT_BOOL, false) // Injects fault in rename op during delete operation +OPTION(fault_inj_rename_op_parse_fail, OPT_BOOL, false) // Injects fault in rename op during parse operation +OPTION(fault_inj_rename_op_sleep_after_copy, OPT_BOOL, false) // Make rename op sleep after copy OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf counter diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 59964af154a1b..fbd54482d131e 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -147,10 +147,11 @@ using ceph::crypto::MD5; #define ERR_INVALID_ACCESS_KEY 2028 #define ERR_BUCKET_ALREADY_OWNED 2029 #define ERR_BAD_RENAME_REQ 2030 -#define ERR_RENAME_COPY_FAILED 2031 -#define ERR_RENAME_PARSE_FAILED 2032 -#define ERR_RENAME_DEL_FAILED 2033 -#define ERR_RENAME_NOT_ENABLED 2034 +#define ERR_RENAME_NOT_ENABLED 2031 +#define ERR_RENAME_FAILED 2022 +#define ERR_RENAME_DATA_LOST 2033 +#define ERR_RENAME_COPY_FAILED 2034 +#define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index 5b4710699315f..b82d70a94d488 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -46,8 +46,7 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_REQUEST_TIME_SKEWED, 403, "RequestTimeTooSkewed" ,"The difference between the request time and the server's time is too large."}, { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" }, { ERR_BAD_RENAME_REQ, 403, "Rename request must have object name, new object name and the HTTP method should be PUT." }, - // Why should we show the web based user how to enable the option? - { ERR_RENAME_NOT_ENABLED, 403, "Rename operation is not enabled. Enable the operation from the config file ceph.conf by setting \\\"rgw enable rename op\\\" to true."}, + { ERR_RENAME_NOT_ENABLED, 403, "Rename operation is not enabled"}, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, @@ -61,9 +60,10 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_UNPROCESSABLE_ENTITY, 422, "UnprocessableEntity" }, { ERR_LOCKED, 423, "Locked" }, { ERR_INTERNAL_ERROR, 500, "InternalError", "We encountered an internal error. Please try again." }, - { ERR_RENAME_COPY_FAILED, 500, "RenameFailed", "Object copy failed during rename" }, - { ERR_RENAME_PARSE_FAILED, 500, "RenameFailed", "Failed to parse copy source location" }, - { ERR_RENAME_DEL_FAILED, 500, "RenameDeleteFailed", "Failed to delete the source object. Please delete the resultant object(s) manually." }, + { ERR_RENAME_COPY_FAILED, 500, "RenameFailed", "Object copy failed during rename. Please file a bug." }, + { ERR_RENAME_FAILED, 500, "RenameFailed", "Rename operation has failed" }, + { ERR_RENAME_DATA_LOST, 500, "Data lost", "Rename operation lost the original data. Please file a bug." }, + { ERR_RENAME_NEW_OBJ_DEL_FAILED, 500, "RenameFailed", "Rename operation failed. Please delete the duplicated object with name same as new name for the object, manually. Please file a bug" }, }; const static struct rgw_http_errors RGW_HTTP_SWIFT_ERRORS[] = { diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index ec959891588a9..fd6e03a802ae9 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3534,78 +3534,139 @@ void RGWRenameObj::execute() { ret = 0; s->err.ret = 0; - rgw_obj_key buffer_object; - buffer_object.dss_duplicate(&(s->object)); + rgw_obj_key orig_object, new_obj; + orig_object.dss_duplicate(&(s->object)); + + /* Check if the original object exists */ + ret = check_obj(orig_object); + if (ret < 0) { + // The passed original object does not exist + s->err.ret = ret; + return; + } + + /* Tweek request params to make this a copy request */ (s->object).name = s->info.args.get("newname"); string copysource = s->bucket_name_str; copysource.append("/"); - copysource.append(buffer_object.name); + copysource.append(orig_object.name); ldout(s->cct, 0) << "DSS INFO: Converting to copy request. s->object: " << (s->object).name << ". Copy source: " << copysource << dendl; s->info.env->set("HTTP_X_JCS_COPY_SOURCE", copysource.c_str()); s->info.env->set("HTTP_X_JCS_METADATA_DIRECTIVE", "COPY"); s->copy_source = s->info.env->get("HTTP_X_JCS_COPY_SOURCE"); - RGWCopyObj_ObjStore_S3* copy_op = new RGWCopyObj_ObjStore_S3; if (s->copy_source) { ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object); - if (!ret) { + if (!ret || (store->ctx()->_conf->fault_inj_rename_op_parse_fail)) { ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; - s->err.ret = -ERR_RENAME_PARSE_FAILED; + s->err.ret = -ERR_RENAME_FAILED; return; } } + /* Perform copy operation */ + RGWCopyObj_ObjStore_S3* copy_op = new RGWCopyObj_ObjStore_S3; perform_external_op(copy_op); - if ((s->err.http_ret != 200) || (s->err.ret != 0)) { + + if ((store->ctx()->_conf->fault_inj_rename_op_sleep_after_copy)) { + // Fault injection to test atomicity + std::this_thread::sleep_for(60s); + } + + if ((s->err.http_ret != 200) || + (s->err.ret != 0)) { ldout(s->cct, 0) << "DSS ERROR: Copy object failed during rename op." << " . Return status: " << s->err.ret << " . Return HTTP code: " << s->err.http_ret << " . Return Message: " << copy_op->get_request_state()->err.message << dendl; - s->err.ret = -ERR_RENAME_COPY_FAILED; + s->err.ret = -ERR_RENAME_FAILED; return; } - ldout(s->cct, 0) << "DSS INFO: Rename op copy done" << dendl; + ldout(s->cct, 0) << "DSS INFO: Rename op: copy done" << dendl; - (s->object).dss_duplicate(&buffer_object); - ldout(s->cct, 0) << "DSS INFO: Rearraging things to continue with delete. s->object: " - << (s->object).name << dendl; - RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; - perform_external_op(del_op); - if ((s->err.http_ret != 200) || (s->err.ret != 0)) { - ldout(s->cct, 0) << "DSS ERROR: Delete object failed during rename op. Please" - << "manually delete the original object specified in the request." + /* Tweek the request for a delete obj operation and perform delete op */ + new_obj.dss_duplicate(&(s->object)); + (s->object).dss_duplicate(&orig_object); + delete_rgw_object(); + if ((s->err.http_ret != 200) || + (s->err.ret != 0)) { + ldout(s->cct, 0) << "DSS ERROR: Delete object failed during rename op." << " . Return status: " << s->err.ret << " . Return HTTP code: " << s->err.http_ret << " . Return Message: " << del_op->get_request_state()->err.message << dendl; - s->err.ret = -ERR_RENAME_DEL_FAILED; + + /* Revert the copy op */ + int ret_orig, ret_newobj; + ret_orig = check_obj(s->object); + ret_newobj = check_obj(new_obj); + if (ret_orig < 0) { + // Delete failed but we don't have original object!! + if (ret_newobj < 0) { + // We are in a soup. Data lost. This is not the case we will ever end up in. + s->err.ret = -ERR_RENAME_DATA_LOST; + } else { + // Everything normal + if (!store->ctx()->_conf->fault_inj_rename_op_delete_fail) { + s->err.http_ret = 200; + s->err.ret = 0; + } + } + } else { + if (ret_newobj >= 0) { + // Delete the new object + (s->object).dss_duplicate(&new_obj); + delete_rgw_object(); + if ((s->err.http_ret != 200) || + (s->err.ret != 0)) { + s->err.ret = -ERR_RENAME_NEW_OBJ_DEL_FAILED; + } else { + // Indicate that there has been a failure + s->err.ret = -ERR_RENAME_FAILED; + } + } else { + // Log major error. Ask user to file a bug. Why didn't copy fail? + s->err.ret = -ERR_RENAME_COPY_FAILED; + } + } return; } - ldout(s->cct, 0) << "DSS INFO: Rename op delete performed" << dendl; + ldout(s->cct, 0) << "DSS INFO: Rename op complete" << dendl; return; } void RGWRenameObj::perform_external_op(RGWOp* bp) { + // Fault injection if tester wants so + if ((store->ctx()->_conf->fault_inj_rename_op_copy_fail) || + (store->ctx()->_conf->fault_inj_rename_op_delete_fail)) { + s->err.http_ret = 403; + s->err.ret = -ERR_RENAME_FAULT_INJ; + } + bool failure = false; bp->init(store, s, dialect_handler); ret = bp->init_processing(); - if ((ret < 0) || failure) { + if (ret < 0) { failure = true; + } else { + ret = bp->verify_op_mask(); } - ret = bp->verify_op_mask(); - if ((ret < 0) || failure) { + if (ret < 0) { failure = true; + } else { + ret = bp->verify_permission(); } - ret = bp->verify_permission(); - if ((ret < 0) || failure) { + if (ret < 0) { failure = true; + } else { + ret = bp->verify_params(); } - ret = bp->verify_params(); - if ((ret < 0) || failure) { + if (ret < 0) { failure = true; } + if (!failure) { s->system_request = true; bp->pre_exec(); @@ -3616,4 +3677,37 @@ void RGWRenameObj::perform_external_op(RGWOp* bp) s->err.ret = ret; } s->err.http_ret = bp->get_request_state()->err.http_ret; + s->err.ret = bp->get_request_state()->err.ret; +} + +int RGWRenameObj::check_obj(rgw_obj_key& object) +{ + rgw_obj lobj(s->bucket, object); + RGWObjectCtx obj_ctx(store); + RGWObjState *ros = obj_ctx.get_state(lobj); + + /*if (ros->exists) { + ldout(s->cct, 0) << "DSS INFO: =============== exists " << dendl; + } else { + ldout(s->cct, 0) << "DSS INFO: =============== DOES NOT exist " << dendl; + }*/ + + ret = store->raw_obj_stat(lobj, NULL, NULL, NULL, NULL, NULL, NULL); + /////ldout(s->cct, 0) << "DSS INFO: ----------------- check obj ret is " << ret << dendl; + if (ret < 0) { + ldout(s->cct, 0) << "DSS ERROR: The object " << object.name + << " does not exist. Error code: " << ret << dendl; + return ret; + } + + return 0; +} + +void RGWRenameObj::delete_rgw_object() +{ + ldout(s->cct, 0) << "DSS INFO: Deleting object. s->object.name: " + << (s->object).name << dendl; + RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; + perform_external_op(del_op); + return; } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 675fbbc2e5d22..949069063960c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -576,6 +576,8 @@ class RGWRenameObj : public RGWOp { void pre_exec(); void execute(); void perform_external_op(RGWOp*); + void delete_rgw_object(); + int check_obj(rgw_obj_key&); virtual const string name() { return "Rename_obj"; } }; From 72853a99a80cbfc02664d222823e1a63dfbff1c5 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Thu, 23 Jun 2016 12:35:23 +0000 Subject: [PATCH 32/60] Getting object state --- src/rgw/rgw_common.h | 1 + src/rgw/rgw_http_errors.h | 1 + src/rgw/rgw_op.cc | 31 ++++++++++++++----------------- src/rgw/rgw_op.h | 2 +- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index fbd54482d131e..94d9e500afeb0 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -152,6 +152,7 @@ using ceph::crypto::MD5; #define ERR_RENAME_DATA_LOST 2033 #define ERR_RENAME_COPY_FAILED 2034 #define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 +#define ERR_RENAME_FAULT_INJ 2036 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index b82d70a94d488..7b0aca926285b 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -47,6 +47,7 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" }, { ERR_BAD_RENAME_REQ, 403, "Rename request must have object name, new object name and the HTTP method should be PUT." }, { ERR_RENAME_NOT_ENABLED, 403, "Rename operation is not enabled"}, + { ERR_RENAME_FAULT_INJ, 403, "Rename operation fault has been activated"}, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index fd6e03a802ae9..b3164f117fcf1 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3,9 +3,9 @@ #include #include +#include #include - #include "common/Clock.h" #include "common/armor.h" #include "common/mime.h" @@ -3570,7 +3570,7 @@ void RGWRenameObj::execute() if ((store->ctx()->_conf->fault_inj_rename_op_sleep_after_copy)) { // Fault injection to test atomicity - std::this_thread::sleep_for(60s); + sleep(60); } if ((s->err.http_ret != 200) || @@ -3588,7 +3588,8 @@ void RGWRenameObj::execute() /* Tweek the request for a delete obj operation and perform delete op */ new_obj.dss_duplicate(&(s->object)); (s->object).dss_duplicate(&orig_object); - delete_rgw_object(); + RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; + delete_rgw_object(del_op); if ((s->err.http_ret != 200) || (s->err.ret != 0)) { ldout(s->cct, 0) << "DSS ERROR: Delete object failed during rename op." @@ -3617,7 +3618,8 @@ void RGWRenameObj::execute() if (ret_newobj >= 0) { // Delete the new object (s->object).dss_duplicate(&new_obj); - delete_rgw_object(); + RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; + delete_rgw_object(del_op); if ((s->err.http_ret != 200) || (s->err.ret != 0)) { s->err.ret = -ERR_RENAME_NEW_OBJ_DEL_FAILED; @@ -3684,30 +3686,25 @@ int RGWRenameObj::check_obj(rgw_obj_key& object) { rgw_obj lobj(s->bucket, object); RGWObjectCtx obj_ctx(store); - RGWObjState *ros = obj_ctx.get_state(lobj); - - /*if (ros->exists) { - ldout(s->cct, 0) << "DSS INFO: =============== exists " << dendl; - } else { - ldout(s->cct, 0) << "DSS INFO: =============== DOES NOT exist " << dendl; - }*/ + RGWObjState *ros = NULL; - ret = store->raw_obj_stat(lobj, NULL, NULL, NULL, NULL, NULL, NULL); - /////ldout(s->cct, 0) << "DSS INFO: ----------------- check obj ret is " << ret << dendl; + ret = store->get_obj_state(&obj_ctx, lobj, &ros, NULL); if (ret < 0) { - ldout(s->cct, 0) << "DSS ERROR: The object " << object.name - << " does not exist. Error code: " << ret << dendl; + ldout(s->cct, 0) << "DSS ERROR: Failed to fetch obj state" << dendl; return ret; } + if (!ros->exists) { + ldout(s->cct, 0) << "DSS ERROR: The object " << object.name << " does not exist." << dendl; + return -ENOENT; + } return 0; } -void RGWRenameObj::delete_rgw_object() +void RGWRenameObj::delete_rgw_object(RGWOp* del_op) { ldout(s->cct, 0) << "DSS INFO: Deleting object. s->object.name: " << (s->object).name << dendl; - RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; perform_external_op(del_op); return; } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 949069063960c..40e629d3af7c6 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -576,7 +576,7 @@ class RGWRenameObj : public RGWOp { void pre_exec(); void execute(); void perform_external_op(RGWOp*); - void delete_rgw_object(); + void delete_rgw_object(RGWOp*); int check_obj(rgw_obj_key&); virtual const string name() { return "Rename_obj"; } }; From a74f35167f9b47e817c9211f3032f837009d10dd Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Fri, 24 Jun 2016 10:39:06 +0000 Subject: [PATCH 33/60] Rajat bugs --- src/rgw/rgw_http_errors.h | 12 ++++++------ src/rgw/rgw_op.cc | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index 7b0aca926285b..331243873045a 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -35,6 +35,7 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_TOO_LARGE, 400, "EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size (5GB in our case)."}, { ERR_TOO_SMALL, 400, "EntityTooSmall", "Your proposed upload is smaller than the minimum allowed size (1 MB in our case for multipart upload)"}, { ERR_TOO_MANY_BUCKETS, 400, "TooManyBuckets" , "You have attempted to create more buckets than allowed"}, + { ERR_RENAME_FAILED, 400, "RenameFailed", "Rename operation has failed." }, {ERR_BUCKET_ALREADY_OWNED, 409, "BucketAlreadyOwnedByYou" , "Your previous request to create the named bucket succeeded and you already own it."}, //{ ERR_MALFORMED_XML, 400, "MalformedXML" }, { ERR_LENGTH_REQUIRED, 411, "MissingContentLength", "You must provide the Content-Length HTTP header."}, @@ -45,9 +46,9 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_USER_SUSPENDED, 403, "UserSuspended" }, { ERR_REQUEST_TIME_SKEWED, 403, "RequestTimeTooSkewed" ,"The difference between the request time and the server's time is too large."}, { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" }, - { ERR_BAD_RENAME_REQ, 403, "Rename request must have object name, new object name and the HTTP method should be PUT." }, - { ERR_RENAME_NOT_ENABLED, 403, "Rename operation is not enabled"}, - { ERR_RENAME_FAULT_INJ, 403, "Rename operation fault has been activated"}, + { ERR_BAD_RENAME_REQ, 403, "BadRenameRequest", "Rename request must have object name, new object name and the HTTP method should be PUT." }, + { ERR_RENAME_NOT_ENABLED, 403, "RenameDisabled", "Rename operation is not enabled"}, + { ERR_RENAME_FAULT_INJ, 403, "FaultInjection", "A rename operation tester fault has been activated"}, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, @@ -62,9 +63,8 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_LOCKED, 423, "Locked" }, { ERR_INTERNAL_ERROR, 500, "InternalError", "We encountered an internal error. Please try again." }, { ERR_RENAME_COPY_FAILED, 500, "RenameFailed", "Object copy failed during rename. Please file a bug." }, - { ERR_RENAME_FAILED, 500, "RenameFailed", "Rename operation has failed" }, - { ERR_RENAME_DATA_LOST, 500, "Data lost", "Rename operation lost the original data. Please file a bug." }, - { ERR_RENAME_NEW_OBJ_DEL_FAILED, 500, "RenameFailed", "Rename operation failed. Please delete the duplicated object with name same as new name for the object, manually. Please file a bug" }, + { ERR_RENAME_DATA_LOST, 500, "DataLost", "Rename operation lost the original data. Please file a bug." }, + { ERR_RENAME_NEW_OBJ_DEL_FAILED, 500, "RenameFailed", "Rename operation failed. Please delete the duplicated object with name same as new name for the object, manually. Please file a bug." }, }; const static struct rgw_http_errors RGW_HTTP_SWIFT_ERRORS[] = { diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index b3164f117fcf1..bb8219c5bf85e 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3645,6 +3645,7 @@ void RGWRenameObj::perform_external_op(RGWOp* bp) (store->ctx()->_conf->fault_inj_rename_op_delete_fail)) { s->err.http_ret = 403; s->err.ret = -ERR_RENAME_FAULT_INJ; + return; } bool failure = false; From 435d8efff61bf55de3883cc79766f7bc6d138777 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Fri, 24 Jun 2016 11:01:12 +0000 Subject: [PATCH 34/60] Rajat bug 2 --- src/rgw/rgw_op.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index bb8219c5bf85e..083cce889aa79 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3640,6 +3640,9 @@ void RGWRenameObj::execute() void RGWRenameObj::perform_external_op(RGWOp* bp) { + bool failure = false; + bp->init(store, s, dialect_handler); + // Fault injection if tester wants so if ((store->ctx()->_conf->fault_inj_rename_op_copy_fail) || (store->ctx()->_conf->fault_inj_rename_op_delete_fail)) { @@ -3648,8 +3651,6 @@ void RGWRenameObj::perform_external_op(RGWOp* bp) return; } - bool failure = false; - bp->init(store, s, dialect_handler); ret = bp->init_processing(); if (ret < 0) { failure = true; From 6f8612819a6920690cea5d6cd0b30d0eac75e6fb Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Fri, 24 Jun 2016 11:45:42 +0000 Subject: [PATCH 35/60] Correct error message for fault injection --- src/rgw/rgw_op.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 083cce889aa79..a4053090314d5 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3576,11 +3576,14 @@ void RGWRenameObj::execute() if ((s->err.http_ret != 200) || (s->err.ret != 0)) { ldout(s->cct, 0) << "DSS ERROR: Copy object failed during rename op." - << " . Return status: " << s->err.ret - << " . Return HTTP code: " << s->err.http_ret - << " . Return Message: " << copy_op->get_request_state()->err.message + << " Return status: " << s->err.ret + << " Return HTTP code: " << s->err.http_ret + << " Return Message: " << copy_op->get_request_state()->err.message << dendl; - s->err.ret = -ERR_RENAME_FAILED; + if (!(store->ctx()->_conf->fault_inj_rename_op_copy_fail)) { + s->err.ret = -ERR_RENAME_FAILED; + s->err.http_ret = 400; + } return; } ldout(s->cct, 0) << "DSS INFO: Rename op: copy done" << dendl; From 57ba33c1e83e30db6977412582d287eaf2f64115 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Sun, 26 Jun 2016 19:29:48 +0000 Subject: [PATCH 36/60] Bad error code in common.h --- src/rgw/rgw_common.h | 2 +- src/rgw/rgw_http_errors.h | 4 +-- src/rgw/rgw_op.cc | 54 ++++++++++++++++++++++++++------------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 94d9e500afeb0..0248420a822b6 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -148,7 +148,7 @@ using ceph::crypto::MD5; #define ERR_BUCKET_ALREADY_OWNED 2029 #define ERR_BAD_RENAME_REQ 2030 #define ERR_RENAME_NOT_ENABLED 2031 -#define ERR_RENAME_FAILED 2022 +#define ERR_RENAME_FAILED 2032 #define ERR_RENAME_DATA_LOST 2033 #define ERR_RENAME_COPY_FAILED 2034 #define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index 331243873045a..2ad89cdeeac6a 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -35,7 +35,6 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_TOO_LARGE, 400, "EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size (5GB in our case)."}, { ERR_TOO_SMALL, 400, "EntityTooSmall", "Your proposed upload is smaller than the minimum allowed size (1 MB in our case for multipart upload)"}, { ERR_TOO_MANY_BUCKETS, 400, "TooManyBuckets" , "You have attempted to create more buckets than allowed"}, - { ERR_RENAME_FAILED, 400, "RenameFailed", "Rename operation has failed." }, {ERR_BUCKET_ALREADY_OWNED, 409, "BucketAlreadyOwnedByYou" , "Your previous request to create the named bucket succeeded and you already own it."}, //{ ERR_MALFORMED_XML, 400, "MalformedXML" }, { ERR_LENGTH_REQUIRED, 411, "MissingContentLength", "You must provide the Content-Length HTTP header."}, @@ -46,9 +45,10 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_USER_SUSPENDED, 403, "UserSuspended" }, { ERR_REQUEST_TIME_SKEWED, 403, "RequestTimeTooSkewed" ,"The difference between the request time and the server's time is too large."}, { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" }, - { ERR_BAD_RENAME_REQ, 403, "BadRenameRequest", "Rename request must have object name, new object name and the HTTP method should be PUT." }, + { ERR_BAD_RENAME_REQ, 403, "BadRenameRequest", "Rename request must have object name, new object name and the HTTP method should be PUT."}, { ERR_RENAME_NOT_ENABLED, 403, "RenameDisabled", "Rename operation is not enabled"}, { ERR_RENAME_FAULT_INJ, 403, "FaultInjection", "A rename operation tester fault has been activated"}, + { ERR_RENAME_FAILED, 403, "RenameFailed", "Rename operation has failed."}, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index a4053090314d5..7d4ca085d42af 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3559,14 +3559,22 @@ void RGWRenameObj::execute() ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object); if (!ret || (store->ctx()->_conf->fault_inj_rename_op_parse_fail)) { ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; - s->err.ret = -ERR_RENAME_FAILED; + s->err.http_ret = 403; + s->err.ret = -ERR_RENAME_FAULT_INJ; return; } } /* Perform copy operation */ RGWCopyObj_ObjStore_S3* copy_op = new RGWCopyObj_ObjStore_S3; - perform_external_op(copy_op); + + // Fault injection + if (!(store->ctx()->_conf->fault_inj_rename_op_copy_fail)) { + perform_external_op(copy_op); + } else { + s->err.http_ret = 403; + s->err.ret = -ERR_RENAME_FAULT_INJ; + } if ((store->ctx()->_conf->fault_inj_rename_op_sleep_after_copy)) { // Fault injection to test atomicity @@ -3592,13 +3600,20 @@ void RGWRenameObj::execute() new_obj.dss_duplicate(&(s->object)); (s->object).dss_duplicate(&orig_object); RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; - delete_rgw_object(del_op); + + // Fault injection + if (!(store->ctx()->_conf->fault_inj_rename_op_delete_fail)) { + delete_rgw_object(del_op); + } else { + s->err.http_ret = 403; + s->err.ret = -ERR_RENAME_FAULT_INJ; + } + if ((s->err.http_ret != 200) || (s->err.ret != 0)) { ldout(s->cct, 0) << "DSS ERROR: Delete object failed during rename op." - << " . Return status: " << s->err.ret - << " . Return HTTP code: " << s->err.http_ret - << " . Return Message: " << del_op->get_request_state()->err.message + << " Return status: " << s->err.ret + << " Return HTTP code: " << s->err.http_ret << dendl; /* Revert the copy op */ @@ -3610,6 +3625,7 @@ void RGWRenameObj::execute() if (ret_newobj < 0) { // We are in a soup. Data lost. This is not the case we will ever end up in. s->err.ret = -ERR_RENAME_DATA_LOST; + s->err.http_ret = 500; } else { // Everything normal if (!store->ctx()->_conf->fault_inj_rename_op_delete_fail) { @@ -3621,23 +3637,30 @@ void RGWRenameObj::execute() if (ret_newobj >= 0) { // Delete the new object (s->object).dss_duplicate(&new_obj); - RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; - delete_rgw_object(del_op); + RGWDeleteObj_ObjStore_S3* ndel_op = new RGWDeleteObj_ObjStore_S3; + delete_rgw_object(ndel_op); + if ((s->err.http_ret != 200) || (s->err.ret != 0)) { + ldout(s->cct, 0) << "DSS INFO: New object del failed. Status: " + << s->err.http_ret << " Ret: " << s->err.ret << dendl; s->err.ret = -ERR_RENAME_NEW_OBJ_DEL_FAILED; + s->err.http_ret = 500; } else { // Indicate that there has been a failure s->err.ret = -ERR_RENAME_FAILED; + s->err.http_ret = 403; + ldout(s->cct, 0) << "DSS INFO: Taking the right route" << dendl; } } else { // Log major error. Ask user to file a bug. Why didn't copy fail? s->err.ret = -ERR_RENAME_COPY_FAILED; + s->err.http_ret = 500; } } return; } - ldout(s->cct, 0) << "DSS INFO: Rename op complete" << dendl; + ldout(s->cct, 0) << "DSS INFO: Rename op complete" << dendl; return; } @@ -3646,14 +3669,6 @@ void RGWRenameObj::perform_external_op(RGWOp* bp) bool failure = false; bp->init(store, s, dialect_handler); - // Fault injection if tester wants so - if ((store->ctx()->_conf->fault_inj_rename_op_copy_fail) || - (store->ctx()->_conf->fault_inj_rename_op_delete_fail)) { - s->err.http_ret = 403; - s->err.ret = -ERR_RENAME_FAULT_INJ; - return; - } - ret = bp->init_processing(); if (ret < 0) { failure = true; @@ -3680,11 +3695,12 @@ void RGWRenameObj::perform_external_op(RGWOp* bp) bp->execute(); s->system_request = false; s->err.ret = bp->get_request_state()->err.ret; + s->err.http_ret = bp->get_request_state()->err.http_ret; } else { s->err.ret = ret; } - s->err.http_ret = bp->get_request_state()->err.http_ret; s->err.ret = bp->get_request_state()->err.ret; + s->err.http_ret = bp->get_request_state()->err.http_ret; } int RGWRenameObj::check_obj(rgw_obj_key& object) @@ -3710,6 +3726,8 @@ void RGWRenameObj::delete_rgw_object(RGWOp* del_op) { ldout(s->cct, 0) << "DSS INFO: Deleting object. s->object.name: " << (s->object).name << dendl; + s->err.http_ret = 200; + s->err.ret = 0; perform_external_op(del_op); return; } From 3b490f4000c58618e9b50335ed7f398c4d2492ce Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Sun, 26 Jun 2016 20:11:23 +0000 Subject: [PATCH 37/60] Crash fix --- src/rgw/rgw_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 7d4ca085d42af..7ea85ff3c1189 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3586,9 +3586,9 @@ void RGWRenameObj::execute() ldout(s->cct, 0) << "DSS ERROR: Copy object failed during rename op." << " Return status: " << s->err.ret << " Return HTTP code: " << s->err.http_ret - << " Return Message: " << copy_op->get_request_state()->err.message << dendl; if (!(store->ctx()->_conf->fault_inj_rename_op_copy_fail)) { + // Cause otherwise its us who set that message to ERR_RENAME_FAULT_INJ s->err.ret = -ERR_RENAME_FAILED; s->err.http_ret = 400; } From e8afe00daed91d2146feabe778efd2b2500ca5d1 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Sun, 26 Jun 2016 23:05:45 +0000 Subject: [PATCH 38/60] Block target overwrite --- src/rgw/rgw_common.h | 1 + src/rgw/rgw_http_errors.h | 1 + src/rgw/rgw_op.cc | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 0248420a822b6..45b095a03e034 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -153,6 +153,7 @@ using ceph::crypto::MD5; #define ERR_RENAME_COPY_FAILED 2034 #define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 #define ERR_RENAME_FAULT_INJ 2036 +#define ERR_RENAME_OBJ_EXISTS 2037 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index 2ad89cdeeac6a..3b11d07f64ae3 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -49,6 +49,7 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_RENAME_NOT_ENABLED, 403, "RenameDisabled", "Rename operation is not enabled"}, { ERR_RENAME_FAULT_INJ, 403, "FaultInjection", "A rename operation tester fault has been activated"}, { ERR_RENAME_FAILED, 403, "RenameFailed", "Rename operation has failed."}, + { ERR_RENAME_OBJ_EXISTS, 403, "ObjectExists", "Rename operation cannot continue as the target object already exists."}, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 7ea85ff3c1189..42856446dead8 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3547,6 +3547,14 @@ void RGWRenameObj::execute() /* Tweek request params to make this a copy request */ (s->object).name = s->info.args.get("newname"); + ret = check_obj(s->object); + if (ret >= 0) { + ldout(s->cct, 0) << "DSS ERROR: Target object already exists." << dendl; + s->err.http_ret = 403; + s->err.ret = -ERR_RENAME_OBJ_EXISTS; + return; + } + string copysource = s->bucket_name_str; copysource.append("/"); copysource.append(orig_object.name); From 7d8411088664f58aaa6ff5457c2c8fed164a5bdb Mon Sep 17 00:00:00 2001 From: Gaurav Bafna Date: Tue, 28 Jun 2016 00:09:15 +1100 Subject: [PATCH 39/60] Fixing HTPP return code and formatting error --- src/common/Formatter.cc | 9 +++++---- src/rgw/rgw_rest.cc | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index 34dcc9cf6b530..eec5943a6733e 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -131,8 +131,6 @@ void JSONFormatter::flush(std::ostream& os) { finish_pending_string(); os << m_ss.str(); - if (m_pretty) - os << "\n"; m_ss.clear(); m_ss.str(""); } @@ -322,8 +320,11 @@ XMLFormatter::XMLFormatter(bool pretty) void XMLFormatter::flush(std::ostream& os) { finish_pending_string(); - os << m_ss.str(); - if (m_pretty) + std::string m_ss_str = m_ss.str(); + os << m_ss_str; + /* There is a small catch here. If the rest of the formatter had NO output, + * we should NOT output a newline. This primarily triggers on HTTP redirects */ + if (m_pretty && !m_ss_str.empty()) os << "\n"; m_ss.clear(); m_ss.str(""); diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 172a6691fbf9d..c954d6a6f8221 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -351,7 +351,7 @@ void dump_errno(struct req_state *s, int err) { char buf[32]; snprintf(buf, sizeof(buf), "%d", err); - dump_status(s, buf, http_status_names[s->err.http_ret]); + dump_status(s, buf, http_status_names[err]); } void dump_string_header(struct req_state *s, const char *name, const char *val) From 15558314714d7e256a430ec608d29c3e2e8dadad Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Mon, 27 Jun 2016 20:14:08 +0000 Subject: [PATCH 40/60] Atomicity --- src/rgw/rgw_op.cc | 70 +++++++++++++++++++++++++++++++++++++++--- src/rgw/rgw_op.h | 2 ++ src/rgw/rgw_rest_s3.cc | 4 +-- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 42856446dead8..c099edc7e7ede 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3537,12 +3537,19 @@ void RGWRenameObj::execute() rgw_obj_key orig_object, new_obj; orig_object.dss_duplicate(&(s->object)); - /* Check if the original object exists */ + /* Check if the original object exists and set it atomic */ ret = check_obj(orig_object); if (ret < 0) { // The passed original object does not exist s->err.ret = ret; return; + } else { + // set this object atomic + ret = set_obj_atomic(true); + if (ret < 0) { + ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " + << ret << dendl; + } } /* Tweek request params to make this a copy request */ @@ -3583,7 +3590,6 @@ void RGWRenameObj::execute() s->err.http_ret = 403; s->err.ret = -ERR_RENAME_FAULT_INJ; } - if ((store->ctx()->_conf->fault_inj_rename_op_sleep_after_copy)) { // Fault injection to test atomicity sleep(60); @@ -3601,6 +3607,16 @@ void RGWRenameObj::execute() s->err.http_ret = 400; } return; + } else { + // Need to clear atomic attribute so that delete op may go through + bool bbb; //<<<<< + get_rename_obj_atomicity(s, store, bbb); + ret = set_obj_atomic(false); + if (ret < 0) { + ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " + << ret << dendl; + } + get_rename_obj_atomicity(s, store, bbb); } ldout(s->cct, 0) << "DSS INFO: Rename op: copy done" << dendl; @@ -3619,8 +3635,8 @@ void RGWRenameObj::execute() if ((s->err.http_ret != 200) || (s->err.ret != 0)) { - ldout(s->cct, 0) << "DSS ERROR: Delete object failed during rename op." - << " Return status: " << s->err.ret + ldout(s->cct, 0) << "DSS ERROR: Delete object failed during rename op. Attempting to revert copy op." + << " Delete object Return status: " << s->err.ret << " Return HTTP code: " << s->err.http_ret << dendl; @@ -3632,11 +3648,13 @@ void RGWRenameObj::execute() // Delete failed but we don't have original object!! if (ret_newobj < 0) { // We are in a soup. Data lost. This is not the case we will ever end up in. + ldout(s->cct, 0) << "DSS ERROR: Data lost during rename operation!!" << dendl; s->err.ret = -ERR_RENAME_DATA_LOST; s->err.http_ret = 500; } else { // Everything normal if (!store->ctx()->_conf->fault_inj_rename_op_delete_fail) { + ldout(s->cct, 0) << "DSS INFO: Rename op: Why did we end up here?" << dendl; s->err.http_ret = 200; s->err.ret = 0; } @@ -3658,10 +3676,11 @@ void RGWRenameObj::execute() // Indicate that there has been a failure s->err.ret = -ERR_RENAME_FAILED; s->err.http_ret = 403; - ldout(s->cct, 0) << "DSS INFO: Taking the right route" << dendl; + ldout(s->cct, 0) << "DSS INFO: Source object delete failed. Cleaned up destination object to revert to original state." << dendl; } } else { // Log major error. Ask user to file a bug. Why didn't copy fail? + ldout(s->cct, 0) << "DSS ERROR: Delete op failure handling: Copy operation did not fail" << dendl; s->err.ret = -ERR_RENAME_COPY_FAILED; s->err.http_ret = 500; } @@ -3739,3 +3758,44 @@ void RGWRenameObj::delete_rgw_object(RGWOp* del_op) perform_external_op(del_op); return; } + +int RGWRenameObj::set_obj_atomic(bool value) +{ + string attrname = "rename_op_atomicity"; + bufferlist bl; + rgw_obj lobj(s->bucket, s->object); + store->set_atomic(s->obj_ctx, lobj); + if (value) { + bl.append("set"); + } else { + bl.append("unset"); + } + ret = store->set_attr(s->cct, lobj, attrname.c_str(), bl, NULL); + if (ret < 0) { + return ret; + } + return 0; +} + +int get_rename_obj_atomicity(req_state* s, RGWRados* store, bool& value) +{ + int ret = 0; + string attrname = "rename_op_atomicity"; + string attrval; + bufferlist bl; + + rgw_obj lobj(s->bucket, s->object); + store->set_atomic(s->obj_ctx, lobj); + ret = store->system_obj_get_attr(lobj, attrname.c_str(), bl); + if (ret < 0) { + return ret; + } + bl.copy(0, bl.length(), attrval); + ldout(s->cct, 0) << "DSS INFO: Atomicity attribute value: " << attrval << dendl; + if (attrval.compare("set") == 0) { + value = true; + } else { + value = false; + } + return 0; +} diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 40e629d3af7c6..09b98e813b2e3 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -578,8 +578,10 @@ class RGWRenameObj : public RGWOp { void perform_external_op(RGWOp*); void delete_rgw_object(RGWOp*); int check_obj(rgw_obj_key&); + int set_obj_atomic(bool); virtual const string name() { return "Rename_obj"; } }; +static int get_rename_obj_atomicity(req_state*, RGWRados*, bool&); class RGWCopyObj : public RGWOp { protected: diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 14e9984a82a50..a0066a5c68890 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -2545,9 +2545,7 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, string bufferprinter = ""; tx_buffer.copy(0, tx_buffer.length(), bufferprinter); dout(0) << "DSS INFO: \n\n" << dendl; - dout(0) << "DSS INFO: Outbound json: " << os.str() << dendl; - dout(0) << "DSS INFO: \n\n" << dendl; - dout(0) << "DSS INFO: Actual TX buffer: " << bufferprinter << dendl; + dout(0) << "DSS INFO: TX buffer: " << bufferprinter << dendl; dout(0) << "DSS INFO: \n\n" << dendl; /* Make request to IAM */ From f1b68c4e332107ca81bd80acb059d4def8bb4bbf Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Mon, 27 Jun 2016 23:33:43 +0000 Subject: [PATCH 41/60] Atomicity warning --- src/rgw/rgw_common.h | 2 ++ src/rgw/rgw_http_errors.h | 1 + src/rgw/rgw_op.cc | 57 ++++++++++++++++++++++++++++----------- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 45b095a03e034..7e5afb9f9c21c 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -69,6 +69,7 @@ using ceph::crypto::MD5; #define RGW_ATTR_SHADOW_OBJ RGW_ATTR_PREFIX "shadow_name" #define RGW_ATTR_MANIFEST RGW_ATTR_PREFIX "manifest" #define RGW_ATTR_USER_MANIFEST RGW_ATTR_PREFIX "user_manifest" +#define RGW_ATTR_RENAME_MUTEX RGW_ATTR_PREFIX "rename_mutex" #define RGW_ATTR_OLH_PREFIX RGW_ATTR_PREFIX "olh." @@ -154,6 +155,7 @@ using ceph::crypto::MD5; #define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 #define ERR_RENAME_FAULT_INJ 2036 #define ERR_RENAME_OBJ_EXISTS 2037 +#define ERR_RENAME_RUNNING 2038 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index 3b11d07f64ae3..df95a0288bbc3 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -50,6 +50,7 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_RENAME_FAULT_INJ, 403, "FaultInjection", "A rename operation tester fault has been activated"}, { ERR_RENAME_FAILED, 403, "RenameFailed", "Rename operation has failed."}, { ERR_RENAME_OBJ_EXISTS, 403, "ObjectExists", "Rename operation cannot continue as the target object already exists."}, + { ERR_RENAME_RUNNING, 403, "RenameOperationRunning", "A rename operation is running on this object. Please try later."}, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index c099edc7e7ede..d4c616e1a5f94 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1746,6 +1746,7 @@ void RGWPutObj::execute() int len; map::iterator iter; bool multipart; + bool isRenameRunning = false; bool need_calc_md5 = (obj_manifest == NULL); @@ -1756,6 +1757,13 @@ void RGWPutObj::execute() goto done; } + // Fail the put object or replace operation if a rename is running on this object + ret = get_rename_obj_atomicity(s, store, isRenameRunning); + if ((ret < 0) || isRenameRunning) { + ret = -ERR_RENAME_RUNNING; + goto done; + } + ret = get_params(); if (ret < 0) goto done; @@ -2224,6 +2232,14 @@ void RGWDeleteObj::execute() { ret = -EINVAL; rgw_obj obj(s->bucket, s->object); + + // Stop deletion of this object if a rename is running on it + bool isRenameRunning = false; + ret = get_rename_obj_atomicity(s, store, isRenameRunning); + if ((ret < 0) || isRenameRunning) { + ret = -ERR_RENAME_RUNNING; + return; + } if (!s->object.empty()) { RGWObjectCtx *obj_ctx = (RGWObjectCtx *)s->obj_ctx; @@ -3536,6 +3552,11 @@ void RGWRenameObj::execute() s->err.ret = 0; rgw_obj_key orig_object, new_obj; orig_object.dss_duplicate(&(s->object)); + string copysource; + int ret_orig, ret_newobj; + RGWCopyObj_ObjStore_S3* copy_op = NULL; + RGWDeleteObj_ObjStore_S3* del_op = NULL; + RGWDeleteObj_ObjStore_S3* ndel_op = NULL; /* Check if the original object exists and set it atomic */ ret = check_obj(orig_object); @@ -3559,10 +3580,10 @@ void RGWRenameObj::execute() ldout(s->cct, 0) << "DSS ERROR: Target object already exists." << dendl; s->err.http_ret = 403; s->err.ret = -ERR_RENAME_OBJ_EXISTS; - return; + goto rename_done; } - string copysource = s->bucket_name_str; + copysource = s->bucket_name_str; copysource.append("/"); copysource.append(orig_object.name); ldout(s->cct, 0) << "DSS INFO: Converting to copy request. s->object: " @@ -3576,12 +3597,12 @@ void RGWRenameObj::execute() ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; s->err.http_ret = 403; s->err.ret = -ERR_RENAME_FAULT_INJ; - return; + goto rename_done; } } /* Perform copy operation */ - RGWCopyObj_ObjStore_S3* copy_op = new RGWCopyObj_ObjStore_S3; + copy_op = new RGWCopyObj_ObjStore_S3; // Fault injection if (!(store->ctx()->_conf->fault_inj_rename_op_copy_fail)) { @@ -3606,24 +3627,22 @@ void RGWRenameObj::execute() s->err.ret = -ERR_RENAME_FAILED; s->err.http_ret = 400; } - return; + goto rename_done; } else { // Need to clear atomic attribute so that delete op may go through - bool bbb; //<<<<< - get_rename_obj_atomicity(s, store, bbb); ret = set_obj_atomic(false); if (ret < 0) { ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " << ret << dendl; } - get_rename_obj_atomicity(s, store, bbb); + //get_rename_obj_atomicity(s, store, bbb); } ldout(s->cct, 0) << "DSS INFO: Rename op: copy done" << dendl; /* Tweek the request for a delete obj operation and perform delete op */ new_obj.dss_duplicate(&(s->object)); (s->object).dss_duplicate(&orig_object); - RGWDeleteObj_ObjStore_S3* del_op = new RGWDeleteObj_ObjStore_S3; + del_op = new RGWDeleteObj_ObjStore_S3; // Fault injection if (!(store->ctx()->_conf->fault_inj_rename_op_delete_fail)) { @@ -3641,7 +3660,6 @@ void RGWRenameObj::execute() << dendl; /* Revert the copy op */ - int ret_orig, ret_newobj; ret_orig = check_obj(s->object); ret_newobj = check_obj(new_obj); if (ret_orig < 0) { @@ -3663,7 +3681,7 @@ void RGWRenameObj::execute() if (ret_newobj >= 0) { // Delete the new object (s->object).dss_duplicate(&new_obj); - RGWDeleteObj_ObjStore_S3* ndel_op = new RGWDeleteObj_ObjStore_S3; + ndel_op = new RGWDeleteObj_ObjStore_S3; delete_rgw_object(ndel_op); if ((s->err.http_ret != 200) || @@ -3685,9 +3703,17 @@ void RGWRenameObj::execute() s->err.http_ret = 500; } } - return; + goto rename_done; } + ldout(s->cct, 0) << "DSS INFO: Rename op complete" << dendl; +rename_done: + // Clear atomicity + ret = set_obj_atomic(false); + if (ret < 0) { + ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " + << ret << dendl; + } return; } @@ -3761,7 +3787,6 @@ void RGWRenameObj::delete_rgw_object(RGWOp* del_op) int RGWRenameObj::set_obj_atomic(bool value) { - string attrname = "rename_op_atomicity"; bufferlist bl; rgw_obj lobj(s->bucket, s->object); store->set_atomic(s->obj_ctx, lobj); @@ -3770,7 +3795,8 @@ int RGWRenameObj::set_obj_atomic(bool value) } else { bl.append("unset"); } - ret = store->set_attr(s->cct, lobj, attrname.c_str(), bl, NULL); + //RGWObjVersionTracker* objv_tracker; + ret = store->set_attr(s->obj_ctx, lobj, RGW_ATTR_RENAME_MUTEX, bl, NULL); if (ret < 0) { return ret; } @@ -3780,13 +3806,12 @@ int RGWRenameObj::set_obj_atomic(bool value) int get_rename_obj_atomicity(req_state* s, RGWRados* store, bool& value) { int ret = 0; - string attrname = "rename_op_atomicity"; string attrval; bufferlist bl; rgw_obj lobj(s->bucket, s->object); store->set_atomic(s->obj_ctx, lobj); - ret = store->system_obj_get_attr(lobj, attrname.c_str(), bl); + ret = store->system_obj_get_attr(lobj, RGW_ATTR_RENAME_MUTEX, bl); if (ret < 0) { return ret; } From edbf7979e2661e25349f8ff21fd508cda9511575 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Tue, 28 Jun 2016 00:32:24 +0000 Subject: [PATCH 42/60] Making faliures probabalistic --- src/rgw/rgw_op.cc | 51 +++++++++++++++++++++++++++++++++++++++++------ src/rgw/rgw_op.h | 8 ++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index d4c616e1a5f94..a991bf5ebb23d 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1759,7 +1759,8 @@ void RGWPutObj::execute() // Fail the put object or replace operation if a rename is running on this object ret = get_rename_obj_atomicity(s, store, isRenameRunning); - if ((ret < 0) || isRenameRunning) { + if ((ret >= 0) && isRenameRunning) { + ldout(s->cct, 0) << "DSS Rename atomicity: ret was" << ret << " Bool was: " << isRenameRunning << dendl; ret = -ERR_RENAME_RUNNING; goto done; } @@ -2236,7 +2237,7 @@ void RGWDeleteObj::execute() // Stop deletion of this object if a rename is running on it bool isRenameRunning = false; ret = get_rename_obj_atomicity(s, store, isRenameRunning); - if ((ret < 0) || isRenameRunning) { + if ((ret >= 0) && isRenameRunning) { ret = -ERR_RENAME_RUNNING; return; } @@ -3605,7 +3606,7 @@ void RGWRenameObj::execute() copy_op = new RGWCopyObj_ObjStore_S3; // Fault injection - if (!(store->ctx()->_conf->fault_inj_rename_op_copy_fail)) { + if (!fail_copy()) { perform_external_op(copy_op); } else { s->err.http_ret = 403; @@ -3635,7 +3636,6 @@ void RGWRenameObj::execute() ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " << ret << dendl; } - //get_rename_obj_atomicity(s, store, bbb); } ldout(s->cct, 0) << "DSS INFO: Rename op: copy done" << dendl; @@ -3645,7 +3645,7 @@ void RGWRenameObj::execute() del_op = new RGWDeleteObj_ObjStore_S3; // Fault injection - if (!(store->ctx()->_conf->fault_inj_rename_op_delete_fail)) { + if (!fail_delete()) { delete_rgw_object(del_op); } else { s->err.http_ret = 403; @@ -3795,7 +3795,6 @@ int RGWRenameObj::set_obj_atomic(bool value) } else { bl.append("unset"); } - //RGWObjVersionTracker* objv_tracker; ret = store->set_attr(s->obj_ctx, lobj, RGW_ATTR_RENAME_MUTEX, bl, NULL); if (ret < 0) { return ret; @@ -3824,3 +3823,43 @@ int get_rename_obj_atomicity(req_state* s, RGWRados* store, bool& value) } return 0; } + +#ifdef RENAME_OP_TESTING_FAULTS +bool RGWRenameObj::fail_copy() +{ + if(store->ctx()->_conf->fault_inj_rename_op_copy_fail) { + int num = getsRandInt(); + if (num % 3) + return true; + } + return false; +} + +bool RGWRenameObj::fail_parse() +{ + if(store->ctx()->_conf->fault_inj_rename_op_parse_fail) { + int num = getsRandInt(); + if (num % 4) + return true; + } + return false; +} + +bool RGWRenameObj::fail_delete() +{ + if(store->ctx()->_conf->fault_inj_rename_op_delete_fail) { + int num = getsRandInt(); + if (num % 2) + return true; + } + return false; +} + +int RGWRenameObj::getsRandInt() +{ + int num = 0; + srand(time(NULL)); + num = rand() % 10; + return num; +} +#endif diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 09b98e813b2e3..6819a554ab102 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -25,6 +25,8 @@ #include "rgw_cors.h" #include "rgw_quota.h" +#define RENAME_OP_TESTING_FAULTS 1 + using namespace std; struct req_state; @@ -580,6 +582,12 @@ class RGWRenameObj : public RGWOp { int check_obj(rgw_obj_key&); int set_obj_atomic(bool); virtual const string name() { return "Rename_obj"; } +#ifdef RENAME_OP_TESTING_FAULTS + bool fail_copy(); + bool fail_parse(); + bool fail_delete(); + int getsRandInt(); +#endif }; static int get_rename_obj_atomicity(req_state*, RGWRados*, bool&); From 964ac4ed1f97188a84b9728be080b787be30725e Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Tue, 28 Jun 2016 12:00:29 +0000 Subject: [PATCH 43/60] Real requirements --- src/rgw/rgw_op.cc | 51 ++++++++++++++++++++++++++++------------------- src/rgw/rgw_op.h | 4 ++-- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index a991bf5ebb23d..ea84aec970f4e 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1757,13 +1757,13 @@ void RGWPutObj::execute() goto done; } - // Fail the put object or replace operation if a rename is running on this object + /* Fail the put object or replace operation if a rename is running on this object ret = get_rename_obj_atomicity(s, store, isRenameRunning); if ((ret >= 0) && isRenameRunning) { ldout(s->cct, 0) << "DSS Rename atomicity: ret was" << ret << " Bool was: " << isRenameRunning << dendl; ret = -ERR_RENAME_RUNNING; goto done; - } + }*/ ret = get_params(); if (ret < 0) @@ -2234,13 +2234,14 @@ void RGWDeleteObj::execute() ret = -EINVAL; rgw_obj obj(s->bucket, s->object); - // Stop deletion of this object if a rename is running on it + /* Stop deletion of this object if a rename is running on it bool isRenameRunning = false; ret = get_rename_obj_atomicity(s, store, isRenameRunning); if ((ret >= 0) && isRenameRunning) { ret = -ERR_RENAME_RUNNING; return; - } + }*/ + if (!s->object.empty()) { RGWObjectCtx *obj_ctx = (RGWObjectCtx *)s->obj_ctx; @@ -3565,14 +3566,14 @@ void RGWRenameObj::execute() // The passed original object does not exist s->err.ret = ret; return; - } else { + }/* else { // set this object atomic ret = set_obj_atomic(true); if (ret < 0) { ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " << ret << dendl; } - } + }*/ /* Tweek request params to make this a copy request */ (s->object).name = s->info.args.get("newname"); @@ -3594,7 +3595,7 @@ void RGWRenameObj::execute() s->copy_source = s->info.env->get("HTTP_X_JCS_COPY_SOURCE"); if (s->copy_source) { ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object); - if (!ret || (store->ctx()->_conf->fault_inj_rename_op_parse_fail)) { + if (!ret || fail_parse()) { ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; s->err.http_ret = 403; s->err.ret = -ERR_RENAME_FAULT_INJ; @@ -3606,11 +3607,11 @@ void RGWRenameObj::execute() copy_op = new RGWCopyObj_ObjStore_S3; // Fault injection - if (!fail_copy()) { - perform_external_op(copy_op); - } else { + if (fail_copy()) { s->err.http_ret = 403; s->err.ret = -ERR_RENAME_FAULT_INJ; + } else { + perform_external_op(copy_op); } if ((store->ctx()->_conf->fault_inj_rename_op_sleep_after_copy)) { // Fault injection to test atomicity @@ -3629,14 +3630,14 @@ void RGWRenameObj::execute() s->err.http_ret = 400; } goto rename_done; - } else { + }/* else { // Need to clear atomic attribute so that delete op may go through ret = set_obj_atomic(false); if (ret < 0) { ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " << ret << dendl; } - } + }*/ ldout(s->cct, 0) << "DSS INFO: Rename op: copy done" << dendl; /* Tweek the request for a delete obj operation and perform delete op */ @@ -3645,11 +3646,11 @@ void RGWRenameObj::execute() del_op = new RGWDeleteObj_ObjStore_S3; // Fault injection - if (!fail_delete()) { - delete_rgw_object(del_op); - } else { + if (fail_delete()) { s->err.http_ret = 403; s->err.ret = -ERR_RENAME_FAULT_INJ; + } else { + delete_rgw_object(del_op); } if ((s->err.http_ret != 200) || @@ -3709,11 +3710,12 @@ void RGWRenameObj::execute() ldout(s->cct, 0) << "DSS INFO: Rename op complete" << dendl; rename_done: // Clear atomicity + /* ret = set_obj_atomic(false); if (ret < 0) { ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " << ret << dendl; - } + }*/ return; } @@ -3785,6 +3787,7 @@ void RGWRenameObj::delete_rgw_object(RGWOp* del_op) return; } +/* int RGWRenameObj::set_obj_atomic(bool value) { bufferlist bl; @@ -3823,14 +3826,17 @@ int get_rename_obj_atomicity(req_state* s, RGWRados* store, bool& value) } return 0; } +*/ #ifdef RENAME_OP_TESTING_FAULTS bool RGWRenameObj::fail_copy() { if(store->ctx()->_conf->fault_inj_rename_op_copy_fail) { int num = getsRandInt(); - if (num % 3) + if (num % 11 == 0) { + ldout(s->cct, 0) << "DSS RANDOM: failing copy " << num << dendl; return true; + } } return false; } @@ -3839,8 +3845,10 @@ bool RGWRenameObj::fail_parse() { if(store->ctx()->_conf->fault_inj_rename_op_parse_fail) { int num = getsRandInt(); - if (num % 4) + if (num % 13 == 0) { + ldout(s->cct, 0) << "DSS RANDOM: failing parse " << num << dendl; return true; + } } return false; } @@ -3849,8 +3857,10 @@ bool RGWRenameObj::fail_delete() { if(store->ctx()->_conf->fault_inj_rename_op_delete_fail) { int num = getsRandInt(); - if (num % 2) + if (num % 17 == 0) { + ldout(s->cct, 0) << "DSS RANDOM: failing delete " << num << dendl; return true; + } } return false; } @@ -3859,7 +3869,8 @@ int RGWRenameObj::getsRandInt() { int num = 0; srand(time(NULL)); - num = rand() % 10; + num = rand() % 100; + ldout(s->cct, 0) << "DSS RANDOM: Num is " << num << dendl; return num; } #endif diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 6819a554ab102..03b614809f6bd 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -580,7 +580,7 @@ class RGWRenameObj : public RGWOp { void perform_external_op(RGWOp*); void delete_rgw_object(RGWOp*); int check_obj(rgw_obj_key&); - int set_obj_atomic(bool); +// int set_obj_atomic(bool); virtual const string name() { return "Rename_obj"; } #ifdef RENAME_OP_TESTING_FAULTS bool fail_copy(); @@ -589,7 +589,7 @@ class RGWRenameObj : public RGWOp { int getsRandInt(); #endif }; -static int get_rename_obj_atomicity(req_state*, RGWRados*, bool&); +//int get_rename_obj_atomicity(req_state*, RGWRados*, bool&); class RGWCopyObj : public RGWOp { protected: From d2b12f232de0ef17b54dc1f0f2fac974f82c1b6b Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Tue, 28 Jun 2016 13:19:10 +0000 Subject: [PATCH 44/60] Cleaning up testing code --- src/common/config_opts.h | 4 - src/rgw/rgw_common.h | 2 - src/rgw/rgw_http_errors.h | 2 - src/rgw/rgw_op.cc | 185 ++++---------------------------------- src/rgw/rgw_op.h | 8 -- 5 files changed, 18 insertions(+), 183 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index b1021df13b19c..07e30ba0429e2 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1078,10 +1078,6 @@ OPTION(rgw_keystone_url_token_api, OPT_STR, "url-auth") // api to validate pre OPTION(rgw_keystone_infinite_url_token_api, OPT_STR, "preauth-token-auth") // api to validate infinite time presigned token URL OPTION(dss_regional_url, OPT_STR, "https://dss.ind-west-1.staging.jiocloudservices.com") // URL to be returned in XMLNS during anonymous list all buckets calls OPTION(rgw_enable_rename_op, OPT_BOOL, true) // Enable the atomic rename op -OPTION(fault_inj_rename_op_copy_fail, OPT_BOOL, false) // Injects fault in rename op during copy operation -OPTION(fault_inj_rename_op_delete_fail, OPT_BOOL, false) // Injects fault in rename op during delete operation -OPTION(fault_inj_rename_op_parse_fail, OPT_BOOL, false) // Injects fault in rename op during parse operation -OPTION(fault_inj_rename_op_sleep_after_copy, OPT_BOOL, false) // Make rename op sleep after copy OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf counter diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 7e5afb9f9c21c..9c392b495224c 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -153,9 +153,7 @@ using ceph::crypto::MD5; #define ERR_RENAME_DATA_LOST 2033 #define ERR_RENAME_COPY_FAILED 2034 #define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 -#define ERR_RENAME_FAULT_INJ 2036 #define ERR_RENAME_OBJ_EXISTS 2037 -#define ERR_RENAME_RUNNING 2038 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index df95a0288bbc3..efa431e78d04f 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -47,10 +47,8 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" }, { ERR_BAD_RENAME_REQ, 403, "BadRenameRequest", "Rename request must have object name, new object name and the HTTP method should be PUT."}, { ERR_RENAME_NOT_ENABLED, 403, "RenameDisabled", "Rename operation is not enabled"}, - { ERR_RENAME_FAULT_INJ, 403, "FaultInjection", "A rename operation tester fault has been activated"}, { ERR_RENAME_FAILED, 403, "RenameFailed", "Rename operation has failed."}, { ERR_RENAME_OBJ_EXISTS, 403, "ObjectExists", "Rename operation cannot continue as the target object already exists."}, - { ERR_RENAME_RUNNING, 403, "RenameOperationRunning", "A rename operation is running on this object. Please try later."}, { ENOENT, 404, "NoSuchKey", "Resource not found."}, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index ea84aec970f4e..2a422a1997745 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1757,14 +1757,6 @@ void RGWPutObj::execute() goto done; } - /* Fail the put object or replace operation if a rename is running on this object - ret = get_rename_obj_atomicity(s, store, isRenameRunning); - if ((ret >= 0) && isRenameRunning) { - ldout(s->cct, 0) << "DSS Rename atomicity: ret was" << ret << " Bool was: " << isRenameRunning << dendl; - ret = -ERR_RENAME_RUNNING; - goto done; - }*/ - ret = get_params(); if (ret < 0) goto done; @@ -2234,14 +2226,6 @@ void RGWDeleteObj::execute() ret = -EINVAL; rgw_obj obj(s->bucket, s->object); - /* Stop deletion of this object if a rename is running on it - bool isRenameRunning = false; - ret = get_rename_obj_atomicity(s, store, isRenameRunning); - if ((ret >= 0) && isRenameRunning) { - ret = -ERR_RENAME_RUNNING; - return; - }*/ - if (!s->object.empty()) { RGWObjectCtx *obj_ctx = (RGWObjectCtx *)s->obj_ctx; @@ -3532,8 +3516,6 @@ void RGWHandler::put_op(RGWOp *op) delete op; } - - /* Object Rename operation */ int RGWRenameObj::verify_permission() @@ -3560,20 +3542,13 @@ void RGWRenameObj::execute() RGWDeleteObj_ObjStore_S3* del_op = NULL; RGWDeleteObj_ObjStore_S3* ndel_op = NULL; - /* Check if the original object exists and set it atomic */ + /* Check if the original object exists */ ret = check_obj(orig_object); if (ret < 0) { // The passed original object does not exist s->err.ret = ret; return; - }/* else { - // set this object atomic - ret = set_obj_atomic(true); - if (ret < 0) { - ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " - << ret << dendl; - } - }*/ + } /* Tweek request params to make this a copy request */ (s->object).name = s->info.args.get("newname"); @@ -3582,7 +3557,7 @@ void RGWRenameObj::execute() ldout(s->cct, 0) << "DSS ERROR: Target object already exists." << dendl; s->err.http_ret = 403; s->err.ret = -ERR_RENAME_OBJ_EXISTS; - goto rename_done; + return; } copysource = s->bucket_name_str; @@ -3595,28 +3570,17 @@ void RGWRenameObj::execute() s->copy_source = s->info.env->get("HTTP_X_JCS_COPY_SOURCE"); if (s->copy_source) { ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object); - if (!ret || fail_parse()) { + if (!ret) { //Surprizingly returns bool ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; + s->err.ret = -ERR_RENAME_FAILED; s->err.http_ret = 403; - s->err.ret = -ERR_RENAME_FAULT_INJ; - goto rename_done; + return; } } /* Perform copy operation */ copy_op = new RGWCopyObj_ObjStore_S3; - - // Fault injection - if (fail_copy()) { - s->err.http_ret = 403; - s->err.ret = -ERR_RENAME_FAULT_INJ; - } else { - perform_external_op(copy_op); - } - if ((store->ctx()->_conf->fault_inj_rename_op_sleep_after_copy)) { - // Fault injection to test atomicity - sleep(60); - } + perform_external_op(copy_op); if ((s->err.http_ret != 200) || (s->err.ret != 0)) { @@ -3624,34 +3588,17 @@ void RGWRenameObj::execute() << " Return status: " << s->err.ret << " Return HTTP code: " << s->err.http_ret << dendl; - if (!(store->ctx()->_conf->fault_inj_rename_op_copy_fail)) { - // Cause otherwise its us who set that message to ERR_RENAME_FAULT_INJ - s->err.ret = -ERR_RENAME_FAILED; - s->err.http_ret = 400; - } - goto rename_done; - }/* else { - // Need to clear atomic attribute so that delete op may go through - ret = set_obj_atomic(false); - if (ret < 0) { - ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " - << ret << dendl; - } - }*/ + s->err.ret = -ERR_RENAME_FAILED; + s->err.http_ret = 403; + return; + } ldout(s->cct, 0) << "DSS INFO: Rename op: copy done" << dendl; /* Tweek the request for a delete obj operation and perform delete op */ new_obj.dss_duplicate(&(s->object)); (s->object).dss_duplicate(&orig_object); del_op = new RGWDeleteObj_ObjStore_S3; - - // Fault injection - if (fail_delete()) { - s->err.http_ret = 403; - s->err.ret = -ERR_RENAME_FAULT_INJ; - } else { - delete_rgw_object(del_op); - } + delete_rgw_object(del_op); if ((s->err.http_ret != 200) || (s->err.ret != 0)) { @@ -3672,11 +3619,9 @@ void RGWRenameObj::execute() s->err.http_ret = 500; } else { // Everything normal - if (!store->ctx()->_conf->fault_inj_rename_op_delete_fail) { - ldout(s->cct, 0) << "DSS INFO: Rename op: Why did we end up here?" << dendl; - s->err.http_ret = 200; - s->err.ret = 0; - } + ldout(s->cct, 0) << "DSS INFO: Rename op: Why did we end up here?" << dendl; + s->err.http_ret = 200; + s->err.ret = 0; } } else { if (ret_newobj >= 0) { @@ -3693,9 +3638,10 @@ void RGWRenameObj::execute() s->err.http_ret = 500; } else { // Indicate that there has been a failure + ldout(s->cct, 0) << "DSS INFO: Source object delete failed." + << "Cleaned up destination object to revert to original state." << dendl; s->err.ret = -ERR_RENAME_FAILED; s->err.http_ret = 403; - ldout(s->cct, 0) << "DSS INFO: Source object delete failed. Cleaned up destination object to revert to original state." << dendl; } } else { // Log major error. Ask user to file a bug. Why didn't copy fail? @@ -3704,18 +3650,10 @@ void RGWRenameObj::execute() s->err.http_ret = 500; } } - goto rename_done; + return; } ldout(s->cct, 0) << "DSS INFO: Rename op complete" << dendl; -rename_done: - // Clear atomicity - /* - ret = set_obj_atomic(false); - if (ret < 0) { - ldout(s->cct, 0) << "DSS ERROR: Failed to set atomicity attribute on the object. Error code: " - << ret << dendl; - }*/ return; } @@ -3787,90 +3725,3 @@ void RGWRenameObj::delete_rgw_object(RGWOp* del_op) return; } -/* -int RGWRenameObj::set_obj_atomic(bool value) -{ - bufferlist bl; - rgw_obj lobj(s->bucket, s->object); - store->set_atomic(s->obj_ctx, lobj); - if (value) { - bl.append("set"); - } else { - bl.append("unset"); - } - ret = store->set_attr(s->obj_ctx, lobj, RGW_ATTR_RENAME_MUTEX, bl, NULL); - if (ret < 0) { - return ret; - } - return 0; -} - -int get_rename_obj_atomicity(req_state* s, RGWRados* store, bool& value) -{ - int ret = 0; - string attrval; - bufferlist bl; - - rgw_obj lobj(s->bucket, s->object); - store->set_atomic(s->obj_ctx, lobj); - ret = store->system_obj_get_attr(lobj, RGW_ATTR_RENAME_MUTEX, bl); - if (ret < 0) { - return ret; - } - bl.copy(0, bl.length(), attrval); - ldout(s->cct, 0) << "DSS INFO: Atomicity attribute value: " << attrval << dendl; - if (attrval.compare("set") == 0) { - value = true; - } else { - value = false; - } - return 0; -} -*/ - -#ifdef RENAME_OP_TESTING_FAULTS -bool RGWRenameObj::fail_copy() -{ - if(store->ctx()->_conf->fault_inj_rename_op_copy_fail) { - int num = getsRandInt(); - if (num % 11 == 0) { - ldout(s->cct, 0) << "DSS RANDOM: failing copy " << num << dendl; - return true; - } - } - return false; -} - -bool RGWRenameObj::fail_parse() -{ - if(store->ctx()->_conf->fault_inj_rename_op_parse_fail) { - int num = getsRandInt(); - if (num % 13 == 0) { - ldout(s->cct, 0) << "DSS RANDOM: failing parse " << num << dendl; - return true; - } - } - return false; -} - -bool RGWRenameObj::fail_delete() -{ - if(store->ctx()->_conf->fault_inj_rename_op_delete_fail) { - int num = getsRandInt(); - if (num % 17 == 0) { - ldout(s->cct, 0) << "DSS RANDOM: failing delete " << num << dendl; - return true; - } - } - return false; -} - -int RGWRenameObj::getsRandInt() -{ - int num = 0; - srand(time(NULL)); - num = rand() % 100; - ldout(s->cct, 0) << "DSS RANDOM: Num is " << num << dendl; - return num; -} -#endif diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 03b614809f6bd..b396687bf28f3 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -580,16 +580,8 @@ class RGWRenameObj : public RGWOp { void perform_external_op(RGWOp*); void delete_rgw_object(RGWOp*); int check_obj(rgw_obj_key&); -// int set_obj_atomic(bool); virtual const string name() { return "Rename_obj"; } -#ifdef RENAME_OP_TESTING_FAULTS - bool fail_copy(); - bool fail_parse(); - bool fail_delete(); - int getsRandInt(); -#endif }; -//int get_rename_obj_atomicity(req_state*, RGWRados*, bool&); class RGWCopyObj : public RGWOp { protected: From df949f63e73324e6b6b8862ef7d1f4d54ed48623 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Tue, 28 Jun 2016 15:16:32 +0000 Subject: [PATCH 45/60] Unused vars cleanup --- src/rgw/rgw_common.h | 3 +-- src/rgw/rgw_op.cc | 1 - src/rgw/rgw_op.h | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 9c392b495224c..c99a2225cf424 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -69,7 +69,6 @@ using ceph::crypto::MD5; #define RGW_ATTR_SHADOW_OBJ RGW_ATTR_PREFIX "shadow_name" #define RGW_ATTR_MANIFEST RGW_ATTR_PREFIX "manifest" #define RGW_ATTR_USER_MANIFEST RGW_ATTR_PREFIX "user_manifest" -#define RGW_ATTR_RENAME_MUTEX RGW_ATTR_PREFIX "rename_mutex" #define RGW_ATTR_OLH_PREFIX RGW_ATTR_PREFIX "olh." @@ -153,7 +152,7 @@ using ceph::crypto::MD5; #define ERR_RENAME_DATA_LOST 2033 #define ERR_RENAME_COPY_FAILED 2034 #define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 -#define ERR_RENAME_OBJ_EXISTS 2037 +#define ERR_RENAME_OBJ_EXISTS 2036 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 2a422a1997745..7107d995d9c7c 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1746,7 +1746,6 @@ void RGWPutObj::execute() int len; map::iterator iter; bool multipart; - bool isRenameRunning = false; bool need_calc_md5 = (obj_manifest == NULL); diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index b396687bf28f3..40e629d3af7c6 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -25,8 +25,6 @@ #include "rgw_cors.h" #include "rgw_quota.h" -#define RENAME_OP_TESTING_FAULTS 1 - using namespace std; struct req_state; From 5b39d683774a28c9d3ab73f1a6bca195b7308469 Mon Sep 17 00:00:00 2001 From: Shivanshu Goswami Date: Tue, 28 Jun 2016 15:17:58 +0000 Subject: [PATCH 46/60] Removing dead code --- src/rgw/rgw_rados.h | 83 --------------------------------------------- 1 file changed, 83 deletions(-) diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 0b4a782f65d24..08a703138188c 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1296,89 +1296,6 @@ class RGWRados zone_name = name; } - - /* Holds info on whether the request should be - * validated via EC2 signature or Auth tokens. - * Holds value when the action is COPY - * Holds value when the token to be validated is from a presigned URL */ -/* - class authorization_method { - private: - bool _token_validation; - bool _copy_action; - bool _url_type_token; - bool _acl_main_override; - bool _acl_copy_override; - string _token; - string _copy_source; - - public: - inline bool get_token_validation() - { - return _token_validation; - } - inline void set_token_validation(bool method) - { - _token_validation = method; - } - inline string get_token() - { - return _token; - } - inline void set_token(string tok) - { - _token = tok; - } - inline bool get_copy_action() - { - return _copy_action; - } - inline void set_copy_action(bool action) - { - _copy_action = action; - } - inline string get_copy_source() - { - return _copy_source; - } - inline void set_copy_source(string source) - { - _copy_source = source; - } - inline bool get_url_type_token() - { - return _url_type_token; - } - inline void set_url_type_token(bool val) - { - _url_type_token = val; - } - inline bool get_acl_main_override() - { - return _acl_main_override; - } - inline void set_acl_main_override(bool val) - { - _acl_main_override = val; - } - inline bool get_acl_copy_override() - { - return _acl_copy_override; - } - inline void set_acl_copy_override(bool val) - { - _acl_copy_override = val; - } - - - authorization_method(bool method, bool action, bool url_token) : - _token_validation(method), - _copy_action(action), - _url_type_token(url_token) { } - ~authorization_method() { } - } auth_method; - */ - RGWRegion region; RGWZoneParams zone; /* internal zone params, e.g., rados pools */ RGWZone zone_public_config; /* external zone params, e.g., entrypoints, log flags, etc. */ From 9e286e4504ff7773bc4f3f205c2354a48fbfbd90 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna Date: Wed, 6 Jul 2016 16:16:02 +1100 Subject: [PATCH 47/60] Removing useless file --- src/rest_blame | 1724 ------------------------------------------------ 1 file changed, 1724 deletions(-) delete mode 100644 src/rest_blame diff --git a/src/rest_blame b/src/rest_blame deleted file mode 100644 index 990a98208220c..0000000000000 --- a/src/rest_blame +++ /dev/null @@ -1,1724 +0,0 @@ -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 1) // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 2) // vim: ts=8 sw=2 smarttab -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 3) /* -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 4) * Ceph - scalable distributed file system -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 5) * -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 6) * Copyright (C) 2004-2009 Sage Weil -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 7) * -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 8) * This is free software; you can redistribute it and/or -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 9) * modify it under the terms of the GNU Lesser General Public -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 10) * License version 2.1, as published by the Free Software -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 11) * Foundation. See file COPYING. -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 12) * -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 13) */ -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 14) -f4b9d9d8 src/rgw/rgw_common.h (Markus Elfring 2010-06-12 15:04:11 +0200 15) #ifndef CEPH_RGW_COMMON_H -f4b9d9d8 src/rgw/rgw_common.h (Markus Elfring 2010-06-12 15:04:11 +0200 16) #define CEPH_RGW_COMMON_H -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 17) -d95367cb src/rgw/rgw_common.h (Tommi Virtanen 2011-03-08 13:49:56 -0800 18) #include "common/ceph_crypto.h" -1c98da66 src/rgw/rgw_common.h (Colin P. McCabe 2011-05-10 14:08:42 -0700 19) #include "common/debug.h" -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 20) #include "common/perf_counters.h" -468c7dce src/rgw/rgw_common.h (Sage Weil 2011-10-09 15:27:10 -0700 21) -468c7dce src/rgw/rgw_common.h (Sage Weil 2011-10-09 15:27:10 -0700 22) #include "acconfig.h" -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 23) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 24) #include -f2424dfb src/rgw/rgw_common.h (Yehuda Sadeh 2010-12-03 14:45:59 -0800 25) #include -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 26) #include -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 27) #include -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 28) #include "include/types.h" -c9135519 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-08 16:49:49 -0800 29) #include "include/utime.h" -e345dfe0 src/rgw/rgw_common.h (Caleb Miles 2013-02-05 14:10:03 -0500 30) #include "rgw_acl.h" -f165049c src/rgw/rgw_common.h (Babu Shanmugam 2013-03-05 09:22:55 +0530 31) #include "rgw_cors.h" -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 32) #include "rgw_quota.h" -fe6cd9bc src/rgw/rgw_common.h (Yehuda Sadeh 2013-10-22 13:53:59 -0700 33) #include "rgw_string.h" -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 34) #include "rgw_website.h" -a0d238c3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-10 22:02:20 -0700 35) #include "cls/version/cls_version_types.h" -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 36) #include "cls/user/cls_user_types.h" -54f2e0ac src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 17:31:12 -0700 37) #include "cls/rgw/cls_rgw_types.h" -a0d238c3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-10 22:02:20 -0700 38) #include "include/rados/librados.hpp" -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 39) -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 40) using namespace std; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 41) -76880e3e src/rgw/rgw_common.h (Colin P. McCabe 2011-08-04 14:43:55 -0700 42) namespace ceph { -76880e3e src/rgw/rgw_common.h (Colin P. McCabe 2011-08-04 14:43:55 -0700 43) class Formatter; -76880e3e src/rgw/rgw_common.h (Colin P. McCabe 2011-08-04 14:43:55 -0700 44) } -76880e3e src/rgw/rgw_common.h (Colin P. McCabe 2011-08-04 14:43:55 -0700 45) -a772c8bb src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-16 10:57:48 -0700 46) using ceph::crypto::MD5; -a772c8bb src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-16 10:57:48 -0700 47) -37c88295 src/rgw/rgw_common.h (Sage Weil 2011-10-04 15:36:15 -0700 48) -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 49) #define RGW_ATTR_PREFIX "user.rgw." -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 50) -7ec64db4 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-13 21:59:37 -0700 51) #define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_" -7ec64db4 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-13 21:59:37 -0700 52) #define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-" -7ec64db4 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-13 21:59:37 -0700 53) -7b3ff369 src/rgw/rgw_common.h (Shivanshu Goswami 2016-02-22 21:16:11 +0000 54) #define RGW_AMZ_META_PREFIX "x-jcs-meta-" -d6d3bf06 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-03 17:36:50 -0700 55) -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 56) #define RGW_SYS_PARAM_PREFIX "rgwx-" -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 57) -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 58) #define RGW_ATTR_ACL RGW_ATTR_PREFIX "acl" -f165049c src/rgw/rgw_common.h (Babu Shanmugam 2013-03-05 09:22:55 +0530 59) #define RGW_ATTR_CORS RGW_ATTR_PREFIX "cors" -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 60) #define RGW_ATTR_ETAG RGW_ATTR_PREFIX "etag" -62e11112 src/rgw/rgw_common.h (Greg Farnum 2010-03-11 14:49:27 -0800 61) #define RGW_ATTR_BUCKETS RGW_ATTR_PREFIX "buckets" -d6d3bf06 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-03 17:36:50 -0700 62) #define RGW_ATTR_META_PREFIX RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 63) #define RGW_ATTR_CONTENT_TYPE RGW_ATTR_PREFIX "content_type" -f4a0b2d9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-08 16:13:04 -0700 64) #define RGW_ATTR_CACHE_CONTROL RGW_ATTR_PREFIX "cache_control" -f4a0b2d9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-08 16:13:04 -0700 65) #define RGW_ATTR_CONTENT_DISP RGW_ATTR_PREFIX "content_disposition" -f4a0b2d9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-08 16:13:04 -0700 66) #define RGW_ATTR_CONTENT_ENC RGW_ATTR_PREFIX "content_encoding" -f4a0b2d9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-08 16:13:04 -0700 67) #define RGW_ATTR_CONTENT_LANG RGW_ATTR_PREFIX "content_language" -f4a0b2d9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-08 16:13:04 -0700 68) #define RGW_ATTR_EXPIRES RGW_ATTR_PREFIX "expires" -8d63e140 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-02 16:29:16 -0700 69) #define RGW_ATTR_ID_TAG RGW_ATTR_PREFIX "idtag" -70cdd5da src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-03 16:59:33 -0700 70) #define RGW_ATTR_SHADOW_OBJ RGW_ATTR_PREFIX "shadow_name" -c076e351 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-01 14:03:57 -0800 71) #define RGW_ATTR_MANIFEST RGW_ATTR_PREFIX "manifest" -4d2a05f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-08-22 17:16:05 -0700 72) #define RGW_ATTR_USER_MANIFEST RGW_ATTR_PREFIX "user_manifest" -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 73) -57629b30 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-03 13:02:26 -0700 74) #define RGW_ATTR_OLH_PREFIX RGW_ATTR_PREFIX "olh." -57629b30 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-03 13:02:26 -0700 75) -57629b30 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-03 13:02:26 -0700 76) #define RGW_ATTR_OLH_INFO RGW_ATTR_OLH_PREFIX "info" -57629b30 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-03 13:02:26 -0700 77) #define RGW_ATTR_OLH_VER RGW_ATTR_OLH_PREFIX "ver" -01b8e614 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-03 17:04:46 -0800 78) #define RGW_ATTR_OLH_ID_TAG RGW_ATTR_OLH_PREFIX "idtag" -57629b30 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-03 13:02:26 -0700 79) #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending." -57629b30 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-03 13:02:26 -0700 80) -478fe5ee src/rgw/rgw_common.h (Yehuda Sadeh 2013-03-22 11:01:02 -0700 81) #define RGW_BUCKETS_OBJ_SUFFIX ".buckets" -fa8fa401 src/rgw/rgw_common.h (Yehuda Sadeh 2011-02-28 15:32:05 -0800 82) -8f1beb1b src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-20 17:09:27 -0700 83) #define RGW_MAX_PENDING_CHUNKS 16 -44cb0763 src/rgw/rgw_common.h (Yehuda Sadeh 2012-01-13 12:19:35 -0800 84) #define RGW_MAX_PUT_SIZE (5ULL*1024*1024*1024) -24523913 src/rgw/rgw_common.h (Caleb Miles 2012-12-04 16:36:17 -0500 85) #define RGW_MIN_MULTIPART_SIZE (5ULL*1024*1024) -925e2092 src/rgw/rgw_common.h (Yehuda Sadeh 2010-07-19 16:50:43 -0700 86) -42d873e9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-24 12:30:29 -0700 87) #define RGW_FORMAT_PLAIN 0 -fc63d973 src/rgw/rgw_common.h (Yehuda Sadeh 2011-02-15 17:30:07 -0800 88) #define RGW_FORMAT_XML 1 -fc63d973 src/rgw/rgw_common.h (Yehuda Sadeh 2011-02-15 17:30:07 -0800 89) #define RGW_FORMAT_JSON 2 -cc544de6 src/rgw/rgw_common.h (Robin H. Johnson 2015-08-24 02:42:56 +0000 90) #define RGW_FORMAT_HTML 3 -fc63d973 src/rgw/rgw_common.h (Yehuda Sadeh 2011-02-15 17:30:07 -0800 91) -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 92) #define RGW_CAP_READ 0x1 -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 93) #define RGW_CAP_WRITE 0x2 -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 94) #define RGW_CAP_ALL (RGW_CAP_READ | RGW_CAP_WRITE) -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 95) -e92e2971 src/rgw/rgw_common.h (Robin H. Johnson 2015-06-09 03:33:22 +0000 96) #define RGW_PROTO_SWIFT 0x1 -e92e2971 src/rgw/rgw_common.h (Robin H. Johnson 2015-06-09 03:33:22 +0000 97) #define RGW_PROTO_SWIFT_AUTH 0x2 -e92e2971 src/rgw/rgw_common.h (Robin H. Johnson 2015-06-09 03:33:22 +0000 98) #define RGW_PROTO_S3 0x4 -e92e2971 src/rgw/rgw_common.h (Robin H. Johnson 2015-06-09 03:33:22 +0000 99) #define RGW_PROTO_WEBSITE 0x8 -b738b72c src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-23 16:50:21 -0700 100) -4ca8054c src/rgw/rgw_common.h (Sage Weil 2011-06-17 09:26:32 -0700 101) #define RGW_SUSPENDED_USER_AUID (uint64_t)-2 -9974b7e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 16:53:38 -0700 102) -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 103) #define RGW_OP_TYPE_READ 0x01 -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 104) #define RGW_OP_TYPE_WRITE 0x02 -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 105) #define RGW_OP_TYPE_DELETE 0x04 -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 106) -c821da95 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 16:43:10 -0700 107) #define RGW_OP_TYPE_MODIFY (RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE) -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 108) #define RGW_OP_TYPE_ALL (RGW_OP_TYPE_READ | RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE) -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 109) -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 110) #define RGW_DEFAULT_MAX_BUCKETS 1000 -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 111) -1d7c2041 src/rgw/rgw_common.h (Liam Monahan 2013-10-01 17:10:05 -0400 112) #define RGW_DEFER_TO_BUCKET_ACLS_RECURSE 1 -1d7c2041 src/rgw/rgw_common.h (Liam Monahan 2013-10-01 17:10:05 -0400 113) #define RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL 2 -1d7c2041 src/rgw/rgw_common.h (Liam Monahan 2013-10-01 17:10:05 -0400 114) -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 115) #define STATUS_CREATED 1900 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 116) #define STATUS_ACCEPTED 1901 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 117) #define STATUS_NO_CONTENT 1902 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 118) #define STATUS_PARTIAL_CONTENT 1903 -3faf6ab5 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-11 09:40:39 -0700 119) #define STATUS_REDIRECT 1904 -4f9855e4 src/rgw/rgw_common.h (Greg Farnum 2013-07-16 12:23:13 -0700 120) #define STATUS_NO_APPLY 1905 -81b62b5c src/rgw/rgw_common.h (Greg Farnum 2013-07-25 16:03:54 -0700 121) #define STATUS_APPLIED 1906 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 122) -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 123) #define ERR_INVALID_BUCKET_NAME 2000 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 124) #define ERR_INVALID_OBJECT_NAME 2001 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 125) #define ERR_NO_SUCH_BUCKET 2002 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 126) #define ERR_METHOD_NOT_ALLOWED 2003 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 127) #define ERR_INVALID_DIGEST 2004 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 128) #define ERR_BAD_DIGEST 2005 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 129) #define ERR_UNRESOLVABLE_EMAIL 2006 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 130) #define ERR_INVALID_PART 2007 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 131) #define ERR_INVALID_PART_ORDER 2008 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 132) #define ERR_NO_SUCH_UPLOAD 2009 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 133) #define ERR_REQUEST_TIMEOUT 2010 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 134) #define ERR_LENGTH_REQUIRED 2011 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 135) #define ERR_REQUEST_TIME_SKEWED 2012 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 136) #define ERR_BUCKET_EXISTS 2013 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 137) #define ERR_BAD_URL 2014 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 138) #define ERR_PRECONDITION_FAILED 2015 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 139) #define ERR_NOT_MODIFIED 2016 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 140) #define ERR_INVALID_UTF8 2017 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 141) #define ERR_UNPROCESSABLE_ENTITY 2018 -44cb0763 src/rgw/rgw_common.h (Yehuda Sadeh 2012-01-13 12:19:35 -0800 142) #define ERR_TOO_LARGE 2019 -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 143) #define ERR_TOO_MANY_BUCKETS 2020 -b415fd21 src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-12 14:42:03 -0700 144) #define ERR_INVALID_REQUEST 2021 -30d11f42 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-11 15:36:07 -0700 145) #define ERR_TOO_SMALL 2022 -f165049c src/rgw/rgw_common.h (Babu Shanmugam 2013-03-05 09:22:55 +0530 146) #define ERR_NOT_FOUND 2023 -0f4c67f1 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-17 13:07:24 -0700 147) #define ERR_PERMANENT_REDIRECT 2024 -068baae7 src/rgw/rgw_common.h (Yehuda Sadeh 2013-08-09 11:52:25 -0700 148) #define ERR_LOCKED 2025 -bc98013f src/rgw/rgw_common.h (Yehuda Sadeh 2013-10-01 11:45:03 -0700 149) #define ERR_QUOTA_EXCEEDED 2026 -ef75d720 src/rgw/rgw_common.h (Yehuda Sadeh 2014-12-16 09:11:20 -0800 150) #define ERR_SIGNATURE_NO_MATCH 2027 -56af795b src/rgw/rgw_common.h (Yehuda Sadeh 2014-12-16 12:27:54 -0800 151) #define ERR_INVALID_ACCESS_KEY 2028 -8bcbfbe1 src/rgw/rgw_common.h (Shivanshu Goswami 2016-03-09 23:30:50 +0000 152) #define ERR_BUCKET_ALREADY_OWNED 2029 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 153) #define ERR_USER_SUSPENDED 2100 -0c78f0dc src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 17:20:51 -0700 154) #define ERR_INTERNAL_ERROR 2200 -9974b7e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 16:53:38 -0700 155) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 156) #ifndef UINT32_MAX -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 157) #define UINT32_MAX (4294967295) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 158) #endif -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 159) -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 160) typedef void *RGWAccessHandle; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 161) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 162) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 163) /* perf counter */ -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 164) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 165) extern PerfCounters *perfcounter; -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 166) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 167) extern int rgw_perf_start(CephContext *cct); -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 168) extern void rgw_perf_stop(CephContext *cct); -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 169) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 170) enum { -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 171) l_rgw_first = 15000, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 172) l_rgw_req, -cf566550 src/rgw/rgw_common.h (Greg Farnum 2011-11-08 09:49:22 -0800 173) l_rgw_failed_req, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 174) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 175) l_rgw_get, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 176) l_rgw_get_b, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 177) l_rgw_get_lat, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 178) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 179) l_rgw_put, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 180) l_rgw_put_b, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 181) l_rgw_put_lat, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 182) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 183) l_rgw_qlen, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 184) l_rgw_qactive, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 185) -cf566550 src/rgw/rgw_common.h (Greg Farnum 2011-11-08 09:49:22 -0800 186) l_rgw_cache_hit, -cf566550 src/rgw/rgw_common.h (Greg Farnum 2011-11-08 09:49:22 -0800 187) l_rgw_cache_miss, -cf566550 src/rgw/rgw_common.h (Greg Farnum 2011-11-08 09:49:22 -0800 188) -c62f3dd8 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-31 14:49:12 -0700 189) l_rgw_keystone_token_cache_hit, -c62f3dd8 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-31 14:49:12 -0700 190) l_rgw_keystone_token_cache_miss, -c62f3dd8 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-31 14:49:12 -0700 191) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 192) l_rgw_last, -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 193) }; -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 194) -0fe0f9db src/rgw/rgw_common.h (Sage Weil 2011-10-11 14:00:42 -0700 195) -232cd6b3 src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-27 14:22:55 -0700 196) /* size should be the required string size + 1 */ -b9097619 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-13 14:59:00 -0700 197) extern int gen_rand_base64(CephContext *cct, char *dest, int size); -b9097619 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-13 14:59:00 -0700 198) extern int gen_rand_alphanumeric(CephContext *cct, char *dest, int size); -f6bb8255 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-04 13:36:34 -0700 199) extern int gen_rand_alphanumeric_lower(CephContext *cct, char *dest, int size); -b9097619 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-13 14:59:00 -0700 200) extern int gen_rand_alphanumeric_upper(CephContext *cct, char *dest, int size); -caefe693 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-23 17:28:14 -0800 201) extern int gen_rand_alphanumeric_no_underscore(CephContext *cct, char *dest, int size); -6b365144 src/rgw/rgw_common.h (Yehuda Sadeh 2015-07-20 20:27:33 -0700 202) extern int gen_rand_alphanumeric_plain(CephContext *cct, char *dest, int size); -232cd6b3 src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-27 14:22:55 -0700 203) -d139f8dd src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-04 14:05:00 -0700 204) extern int gen_rand_alphanumeric_lower(CephContext *cct, string *str, int length); -d139f8dd src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-04 14:05:00 -0700 205) -7cc208bb src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-08 16:44:13 -0700 206) enum RGWIntentEvent { -1fe75ee6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-01-30 17:00:37 -0800 207) DEL_OBJ = 0, -1fe75ee6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-01-30 17:00:37 -0800 208) DEL_DIR = 1, -7cc208bb src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-08 16:44:13 -0700 209) }; -7cc208bb src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-08 16:44:13 -0700 210) -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 211) enum RGWObjCategory { -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 212) RGW_OBJ_CATEGORY_NONE = 0, -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 213) RGW_OBJ_CATEGORY_MAIN = 1, -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 214) RGW_OBJ_CATEGORY_SHADOW = 2, -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 215) RGW_OBJ_CATEGORY_MULTIMETA = 3, -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 216) }; -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 217) -62e11112 src/rgw/rgw_common.h (Greg Farnum 2010-03-11 14:49:27 -0800 218) /** Store error returns for output at a different point in the program */ -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 219) struct rgw_err { -b5f6eb12 src/rgw/rgw_common.h (Colin P. McCabe 2011-04-14 16:14:48 -0700 220) rgw_err(); -303420bf src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 10:52:14 -0700 221) rgw_err(int http, const std::string &s3); -b5f6eb12 src/rgw/rgw_common.h (Colin P. McCabe 2011-04-14 16:14:48 -0700 222) void clear(); -b5f6eb12 src/rgw/rgw_common.h (Colin P. McCabe 2011-04-14 16:14:48 -0700 223) bool is_clear() const; -9b7f223a src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 11:15:11 -0700 224) bool is_err() const; -b5f6eb12 src/rgw/rgw_common.h (Colin P. McCabe 2011-04-14 16:14:48 -0700 225) friend std::ostream& operator<<(std::ostream& oss, const rgw_err &err); -b5f6eb12 src/rgw/rgw_common.h (Colin P. McCabe 2011-04-14 16:14:48 -0700 226) -303420bf src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 10:52:14 -0700 227) int http_ret; -8836b844 src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-17 04:15:10 -0700 228) int ret; -303420bf src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 10:52:14 -0700 229) std::string s3_code; -b5f6eb12 src/rgw/rgw_common.h (Colin P. McCabe 2011-04-14 16:14:48 -0700 230) std::string message; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 231) }; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 232) -d57cdd0c src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-18 13:48:11 -0800 233) /* Helper class used for RGWHTTPArgs parsing */ -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 234) class NameVal -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 235) { -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 236) string str; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 237) string name; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 238) string val; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 239) public: -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 240) NameVal(string nv) : str(nv) {} -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 241) -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 242) int parse(); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 243) -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 244) string& get_name() { return name; } -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 245) string& get_val() { return val; } -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 246) }; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 247) -62e11112 src/rgw/rgw_common.h (Greg Farnum 2010-03-11 14:49:27 -0800 248) /** Stores the XML arguments associated with the HTTP request in req_state*/ -d57cdd0c src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-18 13:48:11 -0800 249) class RGWHTTPArgs -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 250) { -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 251) string str, empty_str; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 252) map val_map; -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 253) map sys_val_map; -a0d521b2 src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-27 13:35:47 -0700 254) map sub_resources; -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 255) -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 256) bool has_resp_modifier; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 257) public: -d57cdd0c src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-18 13:48:11 -0800 258) RGWHTTPArgs() : has_resp_modifier(false) {} -e92e2971 src/rgw/rgw_common.h (Robin H. Johnson 2015-06-09 03:33:22 +0000 259) -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 260) /** Set the arguments; as received */ -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 261) void set(string s) { -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 262) has_resp_modifier = false; -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 263) val_map.clear(); -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 264) sub_resources.clear(); -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 265) str = s; -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 266) } -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 267) /** parse the received arguments */ -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 268) int parse(); -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 269) /** Get the value for a specific argument parameter */ -ed04755a src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-24 14:34:23 -0700 270) string& get(const string& name, bool *exists = NULL); -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 271) string& get(const char *name, bool *exists = NULL); -9abec309 src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-23 12:31:31 -0700 272) int get_bool(const string& name, bool *val, bool *exists); -9abec309 src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-23 12:31:31 -0700 273) int get_bool(const char *name, bool *val, bool *exists); -e274e109 src/rgw/rgw_common.h (Yehuda Sadeh 2014-05-07 16:19:56 -0700 274) void get_bool(const char *name, bool *val, bool def_val); -9abec309 src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-23 12:31:31 -0700 275) -d57cdd0c src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-18 13:48:11 -0800 276) /** see if a parameter is contained in this RGWHTTPArgs */ -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 277) bool exists(const char *name) { -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 278) map::iterator iter = val_map.find(name); -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 279) return (iter != val_map.end()); -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 280) } -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 281) bool sub_resource_exists(const char *name) { -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 282) map::iterator iter = sub_resources.find(name); -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 283) return (iter != sub_resources.end()); -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 284) } -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 285) map& get_params() { -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 286) return val_map; -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 287) } -cb2d3660 src/rgw/rgw_common.h (Sage Weil 2011-09-29 23:17:12 -0700 288) map& get_sub_resources() { return sub_resources; } -648c3bc2 src/rgw/rgw_common.h (Babu Shanmugam 2013-04-26 18:44:16 +0530 289) unsigned get_num_params() { -648c3bc2 src/rgw/rgw_common.h (Babu Shanmugam 2013-04-26 18:44:16 +0530 290) return val_map.size(); -648c3bc2 src/rgw/rgw_common.h (Babu Shanmugam 2013-04-26 18:44:16 +0530 291) } -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 292) bool has_response_modifier() { -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 293) return has_resp_modifier; -97c1562d src/rgw/rgw_common.h (Yehuda Sadeh 2012-07-06 13:14:53 -0700 294) } -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 295) void set_system() { /* make all system params visible */ -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 296) map::iterator iter; -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 297) for (iter = sys_val_map.begin(); iter != sys_val_map.end(); ++iter) { -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 298) val_map[iter->first] = iter->second; -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 299) } -4dafea43 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-24 18:31:11 -0700 300) } -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 301) }; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 302) -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 303) class RGWConf; -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 304) -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 305) class RGWEnv { -fe6cd9bc src/rgw/rgw_common.h (Yehuda Sadeh 2013-10-22 13:53:59 -0700 306) std::map env_map; -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 307) public: -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 308) RGWConf *conf; -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 309) -cc40f115 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 14:26:16 -0700 310) RGWEnv(); -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 311) ~RGWEnv(); -fe6cd9bc src/rgw/rgw_common.h (Yehuda Sadeh 2013-10-22 13:53:59 -0700 312) void init(CephContext *cct); -b9097619 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-13 14:59:00 -0700 313) void init(CephContext *cct, char **envp); -fe6cd9bc src/rgw/rgw_common.h (Yehuda Sadeh 2013-10-22 13:53:59 -0700 314) void set(const char *name, const char *val); -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 315) const char *get(const char *name, const char *def_val = NULL); -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 316) int get_int(const char *name, int def_val = 0); -5d606c22 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-14 15:55:05 -0700 317) bool get_bool(const char *name, bool def_val = 0); -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 318) size_t get_size(const char *name, size_t def_val = 0); -eb0f49d4 src/rgw/rgw_common.h (Caleb Miles 2013-02-19 12:15:30 -0500 319) bool exists(const char *name); -eb0f49d4 src/rgw/rgw_common.h (Caleb Miles 2013-02-19 12:15:30 -0500 320) bool exists_prefix(const char *prefix); -0c805b6c src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 13:09:08 -0700 321) -0c805b6c src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 13:09:08 -0700 322) void remove(const char *name); -fe6cd9bc src/rgw/rgw_common.h (Yehuda Sadeh 2013-10-22 13:53:59 -0700 323) -fe6cd9bc src/rgw/rgw_common.h (Yehuda Sadeh 2013-10-22 13:53:59 -0700 324) std::map& get_map() { return env_map; } -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 325) }; -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 326) -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 327) class RGWConf { -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 328) friend class RGWEnv; -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 329) protected: -b9097619 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-13 14:59:00 -0700 330) void init(CephContext *cct, RGWEnv * env); -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 331) public: -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 332) RGWConf() : -1d7c2041 src/rgw/rgw_common.h (Liam Monahan 2013-10-01 17:10:05 -0400 333) enable_ops_log(1), enable_usage_log(1), defer_to_bucket_acls(0) {} -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 334) -ec689e3e src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-05 11:57:35 -0700 335) int enable_ops_log; -744a1b31 src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 11:01:56 -0700 336) int enable_usage_log; -1d7c2041 src/rgw/rgw_common.h (Liam Monahan 2013-10-01 17:10:05 -0400 337) uint8_t defer_to_bucket_acls; -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 338) }; -059019c9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-11 13:41:40 -0700 339) -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 340) enum http_op { -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 341) OP_GET, -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 342) OP_PUT, -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 343) OP_DELETE, -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 344) OP_HEAD, -9e8484e8 src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-26 15:18:48 -0700 345) OP_POST, -87941128 src/rgw/rgw_common.h (Yehuda Sadeh 2011-11-10 14:56:10 -0800 346) OP_COPY, -f165049c src/rgw/rgw_common.h (Babu Shanmugam 2013-03-05 09:22:55 +0530 347) OP_OPTIONS, -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 348) OP_UNKNOWN, -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 349) }; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 350) -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 351) class RGWAccessControlPolicy; -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 352) class JSONObj; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 353) -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 354) struct RGWAccessKey { -6dcf462c src/rgw/rgw_common.h (Robin H. Johnson 2014-01-18 17:49:06 -0800 355) string id; // AccessKey -6dcf462c src/rgw/rgw_common.h (Robin H. Johnson 2014-01-18 17:49:06 -0800 356) string key; // SecretKey -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 357) string subuser; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 358) -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 359) RGWAccessKey() {} -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 360) void encode(bufferlist& bl) const { -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 361) ENCODE_START(2, 2, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 362) ::encode(id, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 363) ::encode(key, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 364) ::encode(subuser, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 365) ENCODE_FINISH(bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 366) } -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 367) -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 368) void decode(bufferlist::iterator& bl) { -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 369) DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 370) ::decode(id, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 371) ::decode(key, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 372) ::decode(subuser, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 373) DECODE_FINISH(bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 374) } -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 375) void dump(Formatter *f) const; -a7096f8f src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-20 12:13:31 -0700 376) void dump_plain(Formatter *f) const; -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 377) void dump(Formatter *f, const string& user, bool swift) const; -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 378) static void generate_test_instances(list& o); -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 379) -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 380) void decode_json(JSONObj *obj); -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 381) void decode_json(JSONObj *obj, bool swift); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 382) }; -60b1071d src/rgw/rgw_common.h (Daniel J. Hofmann 2014-05-09 15:07:15 +0200 383) WRITE_CLASS_ENCODER(RGWAccessKey) -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 384) -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 385) struct RGWSubUser { -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 386) string name; -c167a28d src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-23 15:12:48 -0700 387) uint32_t perm_mask; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 388) -c167a28d src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-23 15:12:48 -0700 389) RGWSubUser() : perm_mask(0) {} -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 390) void encode(bufferlist& bl) const { -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 391) ENCODE_START(2, 2, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 392) ::encode(name, bl); -c167a28d src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-23 15:12:48 -0700 393) ::encode(perm_mask, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 394) ENCODE_FINISH(bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 395) } -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 396) -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 397) void decode(bufferlist::iterator& bl) { -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 398) DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 399) ::decode(name, bl); -c167a28d src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-23 15:12:48 -0700 400) ::decode(perm_mask, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 401) DECODE_FINISH(bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 402) } -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 403) void dump(Formatter *f) const; -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 404) void dump(Formatter *f, const string& user) const; -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 405) static void generate_test_instances(list& o); -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 406) -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 407) void decode_json(JSONObj *obj); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 408) }; -60b1071d src/rgw/rgw_common.h (Daniel J. Hofmann 2014-05-09 15:07:15 +0200 409) WRITE_CLASS_ENCODER(RGWSubUser) -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 410) -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 411) class RGWUserCaps -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 412) { -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 413) map caps; -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 414) -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 415) int get_cap(const string& cap, string& type, uint32_t *perm); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 416) int add_cap(const string& cap); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 417) int remove_cap(const string& cap); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 418) public: -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 419) static int parse_cap_perm(const string& str, uint32_t *perm); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 420) int add_from_string(const string& str); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 421) int remove_from_string(const string& str); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 422) -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 423) void encode(bufferlist& bl) const { -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 424) ENCODE_START(1, 1, bl); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 425) ::encode(caps, bl); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 426) ENCODE_FINISH(bl); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 427) } -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 428) void decode(bufferlist::iterator& bl) { -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 429) DECODE_START(1, bl); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 430) ::decode(caps, bl); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 431) DECODE_FINISH(bl); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 432) } -511e639e src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 11:57:04 -0700 433) int check_cap(const string& cap, uint32_t perm); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 434) void dump(Formatter *f) const; -b07f3cda src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-12 09:40:12 -0800 435) void dump(Formatter *f, const char *name) const; -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 436) -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 437) void decode_json(JSONObj *obj); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 438) }; -60b1071d src/rgw/rgw_common.h (Daniel J. Hofmann 2014-05-09 15:07:15 +0200 439) WRITE_CLASS_ENCODER(RGWUserCaps) -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 440) -3e5cead0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-29 08:11:59 -0700 441) void encode_json(const char *name, const obj_version& v, Formatter *f); -b07f3cda src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-12 09:40:12 -0800 442) void encode_json(const char *name, const RGWUserCaps& val, Formatter *f); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 443) -3e5cead0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-29 08:11:59 -0700 444) void decode_json_obj(obj_version& v, JSONObj *obj); -3e5cead0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-29 08:11:59 -0700 445) -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 446) struct RGWUserInfo -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 447) { -edc92490 src/rgw/rgw_common.h (Sage Weil 2010-05-07 14:33:42 -0700 448) uint64_t auid; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 449) string user_id; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 450) string display_name; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 451) string user_email; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 452) map access_keys; -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 453) map swift_keys; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 454) map subusers; -9974b7e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 16:53:38 -0700 455) __u8 suspended; -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 456) uint32_t max_buckets; -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 457) uint32_t op_mask; -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 458) RGWUserCaps caps; -903d4a04 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-22 11:08:33 -0700 459) __u8 system; -2fcbf2ba src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 16:16:47 -0700 460) string default_placement; -2fcbf2ba src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 16:16:47 -0700 461) list placement_tags; -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 462) RGWQuotaInfo bucket_quota; -7ccb513c src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-09 10:35:53 -0800 463) map temp_url_keys; -15c01895 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-10 15:11:08 -0800 464) RGWQuotaInfo user_quota; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 465) -d30fc4bd src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-03 17:09:56 -0700 466) RGWUserInfo() : auid(0), suspended(0), max_buckets(RGW_DEFAULT_MAX_BUCKETS), op_mask(RGW_OP_TYPE_ALL), system(0) {} -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 467) -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 468) void encode(bufferlist& bl) const { -cacdfd91 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-27 11:27:56 -0800 469) ENCODE_START(16, 9, bl); -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 470) ::encode(auid, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 471) string access_key; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 472) string secret_key; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 473) if (!access_keys.empty()) { -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 474) map::const_iterator iter = access_keys.begin(); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 475) const RGWAccessKey& k = iter->second; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 476) access_key = k.id; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 477) secret_key = k.key; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 478) } -544ce94a src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 17:20:44 -0700 479) ::encode(access_key, bl); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 480) ::encode(secret_key, bl); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 481) ::encode(display_name, bl); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 482) ::encode(user_email, bl); -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 483) string swift_name; -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 484) string swift_key; -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 485) if (!swift_keys.empty()) { -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 486) map::const_iterator iter = swift_keys.begin(); -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 487) const RGWAccessKey& k = iter->second; -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 488) swift_name = k.id; -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 489) swift_key = k.key; -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 490) } -ba7ab2f6 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-02 14:56:50 -0700 491) ::encode(swift_name, bl); -ba7ab2f6 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-02 14:56:50 -0700 492) ::encode(swift_key, bl); -544ce94a src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 17:20:44 -0700 493) ::encode(user_id, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 494) ::encode(access_keys, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 495) ::encode(subusers, bl); -9974b7e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 16:53:38 -0700 496) ::encode(suspended, bl); -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 497) ::encode(swift_keys, bl); -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 498) ::encode(max_buckets, bl); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 499) ::encode(caps, bl); -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 500) ::encode(op_mask, bl); -903d4a04 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-22 11:08:33 -0700 501) ::encode(system, bl); -2fcbf2ba src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 16:16:47 -0700 502) ::encode(default_placement, bl); -2fcbf2ba src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 16:16:47 -0700 503) ::encode(placement_tags, bl); -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 504) ::encode(bucket_quota, bl); -7ccb513c src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-09 10:35:53 -0800 505) ::encode(temp_url_keys, bl); -15c01895 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-10 15:11:08 -0800 506) ::encode(user_quota, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 507) ENCODE_FINISH(bl); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 508) } -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 509) void decode(bufferlist::iterator& bl) { -cacdfd91 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-27 11:27:56 -0800 510) DECODE_START_LEGACY_COMPAT_LEN_32(16, 9, 9, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 511) if (struct_v >= 2) ::decode(auid, bl); -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 512) else auid = CEPH_AUTH_UID_DEFAULT; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 513) string access_key; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 514) string secret_key; -544ce94a src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 17:20:44 -0700 515) ::decode(access_key, bl); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 516) ::decode(secret_key, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 517) if (struct_v < 6) { -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 518) RGWAccessKey k; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 519) k.id = access_key; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 520) k.key = secret_key; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 521) access_keys[access_key] = k; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 522) } -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 523) ::decode(display_name, bl); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 524) ::decode(user_email, bl); -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 525) string swift_name; -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 526) string swift_key; -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 527) if (struct_v >= 3) ::decode(swift_name, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 528) if (struct_v >= 4) ::decode(swift_key, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 529) if (struct_v >= 5) -544ce94a src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 17:20:44 -0700 530) ::decode(user_id, bl); -544ce94a src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 17:20:44 -0700 531) else -544ce94a src/rgw/rgw_common.h (Yehuda Sadeh 2011-04-15 17:20:44 -0700 532) user_id = access_key; -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 533) if (struct_v >= 6) { -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 534) ::decode(access_keys, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 535) ::decode(subusers, bl); -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 536) } -9974b7e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 16:53:38 -0700 537) suspended = 0; -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 538) if (struct_v >= 7) { -9974b7e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 16:53:38 -0700 539) ::decode(suspended, bl); -9974b7e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 16:53:38 -0700 540) } -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 541) if (struct_v >= 8) { -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 542) ::decode(swift_keys, bl); -f883e638 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-11 10:49:27 -0700 543) } -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 544) if (struct_v >= 10) { -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 545) ::decode(max_buckets, bl); -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 546) } else { -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 547) max_buckets = RGW_DEFAULT_MAX_BUCKETS; -5db4509b src/rgw/rgw_common.h (Yehuda Sadeh 2012-06-11 23:31:09 -0700 548) } -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 549) if (struct_v >= 11) { -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 550) ::decode(caps, bl); -d22aa6c9 src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-25 10:55:19 -0700 551) } -903d4a04 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-22 11:08:33 -0700 552) if (struct_v >= 12) { -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 553) ::decode(op_mask, bl); -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 554) } else { -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 555) op_mask = RGW_OP_TYPE_ALL; -903d4a04 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-22 11:08:33 -0700 556) } -d30fc4bd src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-03 17:09:56 -0700 557) system = 0; -2fcbf2ba src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 16:16:47 -0700 558) if (struct_v >= 13) { -d30fc4bd src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-03 17:09:56 -0700 559) ::decode(system, bl); -2fcbf2ba src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 16:16:47 -0700 560) ::decode(default_placement, bl); -2fcbf2ba src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 16:16:47 -0700 561) ::decode(placement_tags, bl); /* tags of allowed placement rules */ -2fcbf2ba src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 16:16:47 -0700 562) } -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 563) if (struct_v >= 14) { -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 564) ::decode(bucket_quota, bl); -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 565) } -2626101f src/rgw/rgw_common.h (Yehuda Sadeh 2013-11-25 13:41:50 -0800 566) if (struct_v >= 15) { -7ccb513c src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-09 10:35:53 -0800 567) ::decode(temp_url_keys, bl); -2626101f src/rgw/rgw_common.h (Yehuda Sadeh 2013-11-25 13:41:50 -0800 568) } -cacdfd91 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-27 11:27:56 -0800 569) if (struct_v >= 16) { -15c01895 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-10 15:11:08 -0800 570) ::decode(user_quota, bl); -15c01895 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-10 15:11:08 -0800 571) } -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 572) DECODE_FINISH(bl); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 573) } -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 574) void dump(Formatter *f) const; -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 575) static void generate_test_instances(list& o); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 576) -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 577) void decode_json(JSONObj *obj); -da337013 src/rgw/rgw_common.h (Yehuda Sadeh 2013-02-06 12:48:01 -0800 578) -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 579) void clear() { -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 580) user_id.clear(); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 581) display_name.clear(); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 582) user_email.clear(); -83a6efef src/rgw/rgw_common.h (Greg Farnum 2010-03-26 16:29:11 -0700 583) auid = CEPH_AUTH_UID_DEFAULT; -2cf5048f src/rgw/rgw_common.h (Yehuda Sadeh 2011-05-20 15:15:48 -0700 584) access_keys.clear(); -9974b7e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 16:53:38 -0700 585) suspended = 0; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 586) } -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 587) }; -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 588) WRITE_CLASS_ENCODER(RGWUserInfo) -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 589) -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 590) struct rgw_bucket { -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 591) std::string name; -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 592) std::string data_pool; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 593) std::string data_extra_pool; /* if not set, then we should use data_pool instead */ -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 594) std::string index_pool; -25499b64 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-19 16:41:44 -0700 595) std::string marker; -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 596) std::string bucket_id; -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 597) -68730d80 src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-26 16:40:34 -0700 598) std::string oid; /* -68730d80 src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-26 16:40:34 -0700 599) * runtime in-memory only info. If not empty, points to the bucket instance object -68730d80 src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-26 16:40:34 -0700 600) */ -68730d80 src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-26 16:40:34 -0700 601) -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 602) rgw_bucket() { } -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 603) rgw_bucket(const cls_user_bucket& b) { -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 604) name = b.name; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 605) data_pool = b.data_pool; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 606) data_extra_pool = b.data_extra_pool; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 607) index_pool = b.index_pool; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 608) marker = b.marker; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 609) bucket_id = b.bucket_id; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 610) } -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 611) rgw_bucket(const char *n) : name(n) { -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 612) assert(*n == '.'); // only rgw private buckets should be initialized without pool -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 613) data_pool = index_pool = n; -25499b64 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-19 16:41:44 -0700 614) marker = ""; -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 615) } -0f4c67f1 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-17 13:07:24 -0700 616) rgw_bucket(const char *n, const char *dp, const char *ip, const char *m, const char *id, const char *h) : -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 617) name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {} -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 618) -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 619) void convert(cls_user_bucket *b) { -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 620) b->name = name; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 621) b->data_pool = data_pool; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 622) b->data_extra_pool = data_extra_pool; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 623) b->index_pool = index_pool; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 624) b->marker = marker; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 625) b->bucket_id = bucket_id; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 626) } -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 627) -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 628) void encode(bufferlist& bl) const { -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 629) ENCODE_START(7, 3, bl); -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 630) ::encode(name, bl); -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 631) ::encode(data_pool, bl); -25499b64 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-19 16:41:44 -0700 632) ::encode(marker, bl); -c2fbde4d src/rgw/rgw_common.h (Greg Farnum 2011-09-28 10:50:48 -0700 633) ::encode(bucket_id, bl); -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 634) ::encode(index_pool, bl); -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 635) ::encode(data_extra_pool, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 636) ENCODE_FINISH(bl); -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 637) } -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 638) void decode(bufferlist::iterator& bl) { -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 639) DECODE_START_LEGACY_COMPAT_LEN(7, 3, 3, bl); -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 640) ::decode(name, bl); -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 641) ::decode(data_pool, bl); -c2fbde4d src/rgw/rgw_common.h (Greg Farnum 2011-09-28 10:50:48 -0700 642) if (struct_v >= 2) { -25499b64 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-19 16:41:44 -0700 643) ::decode(marker, bl); -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 644) if (struct_v <= 3) { -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 645) uint64_t id; -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 646) ::decode(id, bl); -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 647) char buf[16]; -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 648) snprintf(buf, sizeof(buf), "%llu", (long long)id); -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 649) bucket_id = buf; -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 650) } else { -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 651) ::decode(bucket_id, bl); -6a5cbec3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-20 15:15:09 -0700 652) } -c2fbde4d src/rgw/rgw_common.h (Greg Farnum 2011-09-28 10:50:48 -0700 653) } -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 654) if (struct_v >= 5) { -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 655) ::decode(index_pool, bl); -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 656) } else { -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 657) index_pool = data_pool; -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 658) } -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 659) if (struct_v >= 7) { -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 660) ::decode(data_extra_pool, bl); -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 661) } -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 662) DECODE_FINISH(bl); -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 663) } -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 664) -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 665) const string& get_data_extra_pool() { -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 666) if (data_extra_pool.empty()) { -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 667) return data_pool; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 668) } -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 669) return data_extra_pool; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 670) } -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 671) -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 672) void dump(Formatter *f) const; -770d94d3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-08 20:40:48 -0700 673) void decode_json(JSONObj *obj); -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 674) static void generate_test_instances(list& o); -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 675) -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 676) bool operator<(const rgw_bucket& b) const { -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 677) return name.compare(b.name) < 0; -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 678) } -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 679) }; -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 680) WRITE_CLASS_ENCODER(rgw_bucket) -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 681) -f106e1a5 src/rgw/rgw_common.h (Danny Al-Gaaf 2013-03-11 15:35:53 +0100 682) inline ostream& operator<<(ostream& out, const rgw_bucket &b) { -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 683) out << b.name; -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 684) if (b.name.compare(b.data_pool)) { -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 685) out << "(@"; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 686) string s; -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 687) if (!b.index_pool.empty() && b.data_pool.compare(b.index_pool)) -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 688) s = "i=" + b.index_pool; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 689) if (!b.data_extra_pool.empty() && b.data_pool.compare(b.data_extra_pool)) { -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 690) if (!s.empty()) { -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 691) s += ","; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 692) } -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 693) s += "e=" + b.data_extra_pool; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 694) } -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 695) if (!s.empty()) { -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 696) out << "{" << s << "}"; -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 697) } -3677076b src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 12:01:31 -0700 698) -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 699) out << b.data_pool << "[" << b.marker << "])"; -988dab3e src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-18 14:31:50 -0700 700) } -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 701) return out; -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 702) } -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 703) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 704) struct rgw_bucket_shard { -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 705) rgw_bucket bucket; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 706) int shard_id; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 707) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 708) rgw_bucket_shard() : shard_id(-1) {} -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 709) rgw_bucket_shard(rgw_bucket& _b, int _sid) : bucket(_b), shard_id(_sid) {} -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 710) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 711) bool operator<(const rgw_bucket_shard& b) const { -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 712) if (bucket < b.bucket) { -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 713) return true; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 714) } -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 715) if (b.bucket < bucket) { -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 716) return false; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 717) } -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 718) return shard_id < b.shard_id; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 719) } -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 720) }; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 721) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 722) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 723) struct RGWObjVersionTracker { -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 724) obj_version read_version; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 725) obj_version write_version; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 726) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 727) obj_version *version_for_read() { -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 728) return &read_version; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 729) } -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 730) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 731) obj_version *version_for_write() { -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 732) if (write_version.ver == 0) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 733) return NULL; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 734) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 735) return &write_version; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 736) } -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 737) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 738) obj_version *version_for_check() { -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 739) if (read_version.ver == 0) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 740) return NULL; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 741) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 742) return &read_version; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 743) } -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 744) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 745) void prepare_op_for_read(librados::ObjectReadOperation *op); -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 746) void prepare_op_for_write(librados::ObjectWriteOperation *op); -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 747) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 748) void apply_write() { -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 749) read_version = write_version; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 750) write_version = obj_version(); -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 751) } -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 752) -859ed33e src/rgw/rgw_common.h (Yehuda Sadeh 2014-02-19 08:11:56 -0800 753) void clear() { -859ed33e src/rgw/rgw_common.h (Yehuda Sadeh 2014-02-19 08:11:56 -0800 754) read_version = obj_version(); -859ed33e src/rgw/rgw_common.h (Yehuda Sadeh 2014-02-19 08:11:56 -0800 755) write_version = obj_version(); -859ed33e src/rgw/rgw_common.h (Yehuda Sadeh 2014-02-19 08:11:56 -0800 756) } -859ed33e src/rgw/rgw_common.h (Yehuda Sadeh 2014-02-19 08:11:56 -0800 757) -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 758) void generate_new_write_ver(CephContext *cct); -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 759) }; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 760) -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 761) enum RGWBucketFlags { -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 762) BUCKET_SUSPENDED = 0x1, -8d25ec63 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-29 15:10:02 -0700 763) BUCKET_VERSIONED = 0x2, -8d25ec63 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-29 15:10:02 -0700 764) BUCKET_VERSIONS_SUSPENDED = 0x4, -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 765) }; -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 766) -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 767) struct RGWBucketInfo -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 768) { -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 769) enum BIShardsHashType { -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 770) MOD = 0 -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 771) }; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 772) -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 773) rgw_bucket bucket; -0eb433d4 src/rgw/rgw_common.h (Greg Farnum 2011-09-28 13:51:20 -0700 774) string owner; -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 775) uint32_t flags; -c8ac2879 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-14 18:26:37 -0700 776) string region; -a2cf14fe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-15 18:41:21 -0700 777) time_t creation_time; -d7af5e14 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 20:54:28 -0700 778) string placement_rule; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 779) bool has_instance_obj; -dab57ef8 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 18:52:39 -0700 780) RGWObjVersionTracker objv_tracker; /* we don't need to serialize this, for runtime tracking */ -7cd0bd85 src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-18 17:40:52 -0700 781) obj_version ep_objv; /* entry point object version, for runtime tracking only */ -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 782) RGWQuotaInfo quota; -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 783) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 784) // Represents the number of bucket index object shards: -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 785) // - value of 0 indicates there is no sharding (this is by default before this -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 786) // feature is implemented). -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 787) // - value of UINT32_T::MAX indicates this is a blind bucket. -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 788) uint32_t num_shards; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 789) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 790) // Represents the bucket index shard hash type. -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 791) uint8_t bucket_index_shard_hash_type; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 792) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 793) // Represents the shard number for blind bucket. -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 794) const static uint32_t NUM_SHARDS_BLIND_BUCKET; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 795) -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 796) bool has_website; -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 797) RGWBucketWebsiteConf website_conf; -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 798) -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 799) void encode(bufferlist& bl) const { -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 800) ENCODE_START(12, 4, bl); -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 801) ::encode(bucket, bl); -560f90c5 src/rgw/rgw_common.h (Sage Weil 2011-09-29 16:32:14 -0700 802) ::encode(owner, bl); -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 803) ::encode(flags, bl); -c8ac2879 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-14 18:26:37 -0700 804) ::encode(region, bl); -cfc1f2ee src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-01 11:01:39 -0700 805) uint64_t ct = (uint64_t)creation_time; -cfc1f2ee src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-01 11:01:39 -0700 806) ::encode(ct, bl); -d7af5e14 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 20:54:28 -0700 807) ::encode(placement_rule, bl); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 808) ::encode(has_instance_obj, bl); -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 809) ::encode(quota, bl); -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 810) ::encode(num_shards, bl); -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 811) ::encode(bucket_index_shard_hash_type, bl); -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 812) ::encode(has_website, bl); -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 813) if (has_website) { -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 814) ::encode(website_conf, bl); -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 815) } -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 816) ENCODE_FINISH(bl); -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 817) } -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 818) void decode(bufferlist::iterator& bl) { -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 819) DECODE_START_LEGACY_COMPAT_LEN_32(12, 4, 4, bl); -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 820) ::decode(bucket, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 821) if (struct_v >= 2) -560f90c5 src/rgw/rgw_common.h (Sage Weil 2011-09-29 16:32:14 -0700 822) ::decode(owner, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 823) if (struct_v >= 3) -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 824) ::decode(flags, bl); -c8ac2879 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-14 18:26:37 -0700 825) if (struct_v >= 5) -c8ac2879 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-14 18:26:37 -0700 826) ::decode(region, bl); -cfc1f2ee src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-01 11:01:39 -0700 827) if (struct_v >= 6) { -cfc1f2ee src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-01 11:01:39 -0700 828) uint64_t ct; -cfc1f2ee src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-01 11:01:39 -0700 829) ::decode(ct, bl); -cfc1f2ee src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-01 11:01:39 -0700 830) creation_time = (time_t)ct; -cfc1f2ee src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-01 11:01:39 -0700 831) } -d7af5e14 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 20:54:28 -0700 832) if (struct_v >= 7) -d7af5e14 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-21 20:54:28 -0700 833) ::decode(placement_rule, bl); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 834) if (struct_v >= 8) -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 835) ::decode(has_instance_obj, bl); -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 836) if (struct_v >= 9) -434ad764 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-27 14:51:43 -0700 837) ::decode(quota, bl); -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 838) if (struct_v >= 10) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 839) ::decode(num_shards, bl); -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 840) if (struct_v >= 11) -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 841) ::decode(bucket_index_shard_hash_type, bl); -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 842) if (struct_v >= 12) { -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 843) ::decode(has_website, bl); -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 844) if (has_website) { -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 845) ::decode(website_conf, bl); -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 846) } else { -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 847) website_conf = RGWBucketWebsiteConf(); -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 848) } -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 849) } -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 850) DECODE_FINISH(bl); -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 851) } -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 852) void dump(Formatter *f) const; -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 853) static void generate_test_instances(list& o); -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 854) -770d94d3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-08 20:40:48 -0700 855) void decode_json(JSONObj *obj); -770d94d3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-08 20:40:48 -0700 856) -8d25ec63 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-29 15:10:02 -0700 857) bool versioned() { return (flags & BUCKET_VERSIONED) != 0; } -7f139286 src/rgw/rgw_common.h (Yehuda Sadeh 2014-10-27 17:09:57 -0700 858) int versioning_status() { return flags & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED); } -7f139286 src/rgw/rgw_common.h (Yehuda Sadeh 2014-10-27 17:09:57 -0700 859) bool versioning_enabled() { return versioning_status() == BUCKET_VERSIONED; } -8d25ec63 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-29 15:10:02 -0700 860) -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 861) RGWBucketInfo() : flags(0), creation_time(0), has_instance_obj(false), num_shards(0), bucket_index_shard_hash_type(MOD), -c0bacef4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-26 16:34:04 -0800 862) has_website(false) {} -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 863) }; -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 864) WRITE_CLASS_ENCODER(RGWBucketInfo) -3ebe6b77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-18 13:21:31 -0700 865) -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 866) struct RGWBucketEntryPoint -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 867) { -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 868) rgw_bucket bucket; -7e41c103 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 17:27:34 -0700 869) string owner; -63e81afe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 23:43:50 -0700 870) time_t creation_time; -6673b2d3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 14:00:59 -0700 871) bool linked; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 872) -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 873) bool has_bucket_info; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 874) RGWBucketInfo old_bucket_info; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 875) -6673b2d3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 14:00:59 -0700 876) RGWBucketEntryPoint() : creation_time(0), linked(false), has_bucket_info(false) {} -ad640672 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 13:12:59 -0700 877) -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 878) void encode(bufferlist& bl) const { -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 879) ENCODE_START(8, 8, bl); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 880) ::encode(bucket, bl); -7e41c103 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 17:27:34 -0700 881) ::encode(owner, bl); -6673b2d3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 14:00:59 -0700 882) ::encode(linked, bl); -63e81afe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 23:43:50 -0700 883) uint64_t ctime = (uint64_t)creation_time; -63e81afe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 23:43:50 -0700 884) ::encode(ctime, bl); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 885) ENCODE_FINISH(bl); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 886) } -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 887) void decode(bufferlist::iterator& bl) { -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 888) bufferlist::iterator orig_iter = bl; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 889) DECODE_START_LEGACY_COMPAT_LEN_32(8, 4, 4, bl); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 890) if (struct_v < 8) { -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 891) /* ouch, old entry, contains the bucket info itself */ -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 892) old_bucket_info.decode(orig_iter); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 893) has_bucket_info = true; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 894) return; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 895) } -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 896) has_bucket_info = false; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 897) ::decode(bucket, bl); -7e41c103 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 17:27:34 -0700 898) ::decode(owner, bl); -6673b2d3 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 14:00:59 -0700 899) ::decode(linked, bl); -63e81afe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 23:43:50 -0700 900) uint64_t ctime; -63e81afe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 23:43:50 -0700 901) ::decode(ctime, bl); -63e81afe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-24 23:43:50 -0700 902) creation_time = (uint64_t)ctime; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 903) DECODE_FINISH(bl); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 904) } -71869c4b src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 21:00:00 -0700 905) -71869c4b src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 21:00:00 -0700 906) void dump(Formatter *f) const; -71869c4b src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-23 21:00:00 -0700 907) void decode_json(JSONObj *obj); -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 908) }; -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 909) WRITE_CLASS_ENCODER(RGWBucketEntryPoint) -c3260b27 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-22 22:47:48 -0700 910) -4aee3fa2 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-09 21:37:34 -0800 911) struct RGWStorageStats -0b1ad608 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-24 16:41:10 -0700 912) { -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 913) RGWObjCategory category; -0b1ad608 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-24 16:41:10 -0700 914) uint64_t num_kb; -7a32cc60 src/rgw/rgw_common.h (Yehuda Sadeh 2011-11-08 13:42:55 -0800 915) uint64_t num_kb_rounded; -0b1ad608 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-24 16:41:10 -0700 916) uint64_t num_objects; -80659cce src/rgw/rgw_common.h (Yehuda Sadeh 2013-10-02 16:34:40 -0700 917) -c13634e8 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-10 13:44:56 -0800 918) RGWStorageStats() : category(RGW_OBJ_CATEGORY_NONE), num_kb(0), num_kb_rounded(0), num_objects(0) {} -7e13ac8e src/rgw/rgw_common.h (Ray Lv 2014-09-10 15:33:22 +0800 919) -7e13ac8e src/rgw/rgw_common.h (Ray Lv 2014-09-10 15:33:22 +0800 920) void dump(Formatter *f) const; -0b1ad608 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-24 16:41:10 -0700 921) }; -0b1ad608 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-24 16:41:10 -0700 922) -fc63d973 src/rgw/rgw_common.h (Yehuda Sadeh 2011-02-15 17:30:07 -0800 923) struct req_state; -fc63d973 src/rgw/rgw_common.h (Yehuda Sadeh 2011-02-15 17:30:07 -0800 924) -e1666d04 src/rgw/rgw_common.h (Christophe Courtaut 2013-08-09 11:58:58 +0200 925) class RGWEnv; -a9c9f96b src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-19 12:36:44 -0700 926) -a9c9f96b src/rgw/rgw_common.h (Yehuda Sadeh 2012-09-19 12:36:44 -0700 927) class RGWClientIO; -43575c7a src/rgw/rgw_common.h (Yehuda Sadeh 2011-07-19 14:02:11 -0700 928) -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 929) struct req_info { -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 930) RGWEnv *env; -d57cdd0c src/rgw/rgw_common.h (Yehuda Sadeh 2014-11-18 13:48:11 -0800 931) RGWHTTPArgs args; -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 932) map x_meta_map; -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 933) -a3afb3f5 src/rgw/rgw_common.h (Sage Weil 2015-06-09 14:15:10 -0400 934) string host; -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 935) const char *method; -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 936) string script_uri; -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 937) string request_uri; -8a2eb184 src/rgw/rgw_common.h (Yehuda Sadeh 2013-07-22 13:33:33 -0700 938) string effective_uri; -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 939) string request_params; -90c5869b src/rgw/rgw_common.h (Yehuda Sadeh 2014-02-19 18:00:10 -0800 940) string domain; -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 941) -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 942) req_info(CephContext *cct, RGWEnv *_env); -580a08c6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 23:14:35 -0700 943) void rebuild_from(req_info& src); -f67bfa24 src/rgw/rgw_common.h (Dmytro Iurchenko 2015-02-03 17:54:38 +0200 944) void init_meta_info(bool *found_bad_meta); -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 945) }; -c812bb51 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-23 11:39:13 -0700 946) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 947) struct rgw_obj_key { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 948) string name; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 949) string instance; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 950) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 951) rgw_obj_key() {} -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 952) rgw_obj_key(const string& n) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 953) set(n); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 954) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 955) rgw_obj_key(const string& n, const string& i) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 956) set(n, i); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 957) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 958) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 959) void set(const cls_rgw_obj_key& k) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 960) name = k.name; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 961) instance = k.instance; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 962) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 963) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 964) void transform(cls_rgw_obj_key *k) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 965) k->name = name; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 966) k->instance = instance; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 967) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 968) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 969) void set(const string& n) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 970) name = n; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 971) instance.clear(); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 972) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 973) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 974) void set(const string& n, const string& i) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 975) name = n; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 976) instance = i; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 977) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 978) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 979) bool empty() { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 980) return name.empty(); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 981) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 982) bool operator==(const rgw_obj_key& k) const { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 983) return (name.compare(k.name) == 0) && -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 984) (instance.compare(k.instance) == 0); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 985) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 986) bool operator<(const rgw_obj_key& k) const { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 987) int r = name.compare(k.name); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 988) if (r == 0) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 989) r = instance.compare(k.instance); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 990) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 991) return (r < 0); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 992) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 993) void encode(bufferlist& bl) const { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 994) ENCODE_START(1, 1, bl); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 995) ::encode(name, bl); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 996) ::encode(instance, bl); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 997) ENCODE_FINISH(bl); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 998) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 999) void decode(bufferlist::iterator& bl) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1000) DECODE_START(1, bl); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1001) ::decode(name, bl); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1002) ::decode(instance, bl); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1003) DECODE_FINISH(bl); -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1004) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1005) void dump(Formatter *f) const; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1006) }; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1007) WRITE_CLASS_ENCODER(rgw_obj_key) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1008) -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1009) inline ostream& operator<<(ostream& out, const rgw_obj_key &o) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1010) if (o.instance.empty()) { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1011) return out << o.name; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1012) } else { -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1013) return out << o.name << "[" << o.instance << "]"; -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1014) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1015) } -7f26ab76 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 14:18:41 -0700 1016) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1017) /* Holds info on whether the request should be -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1018) * validated via EC2 signature or Auth tokens. -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1019) * Holds value when the action is COPY -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1020) * Holds value when the token to be validated is from a presigned URL */ -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1021) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1022) class authorization_method { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1023) private: -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1024) bool _token_validation; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1025) bool _copy_action; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1026) bool _url_type_token; -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1027) bool _infinite_url_type_token; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1028) bool _acl_main_override; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1029) bool _acl_copy_override; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1030) string _token; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1031) string _copy_source; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1032) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1033) public: -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1034) inline bool get_token_validation() -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1035) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1036) return _token_validation; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1037) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1038) inline void set_token_validation(bool method) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1039) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1040) _token_validation = method; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1041) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1042) inline string get_token() -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1043) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1044) return _token; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1045) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1046) inline void set_token(string tok) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1047) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1048) _token = tok; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1049) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1050) inline bool get_copy_action() -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1051) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1052) return _copy_action; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1053) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1054) inline void set_copy_action(bool action) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1055) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1056) _copy_action = action; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1057) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1058) inline string get_copy_source() -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1059) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1060) return _copy_source; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1061) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1062) inline void set_copy_source(string source) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1063) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1064) _copy_source = source; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1065) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1066) inline bool get_url_type_token() -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1067) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1068) return _url_type_token; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1069) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1070) inline void set_url_type_token(bool val) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1071) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1072) _url_type_token = val; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1073) } -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1074) inline bool get_infinite_url_type_token() -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1075) { -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1076) return _infinite_url_type_token; -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1077) } -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1078) inline void set_infinite_url_type_token(bool val) -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1079) { -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1080) _infinite_url_type_token = val; -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1081) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1082) inline bool get_acl_main_override() -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1083) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1084) return _acl_main_override; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1085) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1086) inline void set_acl_main_override(bool val) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1087) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1088) _acl_main_override = val; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1089) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1090) inline bool get_acl_copy_override() -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1091) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1092) return _acl_copy_override; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1093) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1094) inline void set_acl_copy_override(bool val) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1095) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1096) _acl_copy_override = val; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1097) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1098) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1099) -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1100) authorization_method(bool method, bool action, bool url_token, -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1101) bool infini_token, bool acl_main, bool acl_copy) : -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1102) _token_validation(method), -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1103) _copy_action(action), -6fc470eb src/rgw/rgw_common.h (root 2016-04-08 18:17:45 +0530 1104) _url_type_token(url_token), -4da7376c src/rgw/rgw_common.h (Shivanshu Goswami 2016-04-26 10:14:00 +0000 1105) _infinite_url_type_token(infini_token), -6fc470eb src/rgw/rgw_common.h (root 2016-04-08 18:17:45 +0530 1106) _acl_main_override(acl_main), -6fc470eb src/rgw/rgw_common.h (root 2016-04-08 18:17:45 +0530 1107) _acl_copy_override(acl_copy) { } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1108) ~authorization_method() { } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1109) }; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1110) -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1111) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1112) /** Store all the state necessary to complete and respond to an HTTP request*/ -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1113) struct req_state { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1114) CephContext *cct; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1115) RGWClientIO *cio; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1116) http_op op; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1117) bool content_started; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1118) int format; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1119) ceph::Formatter *formatter; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1120) string decoded_uri; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1121) string relative_uri; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1122) const char *length; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1123) int64_t content_length; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1124) map generic_attrs; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1125) struct rgw_err err; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1126) bool expect_cont; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1127) bool header_ended; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1128) uint64_t obj_size; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1129) bool enable_ops_log; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1130) bool enable_usage_log; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1131) uint8_t defer_to_bucket_acls; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1132) uint32_t perm_mask; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1133) utime_t header_time; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1134) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1135) rgw_bucket bucket; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1136) string bucket_name_str; -c270db3a src/rgw/rgw_common.h (Gaurav Bafna 2016-04-14 15:00:01 +0530 1137) string bucket_owner_id; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1138) rgw_obj_key object; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1139) string src_bucket_name; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1140) rgw_obj_key src_object; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1141) ACLOwner bucket_owner; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1142) ACLOwner owner; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1143) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1144) string region_endpoint; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1145) string bucket_instance_id; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1146) -ec07cbc1 src/rgw/rgw_common.h (Yehuda Sadeh 2015-03-26 21:25:54 -0700 1147) string redirect; -ec07cbc1 src/rgw/rgw_common.h (Yehuda Sadeh 2015-03-26 21:25:54 -0700 1148) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1149) RGWBucketInfo bucket_info; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1150) map bucket_attrs; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1151) bool bucket_exists; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1152) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1153) bool has_bad_meta; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1154) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1155) RGWUserInfo user; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1156) RGWAccessControlPolicy *bucket_acl; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1157) RGWAccessControlPolicy *object_acl; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1158) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1159) bool system_request; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1160) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1161) string canned_acl; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1162) bool has_acl_header; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1163) const char *copy_source; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1164) const char *http_auth; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1165) bool local_source; /* source is local */ -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1166) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1167) int prot_flags; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1168) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1169) const char *os_auth_token; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1170) string swift_user; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1171) string swift_groups; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1172) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1173) utime_t time; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1174) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1175) void *obj_ctx; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1176) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1177) string dialect; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1178) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1179) string req_id; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1180) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1181) string trans_id; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1182) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1183) req_info info; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1184) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1185) authorization_method auth_method; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1186) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1187) req_state(CephContext *_cct, class RGWEnv *e); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1188) ~req_state(); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 1189) }; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 1190) -62e11112 src/rgw/rgw_common.h (Greg Farnum 2010-03-11 14:49:27 -0800 1191) /** Store basic data on an object */ -d0f58ad8 src/radosgw/rgw_common.h (Yehuda Sadeh 2009-08-07 10:46:29 -0700 1192) struct RGWObjEnt { -cb3694fb src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 15:12:21 -0700 1193) rgw_obj_key key; -49b3d2e0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-16 17:44:46 -0700 1194) std::string ns; -edc6659b src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-10 15:01:04 -0700 1195) std::string owner; -edc6659b src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-10 15:01:04 -0700 1196) std::string owner_display_name; -df2967a6 src/rgw/rgw_common.h (Greg Farnum 2011-10-24 15:22:35 -0700 1197) uint64_t size; -b295c649 src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-15 22:49:26 -0700 1198) utime_t mtime; -98d0361d src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-21 17:14:48 -0700 1199) string etag; -b738b72c src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-23 16:50:21 -0700 1200) string content_type; -b295c649 src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-15 22:49:26 -0700 1201) string tag; -105ba489 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 09:48:17 -0700 1202) uint32_t flags; -ecd5496d src/rgw/rgw_common.h (Yehuda Sadeh 2014-12-18 16:43:17 -0800 1203) uint64_t versioned_epoch; -8b4b8384 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-16 13:26:19 -0700 1204) -ecd5496d src/rgw/rgw_common.h (Yehuda Sadeh 2014-12-18 16:43:17 -0800 1205) RGWObjEnt() : size(0), flags(0), versioned_epoch(0) {} -e6e36c0a src/rgw/rgw_common.h (Sage Weil 2012-07-03 18:51:02 -0700 1206) -b295c649 src/rgw/rgw_common.h (Yehuda Sadeh 2013-04-15 22:49:26 -0700 1207) void dump(Formatter *f) const; -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1208) -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1209) bool is_current() { -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1210) uint32_t test_flags = RGW_BUCKET_DIRENT_FLAG_VER | RGW_BUCKET_DIRENT_FLAG_CURRENT; -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1211) return (flags & RGW_BUCKET_DIRENT_FLAG_VER) == 0 || -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1212) (flags & test_flags) == test_flags; -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1213) } -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1214) bool is_delete_marker() { return (flags & RGW_BUCKET_DIRENT_FLAG_DELETE_MARKER) != 0; } -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1215) bool is_visible() { -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1216) return is_current() && !is_delete_marker(); -debee800 src/rgw/rgw_common.h (Yehuda Sadeh 2014-09-26 11:11:17 -0700 1217) } -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 1218) }; -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 1219) -6752babd src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-26 14:30:26 -0700 1220) /** Store basic data on bucket */ -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1221) struct RGWBucketEnt { -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 1222) rgw_bucket bucket; -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1223) size_t size; -7a32cc60 src/rgw/rgw_common.h (Yehuda Sadeh 2011-11-08 13:42:55 -0800 1224) size_t size_rounded; -a2cf14fe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-15 18:41:21 -0700 1225) time_t creation_time; -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1226) uint64_t count; -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1227) -a2cf14fe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-15 18:41:21 -0700 1228) RGWBucketEnt() : size(0), size_rounded(0), creation_time(0), count(0) {} -949f24d5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-12-30 14:18:40 -0800 1229) -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 1230) RGWBucketEnt(const cls_user_bucket_entry& e) { -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 1231) bucket = e.bucket; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 1232) size = e.size; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 1233) size_rounded = e.size_rounded; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 1234) creation_time = e.creation_time; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 1235) count = e.count; -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 1236) } -23aa65f6 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 15:22:37 -0800 1237) -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 1238) void convert(cls_user_bucket_entry *b) { -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 1239) bucket.convert(&b->bucket); -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 1240) b->size = size; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 1241) b->size_rounded = size_rounded; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 1242) b->creation_time = creation_time; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 1243) b->count = count; -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 1244) } -c7b4d008 src/rgw/rgw_common.h (Yehuda Sadeh 2013-12-06 16:23:33 -0800 1245) -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1246) void encode(bufferlist& bl) const { -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 1247) ENCODE_START(5, 5, bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1248) uint64_t s = size; -a2cf14fe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-15 18:41:21 -0700 1249) __u32 mt = creation_time; -9065dbd3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-02-13 12:07:17 -0800 1250) string empty_str; // originally had the bucket name here, but we encode bucket later -9065dbd3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-02-13 12:07:17 -0800 1251) ::encode(empty_str, bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1252) ::encode(s, bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1253) ::encode(mt, bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1254) ::encode(count, bl); -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 1255) ::encode(bucket, bl); -7a32cc60 src/rgw/rgw_common.h (Yehuda Sadeh 2011-11-08 13:42:55 -0800 1256) s = size_rounded; -7a32cc60 src/rgw/rgw_common.h (Yehuda Sadeh 2011-11-08 13:42:55 -0800 1257) ::encode(s, bl); -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 1258) ENCODE_FINISH(bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1259) } -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1260) void decode(bufferlist::iterator& bl) { -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 1261) DECODE_START_LEGACY_COMPAT_LEN(5, 5, 5, bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1262) __u32 mt; -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1263) uint64_t s; -9065dbd3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-02-13 12:07:17 -0800 1264) string empty_str; // backward compatibility -9065dbd3 src/rgw/rgw_common.h (Yehuda Sadeh 2012-02-13 12:07:17 -0800 1265) ::decode(empty_str, bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1266) ::decode(s, bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1267) ::decode(mt, bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1268) size = s; -a2cf14fe src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-15 18:41:21 -0700 1269) creation_time = mt; -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1270) if (struct_v >= 2) -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1271) ::decode(count, bl); -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 1272) if (struct_v >= 3) -ff9537e5 src/rgw/rgw_common.h (Yehuda Sadeh 2011-08-17 17:11:55 -0700 1273) ::decode(bucket, bl); -7a32cc60 src/rgw/rgw_common.h (Yehuda Sadeh 2011-11-08 13:42:55 -0800 1274) if (struct_v >= 4) -7a32cc60 src/rgw/rgw_common.h (Yehuda Sadeh 2011-11-08 13:42:55 -0800 1275) ::decode(s, bl); -7a32cc60 src/rgw/rgw_common.h (Yehuda Sadeh 2011-11-08 13:42:55 -0800 1276) size_rounded = s; -ffcf62f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-07 16:29:11 -0800 1277) DECODE_FINISH(bl); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1278) } -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 1279) void dump(Formatter *f) const; -2277fb45 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-08 16:58:00 -0800 1280) static void generate_test_instances(list& o); -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1281) }; -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1282) WRITE_CLASS_ENCODER(RGWBucketEnt) -9bd627d1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-03 14:14:19 -0800 1283) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1284) class rgw_obj { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1285) std::string orig_obj; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1286) std::string loc; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1287) std::string object; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1288) std::string instance; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1289) public: -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1290) const std::string& get_object() const { return object; } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1291) const std::string& get_orig_obj() const { return orig_obj; } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1292) const std::string& get_loc() const { return loc; } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1293) const std::string& get_instance() const { return instance; } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1294) rgw_bucket bucket; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1295) std::string ns; -8bd984d9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-08 12:32:50 -0700 1296) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1297) bool in_extra_data; /* in-memory only member, does not serialize */ -7989cbd4 src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-11 13:24:55 -0700 1298) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1299) // Represents the hash index source for this object once it is set (non-empty) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1300) std::string index_hash_source; -6f44f7a0 src/rgw/rgw_common.h (Yehuda Sadeh 2015-01-21 17:30:32 -0800 1301) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1302) rgw_obj() : in_extra_data(false) {} -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1303) rgw_obj(rgw_bucket& b, const std::string& o) : in_extra_data(false) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1304) init(b, o); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1305) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1306) rgw_obj(rgw_bucket& b, const rgw_obj_key& k) : in_extra_data(false) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1307) init(b, k.name); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1308) set_instance(k.instance); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1309) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1310) void init(rgw_bucket& b, const std::string& o) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1311) bucket = b; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1312) set_obj(o); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1313) reset_loc(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1314) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1315) void init_ns(rgw_bucket& b, const std::string& o, const std::string& n) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1316) bucket = b; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1317) set_ns(n); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1318) set_obj(o); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1319) reset_loc(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1320) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1321) int set_ns(const char *n) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1322) if (!n) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1323) return -EINVAL; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1324) string ns_str(n); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1325) return set_ns(ns_str); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1326) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1327) int set_ns(const string& n) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1328) if (n[0] == '_') -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1329) return -EINVAL; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1330) ns = n; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1331) set_obj(orig_obj); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1332) return 0; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1333) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1334) int set_instance(const string& i) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1335) if (i[0] == '_') -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1336) return -EINVAL; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1337) instance = i; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1338) set_obj(orig_obj); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1339) return 0; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1340) } -802e9e5a src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-09 13:25:46 -0700 1341) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1342) int clear_instance() { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1343) return set_instance(string()); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1344) } -cb94d55c src/rgw/rgw_common.h (Yehuda Sadeh 2014-12-15 22:12:48 -0800 1345) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1346) void set_loc(const string& k) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1347) loc = k; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1348) } -802e9e5a src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-09 13:25:46 -0700 1349) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1350) void reset_loc() { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1351) loc.clear(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1352) /* -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1353) * For backward compatibility. Older versions used to have object locator on all objects, -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1354) * however, the orig_obj was the effective object locator. This had the same effect as not -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1355) * having object locator at all for most objects but the ones that started with underscore as -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1356) * these were escaped. -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1357) */ -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1358) if (orig_obj[0] == '_' && ns.empty()) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1359) loc = orig_obj; -512ae4cb src/rgw/rgw_common.h (Yehuda Sadeh 2015-04-21 17:31:41 -0700 1360) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1361) } -4e9ebd6b src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-05 12:37:05 -0800 1362) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1363) bool have_null_instance() { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1364) return instance == "null"; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1365) } -3e48a49f src/rgw/rgw_common.h (Yehuda Sadeh 2014-10-28 17:01:34 -0700 1366) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1367) bool have_instance() { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1368) return !instance.empty(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1369) } -7f139286 src/rgw/rgw_common.h (Yehuda Sadeh 2014-10-27 17:09:57 -0700 1370) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1371) bool need_to_encode_instance() { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1372) return have_instance() && !have_null_instance(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1373) } -524a155e src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-25 14:08:01 -0700 1374) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1375) void set_obj(const string& o) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1376) object.reserve(128); -802e9e5a src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-09 13:25:46 -0700 1377) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1378) orig_obj = o; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1379) if (ns.empty() && !need_to_encode_instance()) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1380) if (o.empty()) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1381) return; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1382) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1383) if (o.size() < 1 || o[0] != '_') { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1384) object = o; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1385) return; -45586aa2 src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 12:49:55 -0700 1386) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1387) object = "_"; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1388) object.append(o); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1389) } else { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1390) object = "_"; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1391) object.append(ns); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1392) if (need_to_encode_instance()) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1393) object.append(string(":") + instance); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1394) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1395) object.append("_"); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1396) object.append(o); -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1397) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1398) reset_loc(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1399) } -45586aa2 src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 12:49:55 -0700 1400) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1401) /* -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1402) * get the object's key name as being referred to by the bucket index. -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1403) */ -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1404) string get_index_key_name() const { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1405) if (ns.empty()) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1406) if (orig_obj.size() < 1 || orig_obj[0] != '_') { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1407) return orig_obj; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1408) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1409) return string("_") + orig_obj; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1410) }; -45586aa2 src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 12:49:55 -0700 1411) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1412) char buf[ns.size() + 16]; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1413) snprintf(buf, sizeof(buf), "_%s_", ns.c_str()); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1414) return string(buf) + orig_obj; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1415) }; -cb3694fb src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 15:12:21 -0700 1416) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1417) void get_index_key(rgw_obj_key *key) const { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1418) key->name = get_index_key_name(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1419) key->instance = instance; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1420) } -45586aa2 src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 12:49:55 -0700 1421) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1422) static void parse_ns_field(string& ns, string& instance) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1423) int pos = ns.find(':'); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1424) if (pos >= 0) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1425) instance = ns.substr(pos + 1); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1426) ns = ns.substr(0, pos); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1427) } else { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1428) instance.clear(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1429) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1430) } -802e9e5a src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-09 13:25:46 -0700 1431) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1432) string& get_hash_object() { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1433) return index_hash_source.empty() ? orig_obj : index_hash_source; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1434) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1435) /** -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1436) * Translate a namespace-mangled object name to the user-facing name -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1437) * existing in the given namespace. -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1438) * -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1439) * If the object is part of the given namespace, it returns true -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1440) * and cuts down the name to the unmangled version. If it is not -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1441) * part of the given namespace, it returns false. -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1442) */ -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1443) static bool translate_raw_obj_to_obj_in_ns(string& obj, string& instance, string& ns) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1444) if (obj[0] != '_') { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1445) if (ns.empty()) { -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1446) return true; -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1447) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1448) return false; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1449) } -802e9e5a src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-09 13:25:46 -0700 1450) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1451) string obj_ns; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1452) bool ret = parse_raw_oid(obj, &obj, &instance, &obj_ns); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1453) if (!ret) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1454) return ret; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1455) } -6c6aa5dd src/rgw/rgw_common.h (Yehuda Sadeh 2015-04-29 17:12:00 -0700 1456) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1457) return (ns == obj_ns); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1458) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1459) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1460) static bool parse_raw_oid(const string& oid, string *obj_name, string *obj_instance, string *obj_ns) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1461) obj_instance->clear(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1462) obj_ns->clear(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1463) if (oid[0] != '_') { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1464) *obj_name = oid; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1465) return true; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1466) } -6c6aa5dd src/rgw/rgw_common.h (Yehuda Sadeh 2015-04-29 17:12:00 -0700 1467) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1468) if (oid.size() >= 2 && oid[1] == '_') { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1469) *obj_name = oid.substr(1); -6c6aa5dd src/rgw/rgw_common.h (Yehuda Sadeh 2015-04-29 17:12:00 -0700 1470) return true; -6c6aa5dd src/rgw/rgw_common.h (Yehuda Sadeh 2015-04-29 17:12:00 -0700 1471) } -6c6aa5dd src/rgw/rgw_common.h (Yehuda Sadeh 2015-04-29 17:12:00 -0700 1472) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1473) if (oid[0] != '_' || oid.size() < 3) // for namespace, min size would be 3: _x_ -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1474) return false; -802e9e5a src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-09 13:25:46 -0700 1475) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1476) int pos = oid.find('_', 1); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1477) if (pos <= 1) // if it starts with __, it's not in our namespace -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1478) return false; -802e9e5a src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-09 13:25:46 -0700 1479) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1480) *obj_ns = oid.substr(1, pos - 1); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1481) parse_ns_field(*obj_ns, *obj_instance); -802e9e5a src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-09 13:25:46 -0700 1482) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1483) *obj_name = oid.substr(pos + 1); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1484) return true; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1485) } -e6eef5e9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-30 17:13:42 -0700 1486) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1487) /** -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1488) * Given a mangled object name and an empty namespace string, this -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1489) * function extracts the namespace into the string and sets the object -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1490) * name to be the unmangled version. -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1491) * -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1492) * It returns true after successfully doing so, or -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1493) * false if it fails. -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1494) */ -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1495) static bool strip_namespace_from_object(string& obj, string& ns, string& instance) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1496) ns.clear(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1497) instance.clear(); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1498) if (obj[0] != '_') { -f57c33df src/rgw/rgw_common.h (Greg Farnum 2011-10-20 16:26:15 -0700 1499) return true; -f57c33df src/rgw/rgw_common.h (Greg Farnum 2011-10-20 16:26:15 -0700 1500) } -f57c33df src/rgw/rgw_common.h (Greg Farnum 2011-10-20 16:26:15 -0700 1501) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1502) size_t pos = obj.find('_', 1); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1503) if (pos == string::npos) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1504) return false; -f57c33df src/rgw/rgw_common.h (Greg Farnum 2011-10-20 16:26:15 -0700 1505) } -f57c33df src/rgw/rgw_common.h (Greg Farnum 2011-10-20 16:26:15 -0700 1506) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1507) size_t period_pos = obj.find('.'); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1508) if (period_pos < pos) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1509) return false; -f57c33df src/rgw/rgw_common.h (Greg Farnum 2011-10-20 16:26:15 -0700 1510) } -f57c33df src/rgw/rgw_common.h (Greg Farnum 2011-10-20 16:26:15 -0700 1511) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1512) ns = obj.substr(1, pos-1); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1513) obj = obj.substr(pos+1, string::npos); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1514) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1515) parse_ns_field(ns, instance); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1516) return true; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1517) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1518) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1519) void set_in_extra_data(bool val) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1520) in_extra_data = val; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1521) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1522) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1523) bool is_in_extra_data() const { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1524) return in_extra_data; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1525) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1526) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1527) void encode(bufferlist& bl) const { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1528) ENCODE_START(5, 3, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1529) ::encode(bucket.name, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1530) ::encode(loc, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1531) ::encode(ns, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1532) ::encode(object, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1533) ::encode(bucket, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1534) ::encode(instance, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1535) if (!ns.empty() || !instance.empty()) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1536) ::encode(orig_obj, bl); -3e54acbc src/rgw/rgw_common.h (Yehuda Sadeh 2015-02-09 18:16:13 -0800 1537) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1538) ENCODE_FINISH(bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1539) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1540) void decode(bufferlist::iterator& bl) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1541) DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1542) ::decode(bucket.name, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1543) ::decode(loc, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1544) ::decode(ns, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1545) ::decode(object, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1546) if (struct_v >= 2) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1547) ::decode(bucket, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1548) if (struct_v >= 4) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1549) ::decode(instance, bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1550) if (ns.empty() && instance.empty()) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1551) orig_obj = object; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1552) } else { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1553) if (struct_v >= 5) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1554) ::decode(orig_obj, bl); -3e54acbc src/rgw/rgw_common.h (Yehuda Sadeh 2015-02-09 18:16:13 -0800 1555) } else { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1556) ssize_t pos = object.find('_', 1); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1557) if (pos < 0) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1558) throw buffer::error(); -3e54acbc src/rgw/rgw_common.h (Yehuda Sadeh 2015-02-09 18:16:13 -0800 1559) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1560) orig_obj = object.substr(pos); -3e54acbc src/rgw/rgw_common.h (Yehuda Sadeh 2015-02-09 18:16:13 -0800 1561) } -4d884040 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-24 16:43:14 -0700 1562) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1563) DECODE_FINISH(bl); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1564) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1565) void dump(Formatter *f) const; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1566) static void generate_test_instances(list& o); -4d884040 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-24 16:43:14 -0700 1567) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1568) bool operator==(const rgw_obj& o) const { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1569) return (object.compare(o.object) == 0) && -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1570) (bucket.name.compare(o.bucket.name) == 0) && -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1571) (ns.compare(o.ns) == 0) && -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1572) (instance.compare(o.instance) == 0); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1573) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1574) bool operator<(const rgw_obj& o) const { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1575) int r = bucket.name.compare(o.bucket.name); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1576) if (r == 0) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1577) r = object.compare(o.object); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1578) if (r == 0) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1579) r = ns.compare(o.ns); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1580) if (r == 0) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1581) r = instance.compare(o.instance); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1582) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1583) } -d6641cb4 src/rgw/rgw_common.h (root 2016-04-08 15:28:18 +0530 1584) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1585) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1586) return (r < 0); -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1587) } -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1588) }; -e6eef5e9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-30 17:13:42 -0700 1589) WRITE_CLASS_ENCODER(rgw_obj) -8bd984d9 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-08 12:32:50 -0700 1590) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1591) struct rgw_cache_entry_info { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1592) string cache_locator; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1593) uint64_t gen; -ab764f38 src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-19 16:34:21 -0700 1594) -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1595) rgw_cache_entry_info() : gen(0) {} -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1596) }; -ab764f38 src/rgw/rgw_common.h (Yehuda Sadeh 2014-03-19 16:34:21 -0700 1597) -f106e1a5 src/rgw/rgw_common.h (Danny Al-Gaaf 2013-03-11 15:35:53 +0100 1598) inline ostream& operator<<(ostream& out, const rgw_obj &o) { -45586aa2 src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 12:49:55 -0700 1599) return out << o.bucket.name << ":" << o.get_object(); -7bbdcdba src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 12:55:25 -0700 1600) } -7bbdcdba src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 12:55:25 -0700 1601) -422bb6d0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 11:03:12 -0700 1602) static inline bool str_startswith(const string& str, const string& prefix) -422bb6d0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 11:03:12 -0700 1603) { -422bb6d0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 11:03:12 -0700 1604) return (str.compare(0, prefix.size(), prefix) == 0); -422bb6d0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 11:03:12 -0700 1605) } -422bb6d0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-25 11:03:12 -0700 1606) -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1607) static inline void buf_to_hex(const unsigned char *buf, int len, char *str) -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1608) { -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1609) int i; -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1610) str[0] = '\0'; -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1611) for (i = 0; i < len; i++) { -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1612) sprintf(&str[i*2], "%02x", (int)buf[i]); -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1613) } -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1614) } -e2100bce src/s3/s3common.h (Yehuda Sadeh 2009-07-20 16:41:25 -0700 1615) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1616) static inline int hexdigit(char c) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1617) { -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1618) if (c >= '0' && c <= '9') -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1619) return (c - '0'); -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1620) c = toupper(c); -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1621) if (c >= 'A' && c <= 'F') -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1622) return c - 'A' + 0xa; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1623) return -EINVAL; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1624) } -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1625) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1626) static inline int hex_to_buf(const char *hex, char *buf, int len) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1627) { -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1628) int i = 0; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1629) const char *p = hex; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1630) while (*p) { -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1631) if (i >= len) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1632) return -EINVAL; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1633) buf[i] = 0; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1634) int d = hexdigit(*p); -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1635) if (d < 0) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1636) return d; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1637) buf[i] = d << 4; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1638) p++; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1639) if (!*p) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1640) return -EINVAL; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1641) d = hexdigit(*p); -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1642) if (d < 0) -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1643) return -d; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1644) buf[i] += d; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1645) i++; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1646) p++; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1647) } -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1648) return i; -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1649) } -37fd3b58 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-07 14:13:59 -0700 1650) -f04b6adc src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-10 10:01:24 -0800 1651) static inline int rgw_str_to_bool(const char *s, int def_val) -f04b6adc src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-10 10:01:24 -0800 1652) { -f04b6adc src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-10 10:01:24 -0800 1653) if (!s) -f04b6adc src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-10 10:01:24 -0800 1654) return def_val; -f04b6adc src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-10 10:01:24 -0800 1655) -f04b6adc src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-10 10:01:24 -0800 1656) return (strcasecmp(s, "on") == 0 || -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1657) strcasecmp(s, "yes") == 0 || -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1658) strcasecmp(s, "1") == 0); -f04b6adc src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-10 10:01:24 -0800 1659) } -f04b6adc src/rgw/rgw_common.h (Yehuda Sadeh 2011-03-10 10:01:24 -0800 1660) -45586aa2 src/rgw/rgw_common.h (Yehuda Sadeh 2014-08-06 12:49:55 -0700 1661) static inline void append_rand_alpha(CephContext *cct, const string& src, string& dest, int len) -d0340426 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 14:50:01 -0700 1662) { -d0340426 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 14:50:01 -0700 1663) dest = src; -d0340426 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 14:50:01 -0700 1664) char buf[len + 1]; -b9097619 src/rgw/rgw_common.h (Yehuda Sadeh 2012-03-13 14:59:00 -0700 1665) gen_rand_alphanumeric(cct, buf, len); -d0340426 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 14:50:01 -0700 1666) dest.append("_"); -d0340426 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 14:50:01 -0700 1667) dest.append(buf); -d0340426 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 14:50:01 -0700 1668) } -d0340426 src/rgw/rgw_common.h (Yehuda Sadeh 2011-06-24 14:50:01 -0700 1669) -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 1670) static inline const char *rgw_obj_category_name(RGWObjCategory category) -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 1671) { -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 1672) switch (category) { -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1673) case RGW_OBJ_CATEGORY_NONE: -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1674) return "rgw.none"; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1675) case RGW_OBJ_CATEGORY_MAIN: -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1676) return "rgw.main"; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1677) case RGW_OBJ_CATEGORY_SHADOW: -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1678) return "rgw.shadow"; -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1679) case RGW_OBJ_CATEGORY_MULTIMETA: -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1680) return "rgw.multimeta"; -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 1681) } -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 1682) -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 1683) return "unknown"; -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 1684) } -ca77ba77 src/rgw/rgw_common.h (Yehuda Sadeh 2011-09-23 17:11:49 -0700 1685) -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 1686) static inline uint64_t rgw_rounded_kb(uint64_t bytes) -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 1687) { -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 1688) return (bytes + 1023) / 1024; -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 1689) } -db5bbdd0 src/rgw/rgw_common.h (Yehuda Sadeh 2013-09-26 13:24:48 -0700 1690) -2f9a93d3 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-14 00:18:52 -0800 1691) static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes) -2f9a93d3 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-14 00:18:52 -0800 1692) { -2f9a93d3 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-14 00:18:52 -0800 1693) return ((bytes + 4095) & ~4095) / 1024; -2f9a93d3 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-14 00:18:52 -0800 1694) } -2f9a93d3 src/rgw/rgw_common.h (Yehuda Sadeh 2014-01-14 00:18:52 -0800 1695) -97c19da4 src/rgw/rgw_common.h (Yehuda Sadeh 2012-11-07 15:39:56 -0800 1696) extern string rgw_string_unquote(const string& s); -eb0f49d4 src/rgw/rgw_common.h (Caleb Miles 2013-02-19 12:15:30 -0500 1697) extern void parse_csv_string(const string& ival, vector& ovals); -eb0f49d4 src/rgw/rgw_common.h (Caleb Miles 2013-02-19 12:15:30 -0500 1698) extern int parse_key_value(string& in_str, string& key, string& val); -eb0f49d4 src/rgw/rgw_common.h (Caleb Miles 2013-02-19 12:15:30 -0500 1699) extern int parse_key_value(string& in_str, const char *delim, string& key, string& val); -dd3282c1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-14 13:20:20 -0700 1700) /** time parsing */ -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 1701) extern int parse_time(const char *time_str, time_t *time); -dd3282c1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-14 13:20:20 -0700 1702) extern bool parse_rfc2616(const char *s, struct tm *t); -30d11f42 src/rgw/rgw_common.h (Yehuda Sadeh 2012-10-11 15:36:07 -0700 1703) extern bool parse_iso8601(const char *s, struct tm *t); -da5e443c src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-10 21:58:02 -0700 1704) extern string rgw_trim_whitespace(const string& src); -da5e443c src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-10 21:58:02 -0700 1705) extern string rgw_trim_quotes(const string& val); -da5e443c src/rgw/rgw_common.h (Yehuda Sadeh 2013-06-10 21:58:02 -0700 1706) -dd3282c1 src/rgw/rgw_common.h (Yehuda Sadeh 2011-10-14 13:20:20 -0700 1707) -62e11112 src/rgw/rgw_common.h (Greg Farnum 2010-03-11 14:49:27 -0800 1708) /** Check if the req_state's user has the necessary permissions -62e11112 src/rgw/rgw_common.h (Greg Farnum 2010-03-11 14:49:27 -0800 1709) * to do the requested action */ -2365c77a src/rgw/rgw_common.h (Yehuda Sadeh 2012-02-21 14:39:20 -0800 1710) extern bool verify_bucket_permission(struct req_state *s, int perm); -4d2a05f6 src/rgw/rgw_common.h (Yehuda Sadeh 2012-08-22 17:16:05 -0700 1711) extern bool verify_object_permission(struct req_state *s, RGWAccessControlPolicy *bucket_acl, RGWAccessControlPolicy *object_acl, int perm); -2365c77a src/rgw/rgw_common.h (Yehuda Sadeh 2012-02-21 14:39:20 -0800 1712) extern bool verify_object_permission(struct req_state *s, int perm); -62e11112 src/rgw/rgw_common.h (Greg Farnum 2010-03-11 14:49:27 -0800 1713) /** Convert an input URL into a sane object name -62e11112 src/rgw/rgw_common.h (Greg Farnum 2010-03-11 14:49:27 -0800 1714) * by converting %-escaped strings into characters, etc*/ -21e07eb6 src/rgw/rgw_common.h (Yehuda Sadeh 2014-12-11 09:07:10 -0800 1715) extern bool url_decode(string& src_str, string& dest_str, bool in_query = false); -dd308cd4 src/rgw/rgw_common.h (Josh Durgin 2013-10-24 08:37:25 -0700 1716) extern void url_encode(const string& src, string& dst); -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 1717) -70b021d4 src/rgw/rgw_common.h (Colin P. McCabe 2011-03-23 22:42:02 -0700 1718) extern void calc_hmac_sha1(const char *key, int key_len, -a96dd147 src/rgw/rgw_common.h (root 2016-04-08 15:50:30 +0530 1719) const char *msg, int msg_len, char *dest); -70b021d4 src/rgw/rgw_common.h (Colin P. McCabe 2011-03-23 22:42:02 -0700 1720) /* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */ -70b021d4 src/rgw/rgw_common.h (Colin P. McCabe 2011-03-23 22:42:02 -0700 1721) -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 1722) extern int rgw_parse_op_type_list(const string& str, uint32_t *perm); -38464515 src/rgw/rgw_common.h (Yehuda Sadeh 2013-05-02 21:05:21 -0700 1723) -536a8b64 src/s3/s3common.h (Yehuda Sadeh 2009-07-17 17:49:59 -0700 1724) #endif From 72c48ee55241dbad3f40ef08177ac924bb495fa5 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna Date: Thu, 7 Jul 2016 18:25:24 +1100 Subject: [PATCH 48/60] Fixing the prefix to jcs --- src/rgw/rgw_common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 312251c6763ec..ed5f4e596e240 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -52,7 +52,7 @@ using ceph::crypto::MD5; #define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_" #define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-" -#define RGW_AMZ_PREFIX "x-amz-" +#define RGW_AMZ_PREFIX "x-jcs-" #define RGW_AMZ_META_PREFIX RGW_AMZ_PREFIX "x-jcs-meta-" #define RGW_AMZ_WEBSITE_REDIRECT_LOCATION RGW_AMZ_PREFIX "website-redirect-location" From 37eadf87010859d18da2e2f39e9595edcee067ff Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Thu, 4 Aug 2016 22:30:36 +1100 Subject: [PATCH 49/60] Fixing the bug in rename API where original name contains '%' character Signed-off-by: Harshal Gupta --- src/rgw/rgw_op.cc | 16 ++++++++++++---- src/rgw/rgw_op.h | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 7107d995d9c7c..9ca1037850225 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -2265,7 +2265,6 @@ bool RGWCopyObj::parse_copy_location(const string& url_src, string& bucket_name, string dec_src; - url_decode(name_str, dec_src); const char *src = dec_src.c_str(); @@ -3535,7 +3534,7 @@ void RGWRenameObj::execute() s->err.ret = 0; rgw_obj_key orig_object, new_obj; orig_object.dss_duplicate(&(s->object)); - string copysource; + string copysource, raw_copy_source; int ret_orig, ret_newobj; RGWCopyObj_ObjStore_S3* copy_op = NULL; RGWDeleteObj_ObjStore_S3* del_op = NULL; @@ -3558,7 +3557,7 @@ void RGWRenameObj::execute() s->err.ret = -ERR_RENAME_OBJ_EXISTS; return; } - + raw_copy_source = get_raw_copy_source(); copysource = s->bucket_name_str; copysource.append("/"); copysource.append(orig_object.name); @@ -3568,7 +3567,7 @@ void RGWRenameObj::execute() s->info.env->set("HTTP_X_JCS_METADATA_DIRECTIVE", "COPY"); s->copy_source = s->info.env->get("HTTP_X_JCS_COPY_SOURCE"); if (s->copy_source) { - ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object); + ret = RGWCopyObj::parse_copy_location(raw_copy_source, s->src_bucket_name, s->src_object); if (!ret) { //Surprizingly returns bool ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; s->err.ret = -ERR_RENAME_FAILED; @@ -3724,3 +3723,12 @@ void RGWRenameObj::delete_rgw_object(RGWOp* del_op) return; } +string RGWRenameObj::get_raw_copy_source() +{ + string uri = s->info.request_uri; + int start = uri.find('/'); + int end = uri.find('?'); + string raw_copy_source = uri.substr(start+1, end); + return raw_copy_source; +} + diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 40e629d3af7c6..5429f387891a7 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -578,6 +578,7 @@ class RGWRenameObj : public RGWOp { void perform_external_op(RGWOp*); void delete_rgw_object(RGWOp*); int check_obj(rgw_obj_key&); + string get_raw_copy_source(); virtual const string name() { return "Rename_obj"; } }; From 5ca2d024f384459e1f782f53dc3ce1ea69c1bf36 Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Fri, 5 Aug 2016 01:15:27 +1100 Subject: [PATCH 50/60] Adding some additional logs for testing Rename API Signed-off-by: Harshal Gupta --- src/rgw/rgw_op.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 9ca1037850225..39b9045c5fbc9 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3566,10 +3566,12 @@ void RGWRenameObj::execute() s->info.env->set("HTTP_X_JCS_COPY_SOURCE", copysource.c_str()); s->info.env->set("HTTP_X_JCS_METADATA_DIRECTIVE", "COPY"); s->copy_source = s->info.env->get("HTTP_X_JCS_COPY_SOURCE"); + ldout(s->cct, 0) << "DSS INFO: The raw copy location to be parsed: " << raw_copy_source <copy_source) { ret = RGWCopyObj::parse_copy_location(raw_copy_source, s->src_bucket_name, s->src_object); + ldout(s->cct, 0) << "DSS INFO: final source key name: " << s->src_object << "and bucket name: " << s->src_bucket_name << dendl; if (!ret) { //Surprizingly returns bool - ldout(s->cct, 0) << "DSS INFO: Rename op failed to parse copy location" << dendl; + ldout(s->cct, 0) << "DSS ERROR: Rename op failed to parse copy location" << dendl; s->err.ret = -ERR_RENAME_FAILED; s->err.http_ret = 403; return; From 077662d9acc7ad1d98f35018137a9893cfefbce5 Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Fri, 5 Aug 2016 01:29:03 +1100 Subject: [PATCH 51/60] fixing some typo in the previous commit --- src/rgw/rgw_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 39b9045c5fbc9..97d7b5449bf2d 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3569,7 +3569,7 @@ void RGWRenameObj::execute() ldout(s->cct, 0) << "DSS INFO: The raw copy location to be parsed: " << raw_copy_source <copy_source) { ret = RGWCopyObj::parse_copy_location(raw_copy_source, s->src_bucket_name, s->src_object); - ldout(s->cct, 0) << "DSS INFO: final source key name: " << s->src_object << "and bucket name: " << s->src_bucket_name << dendl; + ldout(s->cct, 0) << "DSS INFO: final source key name: " << s->src_object << " and bucket name: " << s->src_bucket_name << dendl; if (!ret) { //Surprizingly returns bool ldout(s->cct, 0) << "DSS ERROR: Rename op failed to parse copy location" << dendl; s->err.ret = -ERR_RENAME_FAILED; From 9aa740acc3190b4c91fd95869a7fdedfec63f4d9 Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Mon, 26 Sep 2016 16:25:31 +1100 Subject: [PATCH 52/60] SSE at RadosGW --- src/common/config_opts.h | 2 + src/rgw/rgw_common.h | 4 + src/rgw/rgw_main.cc | 8 +- src/rgw/rgw_op.cc | 142 ++++++++++++++++++++++++++-- src/rgw/rgw_op.h | 9 +- src/rgw/rgw_rados.cc | 132 +++++++++++++++++++++++++- src/rgw/rgw_rados.h | 25 ++++- src/rgw/rgw_rest.cc | 73 +++++++++++++-- src/rgw/rgw_rest.h | 2 +- src/rgw/rgw_rest_metadata.cc | 2 +- src/rgw/rgw_rest_metadata.h | 2 +- src/rgw/rgw_rest_s3.cc | 173 ++++++++++++++++++++++++++++++++++- src/rgw/rgw_rest_s3.h | 52 ++++++++++- 13 files changed, 588 insertions(+), 38 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 07e30ba0429e2..8c8059ddb1250 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -965,6 +965,8 @@ OPTION(rgw_swift_auth_entry, OPT_STR, "auth") // entry point for which a url is OPTION(rgw_swift_tenant_name, OPT_STR, "") // tenant name to use for swift access OPTION(rgw_swift_enforce_content_length, OPT_BOOL, false) // enforce generation of Content-Length even in cost of performance or scalability OPTION(rgw_keystone_url, OPT_STR, "") // url for keystone server +OPTION(rgw_kms_encrypt_url, OPT_STR, "") // url for kms encrypt +OPTION(rgw_kms_decrypt_url, OPT_STR, "") // url for kms decrypt OPTION(rgw_keystone_admin_token, OPT_STR, "") // keystone admin token (shared secret) OPTION(rgw_keystone_admin_user, OPT_STR, "") // keystone admin user name OPTION(rgw_keystone_admin_password, OPT_STR, "") // keystone admin user password diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index c99a2225cf424..692684a4ad427 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -55,6 +55,9 @@ using ceph::crypto::MD5; #define RGW_SYS_PARAM_PREFIX "rgwx-" #define RGW_ATTR_ACL RGW_ATTR_PREFIX "acl" +#define RGW_ATTR_KEY RGW_ATTR_PREFIX "key" +#define RGW_ATTR_IV RGW_ATTR_PREFIX "iv" +#define RGW_ATTR_MKEYVERSION RGW_ATTR_PREFIX "mkeyversion" #define RGW_ATTR_CORS RGW_ATTR_PREFIX "cors" #define RGW_ATTR_ETAG RGW_ATTR_PREFIX "etag" #define RGW_ATTR_BUCKETS RGW_ATTR_PREFIX "buckets" @@ -153,6 +156,7 @@ using ceph::crypto::MD5; #define ERR_RENAME_COPY_FAILED 2034 #define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 #define ERR_RENAME_OBJ_EXISTS 2036 +#define ERR_INVALID_ENC_ALGO 2037 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index d59e56baf488e..fd83365db3370 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -1241,7 +1241,11 @@ int main(int argc, const char **argv) /* setup openssl multi-threading functions */ openssl_thread_setup(); - + /* Initialise the library */ + ERR_load_crypto_strings(); + OpenSSL_add_all_algorithms(); + OPENSSL_config(NULL); + srand ( time(NULL) ); FCGX_Init(); int r = 0; @@ -1437,6 +1441,8 @@ int main(int argc, const char **argv) /* cleanup openssl multi-threading functions */ openssl_thread_cleanup(); + EVP_cleanup(); + ERR_free_strings(); rgw_perf_stop(g_ceph_context); dout(1) << "final shutdown" << dendl; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 7107d995d9c7c..49ae3dd1a1f58 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -893,9 +893,11 @@ void RGWGetObj::pre_exec() void RGWGetObj::execute() { utime_t start_time = s->time; - bufferlist bl; + bufferlist bl,keybl,ivbl,mkeybl; gc_invalidate_time = ceph_clock_now(s->cct); gc_invalidate_time += (s->cct->_conf->rgw_gc_obj_min_wait / 2); + RGW_KMS req_kms(s->cct); + string root_account = s->bucket_owner_id; RGWGetObj_CB cb(this); @@ -932,6 +934,20 @@ void RGWGetObj::execute() if (ret < 0) goto done_err; + keybl = attrs[RGW_ATTR_KEY]; + ivbl = attrs[RGW_ATTR_IV]; + mkeybl = attrs[RGW_ATTR_MKEYVERSION]; + if (keybl.length()) + { + kmsdata = new RGWKmsData(); + keybl.copy(0,keybl.length(),kmsdata->key_enc); + ivbl.copy(0,ivbl.length(),kmsdata->iv_enc); + mkeybl.copy(0,mkeybl.length(),kmsdata->mkey_enc); + ret = req_kms.make_kms_decrypt_request(root_account,kmsdata); + if (ret < 0) + goto done_err; + dout(0) << "SSEINFO decrypted key "<< kmsdata->key_dec.c_str() << " iv " << kmsdata->iv_dec.c_str() << dendl; + } attr_iter = attrs.find(RGW_ATTR_USER_MANIFEST); if (attr_iter != attrs.end()) { ret = handle_user_manifest(attr_iter->second.c_str()); @@ -961,6 +977,9 @@ void RGWGetObj::execute() done_err: send_response_data(bl, 0, 0); + + if (kmsdata) + delete kmsdata; } int RGWGetObj::init_common() @@ -1524,7 +1543,7 @@ class RGWPutObjProcessor_Multipart : public RGWPutObjProcessor_Atomic string upload_id; protected: - int prepare(RGWRados *store, string *oid_rand); + int prepare(RGWRados *store, string *oid_rand, RGWKmsData** kmsdata=NULL); int do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs, const char *if_match = NULL, const char *if_nomatch = NULL); @@ -1535,7 +1554,7 @@ class RGWPutObjProcessor_Multipart : public RGWPutObjProcessor_Atomic RGWPutObjProcessor_Atomic(obj_ctx, bucket_info, _s->bucket, _s->object.name, _p, _s->req_id, false), s(_s) {} }; -int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand) +int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand, RGWKmsData** kmsdata) { int r = prepare_init(store, NULL); if (r < 0) { @@ -1564,6 +1583,37 @@ int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand) return -EINVAL; } + rgw_obj meta_obj; + string meta_oid; + map attrs; + meta_oid = mp.get_meta(); + meta_obj.init_ns(s->bucket, meta_oid, mp_ns); + meta_obj.set_in_extra_data(true); + meta_obj.index_hash_source = s->object.name; + int ret; + + ret = get_obj_attrs(store, s, meta_obj, attrs); + if (ret < 0) { + ldout(s->cct, 0) << "ERROR: failed to get obj attrs, obj=" << meta_obj << " ret=" << ret << dendl; + //return; + } + else { + RGW_KMS req_kms(s->cct); + string root_account = s->bucket_owner_id; + bufferlist keybl = attrs[RGW_ATTR_KEY]; + bufferlist ivbl = attrs[RGW_ATTR_IV]; + bufferlist mkeybl = attrs[RGW_ATTR_MKEYVERSION]; + if (keybl.length() > 0) + { + *kmsdata = new RGWKmsData(); + keybl.copy(0,keybl.length(),(*kmsdata)->key_enc); + ivbl.copy(0,ivbl.length(),(*kmsdata)->iv_enc); + mkeybl.copy(0,mkeybl.length(),(*kmsdata)->mkey_enc); + ret = req_kms.make_kms_decrypt_request(root_account,*kmsdata); + if (ret < 0) + return ret; + } + } string upload_prefix = oid + "."; if (!oid_rand) { @@ -1741,12 +1791,15 @@ void RGWPutObj::execute() char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; MD5 hash; - bufferlist bl, aclbl; + bufferlist bl, aclbl,keybl,ivbl,mkeybl; map attrs; int len; map::iterator iter; - bool multipart; - + bool multipart, is_encrypted = false; + int i = 0; + bool exists; + const char* is_enc; + bool need_calc_md5 = (obj_manifest == NULL); @@ -1796,14 +1849,45 @@ void RGWPutObj::execute() } processor = select_processor(*(RGWObjectCtx *)s->obj_ctx, &multipart); + is_enc = s->info.env->get("HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION"); - ret = processor->prepare(store, NULL); - if (ret < 0) + if (!is_enc) + is_enc = s->info.env->get("HTTP_X_JCS_SERVER_SIDE_ENCRYPTION"); + + if (is_enc && (strcmp(is_enc,"AES256") != 0)) + { + // wrong value algo. Error out + ret = -ERR_INVALID_ENC_ALGO; goto done; + } + else if (is_enc) + is_encrypted = true; + if (!multipart && is_encrypted) + { + RGW_KMS req_kms(s->cct); + kmsdata = new RGWKmsData(); + string root_account = s->bucket_owner_id; + ret = req_kms.make_kms_encrypt_request(root_account,kmsdata); + if (ret < 0) + goto done; + //Get the key from KMS and store it as attribute + keybl.append(kmsdata->key_enc.c_str()); + ivbl.append(kmsdata->iv_enc.c_str()); + mkeybl.append(kmsdata->mkey_enc.c_str()); + attrs[RGW_ATTR_KEY] = keybl; + attrs[RGW_ATTR_IV] = ivbl; + attrs[RGW_ATTR_MKEYVERSION] = mkeybl; + } + + //For multipart, the key gets populated here + ret = processor->prepare(store, NULL, &kmsdata); + if (ret < 0) + goto done; do { + i++; bufferlist data; - len = get_data(data); + len = get_data(data,&hash); if (len < 0) { ret = len; goto done; @@ -1949,6 +2033,8 @@ void RGWPutObj::execute() dispose_processor(processor); perfcounter->tinc(l_rgw_put_lat, (ceph_clock_now(s->cct) - s->time)); + if (kmsdata) + delete kmsdata; } int RGWPostObj::verify_permission() @@ -2785,6 +2871,9 @@ void RGWInitMultipart::execute() map attrs; rgw_obj obj; map::iterator iter; + bool is_encrypted = false; + bufferlist keybl,ivbl,mkeybl; + string key,iv; if (get_params() < 0) return; @@ -2796,6 +2885,41 @@ void RGWInitMultipart::execute() attrs[RGW_ATTR_ACL] = aclbl; + const char* is_enc = s->info.env->get("HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION"); + + if (!is_enc) + is_enc = s->info.env->get("HTTP_X_JCS_SERVER_SIDE_ENCRYPTION"); + + //Get the key from KMS and store it as attribute + if (is_enc && (strcmp(is_enc,"AES256") != 0)) + { + // wrong value algo. Error out + ret = -ERR_INVALID_ENC_ALGO; + return; + } + else if (is_enc) + is_encrypted = true; + + is_encrypted = (is_enc) ? (strcmp(is_enc,"AES256")== 0) : false ; + if (is_encrypted) + { + RGW_KMS req_kms(s->cct); + RGWKmsData* kmsdata = new RGWKmsData(); + string root_account = s->bucket_owner_id; + ret = req_kms.make_kms_encrypt_request(root_account,kmsdata); + if (ret < 0) + return; + //Get the key from KMS and store it as attribute + dout(0) << "SSEINFO : Multipart uploading with key " << key << dendl; + keybl.append(kmsdata->key_enc.c_str()); + ivbl.append(kmsdata->iv_enc.c_str()); + mkeybl.append(kmsdata->mkey_enc.c_str()); + attrs[RGW_ATTR_KEY] = keybl; + attrs[RGW_ATTR_IV] = ivbl; + attrs[RGW_ATTR_MKEYVERSION] = mkeybl; + delete kmsdata; + } + for (iter = s->generic_attrs.begin(); iter != s->generic_attrs.end(); ++iter) { bufferlist& attrbl = attrs[iter->first]; const string& val = iter->second; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 40e629d3af7c6..8ce533fcc5f0b 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -121,6 +121,7 @@ class RGWGetObj : public RGWOp { const char *if_unmod; const char *if_match; const char *if_nomatch; + RGWKmsData* kmsdata; off_t ofs; uint64_t total_len; off_t start; @@ -145,7 +146,7 @@ class RGWGetObj : public RGWOp { if_unmod = NULL; if_match = NULL; if_nomatch = NULL; - start = 0; + kmsdata = NULL; ofs = 0; total_len = 0; end = -1; @@ -412,6 +413,7 @@ class RGWPutObj : public RGWOp { const char *if_match; const char *if_nomatch; string etag; + RGWKmsData* kmsdata; bool chunked_upload; RGWAccessControlPolicy policy; const char *obj_manifest; @@ -428,6 +430,7 @@ class RGWPutObj : public RGWOp { ofs = 0; supplied_md5_b64 = NULL; supplied_etag = NULL; + kmsdata = NULL; if_match = NULL; if_nomatch = NULL; chunked_upload = false; @@ -450,7 +453,7 @@ class RGWPutObj : public RGWOp { void execute(); virtual int get_params() = 0; - virtual int get_data(bufferlist& bl) = 0; + virtual int get_data(bufferlist& bl,MD5* hash= NULL) = 0; virtual void send_response() = 0; virtual const string name() { return "put_obj"; } virtual RGWOpType get_type() { return RGW_OP_PUT_OBJ; } @@ -494,7 +497,7 @@ class RGWPostObj : public RGWOp { void dispose_processor(RGWPutObjProcessor *processor); virtual int get_params() = 0; - virtual int get_data(bufferlist& bl) = 0; + virtual int get_data(bufferlist& bl,MD5* hash= NULL) = 0; virtual void send_response() = 0; virtual const string name() { return "post_obj"; } virtual RGWOpType get_type() { return RGW_OP_POST_OBJ; } diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 05c41ef4c33db..4a53245d62bc3 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1089,7 +1089,7 @@ int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, MD5 *hash, first_chunk.claim(bl); obj_len = (uint64_t)first_chunk.length(); if (hash) { - hash->Update((const byte *)first_chunk.c_str(), obj_len); + //hash->Update((const byte *)first_chunk.c_str(), obj_len); } int r = prepare_next_part(obj_len); if (r < 0) { @@ -1106,7 +1106,7 @@ int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, MD5 *hash, int ret = write_data(bl, write_ofs, phandle, exclusive); if (ret >= 0) { /* we might return, need to clear bl as it was already sent */ if (hash) { - hash->Update((const byte *)bl.c_str(), bl.length()); + //hash->Update((const byte *)bl.c_str(), bl.length()); } bl.clear(); } @@ -1115,7 +1115,7 @@ int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, MD5 *hash, void RGWPutObjProcessor_Atomic::complete_hash(MD5 *hash) { - hash->Update((const byte *)pending_data_bl.c_str(), pending_data_bl.length()); + //hash->Update((const byte *)pending_data_bl.c_str(), pending_data_bl.length()); } @@ -1131,7 +1131,7 @@ int RGWPutObjProcessor_Atomic::prepare_init(RGWRados *store, string *oid_rand) return 0; } -int RGWPutObjProcessor_Atomic::prepare(RGWRados *store, string *oid_rand) +int RGWPutObjProcessor_Atomic::prepare(RGWRados *store, string *oid_rand, RGWKmsData** kmsdata) { int r = prepare_init(store, oid_rand); if (r < 0) { @@ -9013,3 +9013,127 @@ librados::Rados* RGWRados::get_rados_handle() } } +int encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, + unsigned char *iv, unsigned char *ciphertext) +{ + EVP_CIPHER_CTX *ctx; + + int len; + + int ciphertext_len; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) + return -1; + /* Initialise the encryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits */ + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_xts(), NULL, key, iv)) + return -1; + + /* Provide the message to be encrypted, and obtain the encrypted output. + * EVP_EncryptUpdate can be called multiple times if necessary + */ + if(1 != EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len)) + return -1; + ciphertext_len = len; + + /* Finalise the encryption. Further ciphertext bytes may be written at + * this stage. + */ + if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + len, &len)) + return -1; + ciphertext_len += len; + + /* Clean up */ + EVP_CIPHER_CTX_free(ctx); + + return ciphertext_len; +} + +int decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char *key, + unsigned char *iv, unsigned char *plaintext) +{ + EVP_CIPHER_CTX *ctx; + + int len; + + int plaintext_len; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) + return -1; + + /* Initialise the decryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits */ + if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_xts(), NULL, key, iv)) + return -1; + + /* Provide the message to be decrypted, and obtain the plaintext output. + * EVP_DecryptUpdate can be called multiple times if necessary + */ + if(1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) + return -1; + plaintext_len = len; + + /* Finalise the decryption. Further plaintext bytes may be written at + * this stage. + */ + if(1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) + return -1; + plaintext_len += len; + + /* Clean up */ + EVP_CIPHER_CTX_free(ctx); + + return plaintext_len; +} + +int RGWKmsData::decode_json_enc(bufferlist& bl, CephContext *cct) +{ + JSONParser parser; + + if (!parser.parse(bl.c_str(), bl.length())) { + ldout(cct, 0) << "Keystone token parse error: malformed json" << dendl; + return -EINVAL; + } + try { + JSONDecoder::decode_json("KMS_RAW_DATA_IV", iv_dec, &parser, true); + JSONDecoder::decode_json("KMS_ENCRYPTED_DATA_IV", iv_enc, &parser, true); + JSONDecoder::decode_json("KMS_ENCRYPTED_DATA_KEY", key_enc, &parser, true); + JSONDecoder::decode_json("KMS_RAW_DATA_KEY", key_dec, &parser, true); + JSONDecoder::decode_json("KMS_ENCRYPTED_MK_VERSION", mkey_enc , &parser, true); + + } catch (JSONDecoder::err& err) { + ldout(cct, 0) << "KMS token parse error: " << err.message << dendl; + return -EINVAL; + } + + return 0; +} + +int RGWKmsData::decode_json_dec(bufferlist& bl, CephContext *cct) +{ + JSONParser parser; + + if (!parser.parse(bl.c_str(), bl.length())) { + ldout(cct, 0) << "KMS token parse error: malformed json" << dendl; + return -EINVAL; + } + try { + JSONDecoder::decode_json("KMS_RAW_DATA_IV", iv_dec, &parser, true); + JSONDecoder::decode_json("KMS_RAW_DATA_KEY", key_dec, &parser, true); + + } catch (JSONDecoder::err& err) { + ldout(cct, 0) << "KMS token parse error: " << err.message << dendl; + return -EINVAL; + } + return 0; +} + + diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 08a703138188c..18265dea9fea3 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -16,6 +16,10 @@ #include "rgw_log.h" #include "rgw_metadata.h" #include "rgw_rest_conn.h" +#include +#include +#include +#include class RGWWatcher; class SafeTimer; @@ -2299,6 +2303,7 @@ class RGWChainedCacheImpl : public RGWChainedCache { entries.clear(); } }; +class RGWKmsData; class RGWPutObjProcessor { @@ -2315,7 +2320,7 @@ class RGWPutObjProcessor public: RGWPutObjProcessor(RGWObjectCtx& _obj_ctx, RGWBucketInfo& _bi) : store(NULL), obj_ctx(_obj_ctx), is_complete(false), bucket_info(_bi) {} virtual ~RGWPutObjProcessor() {} - virtual int prepare(RGWRados *_store, string *oid_rand) { + virtual int prepare(RGWRados *_store, string *oid_rand, RGWKmsData** kmsdata = NULL) { store = _store; return 0; } @@ -2430,7 +2435,7 @@ class RGWPutObjProcessor_Atomic : public RGWPutObjProcessor_Aio bucket(_b), obj_str(_o), unique_tag(_t) {} - int prepare(RGWRados *store, string *oid_rand); + int prepare(RGWRados *store, string *oid_rand, RGWKmsData** kmsdata = NULL); virtual bool immutable_head() { return false; } void set_extra_data_len(uint64_t len) { extra_data_len = len; @@ -2448,4 +2453,20 @@ class RGWPutObjProcessor_Atomic : public RGWPutObjProcessor_Aio } }; +class RGWKmsData { + public: + string key_dec; + string iv_dec; + string key_enc; + string iv_enc; + string mkey_enc; + int decode_json_enc(bufferlist& bl, CephContext *cct); + int decode_json_dec(bufferlist& bl, CephContext *cct); + +}; + +int encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, unsigned char *iv, unsigned char *ciphertext); + +int decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char *key,unsigned char *iv, unsigned char *plaintext); + #endif diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 98322a069d602..c9d542172d0ae 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -550,6 +550,7 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const bool is_options_request = (s->op == OP_OPTIONS); bool is_get_request = (s->op == OP_GET); bool is_put_request = (s->op == OP_PUT); + bool can_encrypted = op && ((op->get_type() == RGW_OP_INIT_MULTIPART) || (op->get_type() == RGW_OP_PUT_OBJ)); char *allowed_origins = new char[s->cct->_conf->rgw_cors_allowed_origin.length() + 1]; strcpy(allowed_origins, s->cct->_conf->rgw_cors_allowed_origin.c_str()); const char *orig = s->info.env->get("HTTP_ORIGIN"); @@ -571,6 +572,22 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const ldout(s->cct, 0) << "WARNING: No matching allowed origin found in config, check ORIGIN header, will not send CORS headers" << dendl; } } + + if (can_encrypted) + { + const char* is_enc; + is_enc = s->info.env->get("HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION"); + + if (!is_enc) + is_enc = s->info.env->get("HTTP_X_JCS_SERVER_SIDE_ENCRYPTION"); + + if (is_enc && (strcmp(is_enc,"AES256") == 0)) + { + s->cio->print("x-jcs-server-side-encryption-customer: %s\r\n", is_enc); + } + } + + if((is_send_cors_headers) && (is_token_based_request || is_options_request)) { string allowed_methods = s->cct->_conf->rgw_cors_allowed_methods; @@ -909,7 +926,7 @@ int RGWPutObj_ObjStore::get_params() return 0; } -int RGWPutObj_ObjStore::get_data(bufferlist& bl) +int RGWPutObj_ObjStore::get_data(bufferlist& bl,MD5* hash) { size_t cl; uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size; @@ -922,15 +939,51 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl) } int len = 0; - if (cl) { - bufferptr bp(cl); - - int read_len; /* cio->read() expects int * */ - int r = s->cio->read(bp.c_str(), cl, &read_len); - len = read_len; - if (r < 0) - return r; - bl.append(bp, 0, len); + if (cl) + { + if (kmsdata && cl > 15) + { + uint64_t buffer_length = cl; + bufferptr bp(buffer_length); + + int read_len; /* cio->read() expects int * */ + int r = s->cio->read(bp.c_str(), cl, &read_len); + if (r < 0) + return r; + unsigned char* read_data = reinterpret_cast(bp.c_str()); + if (hash && read_len) + hash->Update((const byte *)bp.c_str(),read_len); + + len = read_len; + unsigned char* ciphertext = new unsigned char[read_len]; + + int ciphertext_len; + /* Encrypt the plaintext */ + const char* c_key = kmsdata->key_dec.c_str(); + const char* c_iv = kmsdata->iv_dec.c_str(); + ciphertext_len = encrypt(read_data, read_len, (unsigned char*)c_key, (unsigned char*)c_iv, ciphertext); + if (ciphertext_len == -1) + { + dout(0) << " Error while encrypting " << dendl; + return -ERR_INTERNAL_ERROR; + } + dout(0) << "SSEINFO Encryption done " << ciphertext_len << dendl; + bl.append((char*)ciphertext, len); + delete ciphertext; + } + else + { + bufferptr bp(cl); + int read_len; /* cio->read() expects int * */ + int r = s->cio->read(bp.c_str(), cl, &read_len); + if (hash && read_len) + hash->Update((const byte *)bp.c_str(),read_len); + if (r < 0) { + return r; + } + len = read_len; + bl.append(bp, 0, len); + } } if ((uint64_t)ofs + len > RGW_MAX_PUT_SIZE) { diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 532a8a182683f..ae04ab595ab06 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -151,7 +151,7 @@ class RGWPutObj_ObjStore : public RGWPutObj virtual int verify_params(); virtual int get_params(); - int get_data(bufferlist& bl); + int get_data(bufferlist& bl,MD5* hash= NULL); }; class RGWPostObj_ObjStore : public RGWPostObj diff --git a/src/rgw/rgw_rest_metadata.cc b/src/rgw/rgw_rest_metadata.cc index 1068076a7761e..995585746f686 100644 --- a/src/rgw/rgw_rest_metadata.cc +++ b/src/rgw/rgw_rest_metadata.cc @@ -104,7 +104,7 @@ void RGWOp_Metadata_List::execute() { http_ret = 0; } -int RGWOp_Metadata_Put::get_data(bufferlist& bl) { +int RGWOp_Metadata_Put::get_data(bufferlist& bl,MD5* hash) { size_t cl = 0; char *data; int read_len; diff --git a/src/rgw/rgw_rest_metadata.h b/src/rgw/rgw_rest_metadata.h index 7f3cf1f2207ae..3affb1bdb27b9 100644 --- a/src/rgw/rgw_rest_metadata.h +++ b/src/rgw/rgw_rest_metadata.h @@ -39,7 +39,7 @@ class RGWOp_Metadata_Get : public RGWRESTOp { }; class RGWOp_Metadata_Put : public RGWRESTOp { - int get_data(bufferlist& bl); + int get_data(bufferlist& bl,MD5* hash= NULL); string update_status; obj_version ondisk_version; public: diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index a0066a5c68890..989f38d4648a7 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3,6 +3,7 @@ #include #include +#include #include "common/ceph_crypto.h" #include "common/Formatter.h" @@ -72,17 +73,64 @@ static struct response_attr_param resp_attr_params[] = { {NULL, NULL}, }; + int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { const char *content_type = NULL; string content_type_str; map response_attrs; map::iterator riter; - bufferlist metadata_bl; + bufferlist metadata_bl, decrypted_bl; + if (kmsdata) + { + unsigned char* read_data; + int decryptedtext_len,left_data,iter, full_chunks ; + uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size; + unsigned char* decryptedtext = new unsigned char[chunk_size]; + if (ret) + goto done; - if (ret) - goto done; + const char* c_key = kmsdata->key_dec.c_str(); + const char* c_iv = kmsdata->iv_dec.c_str(); + + read_data = reinterpret_cast(bl.c_str()); + full_chunks = floor(((double)bl_len)/chunk_size); + dout(0) << "SSEINFO Total part number " << full_chunks << dendl; + for (iter=0; iter < full_chunks ; iter++) + { + decryptedtext_len = decrypt(read_data, chunk_size, (unsigned char*)c_key,(unsigned char*)c_iv,decryptedtext); + if (decryptedtext_len == -1) + { + dout(0) << " Error while decrypting " << dendl; + return -ERR_INTERNAL_ERROR; + } + read_data += chunk_size; + decrypted_bl.append((char*)decryptedtext, chunk_size); + dout(0) << "SSEINFO Doing for part number " << iter << dendl; + } + left_data = bl_len % chunk_size; + if (left_data > 15) + { + dout(0) << "SSEINFO Length of Encrypted text " << bl_len << " and key is " << c_key << " " << strlen(c_key) << " and iv " << c_iv << " " << strlen(c_iv) << dendl; + decryptedtext_len = decrypt(read_data, left_data, (unsigned char*)c_key,(unsigned char*)c_iv,decryptedtext); + if (decryptedtext_len == -1) + { + dout(0) << " Error while decrypting " << dendl; + return -ERR_INTERNAL_ERROR; + } + decrypted_bl.append((char*)decryptedtext, left_data); + string bufferprinter = ""; + decrypted_bl.copy(0, decrypted_bl.length(), bufferprinter); + dout(0) << "SSEINFO Decrypted text " << bufferprinter << dendl; + } + else if (left_data > 0) + { + dout(0) << "SSEINFO Not decrypting tail" << dendl; + decrypted_bl.append((char*)read_data, left_data); + } + delete decryptedtext; + } if (sent_header) goto send_data; @@ -179,7 +227,11 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ send_data: if (get_data && !ret) { - int r = s->cio->write(bl.c_str() + bl_ofs, bl_len); + int r; + if (kmsdata) + r = s->cio->write(decrypted_bl.c_str() + bl_ofs, bl_len); + else + r = s->cio->write(bl.c_str() + bl_ofs, bl_len); if (r < 0) return r; } @@ -1363,7 +1415,7 @@ int RGWPostObj_ObjStore_S3::complete_get_params() return 0; } -int RGWPostObj_ObjStore_S3::get_data(bufferlist& bl) +int RGWPostObj_ObjStore_S3::get_data(bufferlist& bl,MD5* hash) { bool boundary; bool done; @@ -3239,3 +3291,114 @@ bool RGWResourceKeystoneInfo::get_bucket_public_perm(const string& action, reason = "OK"; return true; } + +// Make a kms request for encrypted as well as decrypted key/iv +// We will use encrypted keys to store in xattr +// Decrypted keys,iv will be used to encode data +int RGW_KMS::make_kms_encrypt_request(string &root_account, RGWKmsData* kmsdata) +{ + string kms_url = cct->_conf->rgw_kms_encrypt_url; + if (kms_url[kms_url.size() -1] != '/') { + kms_url.append("?"); + } + else + kms_url[kms_url.size() -1] = '?'; + + kms_url.append("user_id="); + kms_url.append(root_account); + dout(0)<< "SSEINFO Final KMS URL " << kms_url << dendl; + int ret = 1 ; + + string empty ; + set_tx_buffer(empty); + utime_t begin_time = ceph_clock_now(g_ceph_context); + ret = process("GET", kms_url.c_str()); + utime_t end_time = ceph_clock_now(g_ceph_context); + end_time = end_time - begin_time; + dout(0) << "SSEINFO: KMS Encrypt response time (milliseconds): " << end_time.to_msec() << dendl; + + if (ret < 0) + { + ret = -ERR_INTERNAL_ERROR; + dout(0) << " Unable to obtain encryped and decrypted keys from KMS "<< dendl; + return ret; + } + + //string bufferprinter = ""; + //rx_buffer.copy(0, rx_buffer.length(), bufferprinter); + //dout(0) << "SSEINFO Printing RX buffer: " << bufferprinter << dendl; + + ret = kmsdata->decode_json_enc(rx_buffer, cct); + if (ret < 0) + { + ret = -ERR_INTERNAL_ERROR; + return ret; + } + + if (kmsdata->key_dec.size() != 64 || kmsdata->iv_dec.size() != 16) + { + dout(0) << "SSEINFO KMS Key Size " << kmsdata->key_dec.size() << " or IV Size not right" << " " << kmsdata->iv_dec.size() << dendl; + ret = -ERR_INTERNAL_ERROR; + return ret; + } + + dout(0) << " SSEINFO After parsing " << kmsdata->key_dec << " & " << kmsdata->iv_dec << dendl; + return 1; +} + + +// Make a kms call for decrypted key/iv +// We extract encrypted keys from xattr +// Decrypted keys,iv will be used to decode data +int RGW_KMS::make_kms_decrypt_request(string &root_account, RGWKmsData* kmsdata) +{ + string kms_url = cct->_conf->rgw_kms_decrypt_url; + if (kms_url[kms_url.size()] -1 != '/') { + kms_url.append("?"); + } + kms_url.append("user_id="); + kms_url.append(root_account); + kms_url.append("&encrypted_data_key="); + kms_url.append(kmsdata->key_enc); + kms_url.append("&encrypted_data_iv="); + kms_url.append(kmsdata->iv_enc); + kms_url.append("&encryptedMKVersionId="); + kms_url.append(kmsdata->mkey_enc); + kms_url.append("&randomId="); + int id = rand() % 900000 + 100000; + char* cid = new char[7]; + sprintf(cid,"%d",id); + kms_url.append(cid); + dout(0)<< "SSEINFO Final URL For Decoding KMS" << kms_url << dendl; + string empty ; + set_tx_buffer(empty); + int ret = 1; + + utime_t begin_time = ceph_clock_now(g_ceph_context); + ret = process("GET", kms_url.c_str()); + utime_t end_time = ceph_clock_now(g_ceph_context); + end_time = end_time - begin_time; + + dout(0) << "SSEINFO: KMS Decrypt response time (milliseconds): " << end_time.to_msec() << dendl; + if (ret < 0) + { + ret = -ERR_INTERNAL_ERROR; + dout(0) << " Unable to obtain encryped and decrypted keys from KMS "<< dendl; + return ret; + } + + string bufferprinter = ""; + rx_buffer.copy(0, rx_buffer.length(), bufferprinter); + dout(0) << "SSEINFO Printing RX buffer: " << bufferprinter << dendl; + + ret = kmsdata->decode_json_dec(rx_buffer, cct); + if (ret < 0) + { + ret = -ERR_INTERNAL_ERROR; + return ret; + } + + dout(0) << " SSEINFO After parsing " << kmsdata->key_dec << " & " << kmsdata->iv_dec << dendl; + + return 0; +} diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 756af24aefd94..a792764402f10 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -164,7 +164,7 @@ class RGWPostObj_ObjStore_S3 : public RGWPostObj_ObjStore { int get_params(); int complete_get_params(); void send_response(); - int get_data(bufferlist& bl); + int get_data(bufferlist& bl,MD5* hash= NULL); }; class RGWDeleteObj_ObjStore_S3 : public RGWDeleteObj_ObjStore { @@ -586,6 +586,56 @@ class RGWResourceKeystoneInfo { string& reason); }; +class RGW_KMS: public RGWHTTPClient { +private: + bufferlist rx_buffer; + bufferlist rx_headers_buffer; + bufferlist tx_buffer; + bufferlist::iterator tx_buffer_it; + +private: + void set_tx_buffer(const string& d) { + tx_buffer.clear(); + tx_buffer.append(d); + tx_buffer_it = tx_buffer.begin(); + set_send_length(tx_buffer.length()); + } + +public: + RGW_KMS(CephContext *_cct) + : RGWHTTPClient(_cct) { + } + + int receive_header(void *ptr, size_t len) { + rx_headers_buffer.append((char *)ptr, len); + return 0; + } + int receive_data(void *ptr, size_t len) { + rx_buffer.append((char *)ptr, len); + return 0; + } + + int send_data(void *ptr, size_t len) { + if (!tx_buffer_it.get_remaining()) + return 0; // nothing left to send + + int l = MIN(tx_buffer_it.get_remaining(), len); + memcpy(ptr, tx_buffer_it.get_current_ptr().c_str(), l); + try { + tx_buffer_it.advance(l); + } catch (buffer::end_of_buffer &e) { + assert(0); + } + + return l; + } + + int make_kms_encrypt_request(string &root_acount, RGWKmsData* kmsdata); + int make_kms_decrypt_request(string &root_acount, RGWKmsData* kmsdata);; + +}; + + class dss_endpoint { public: static string endpoint; From 4abdc76f5a6a4275a78f8fe053e58600ab93c0b3 Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Sat, 1 Oct 2016 22:45:57 +1100 Subject: [PATCH 53/60] Adding allowed headers for console sse --- src/common/config_opts.h | 2 +- src/rgw/rgw_rest.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 8c8059ddb1250..ed1f7a2f91b04 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1065,7 +1065,7 @@ OPTION(rgw_user_max_buckets, OPT_U32, 1000) // global option to set max buckets OPTION(rgw_enable_cors_response_headers, OPT_BOOL, true) // send cors response headers in case of a token based request OPTION(rgw_cors_allowed_origin, OPT_STR, "https://console.jiocloudservices.com, http://console.jiocloudservices.com, http://consolepreprod.jiocloudservices.com, https://consolepreprod.jiocloudservices.com, http://console.staging.jiocloudservices.com, https://console.staging.jiocloudservices.com")// cors allowed domains OPTION(rgw_cors_allowed_methods, OPT_STR, "GET, PUT, HEAD, POST, DELETE, COPY, OPTIONS") // cors allowed methods -OPTION(rgw_cors_allowed_headers, OPT_STR, "X-Auth-Token, Content-Disposition, Content-Type") // cors allowed headers +OPTION(rgw_cors_allowed_headers, OPT_STR, "X-Auth-Token, Content-Disposition, Content-Type, X-Jcs-Server-Side-Encryption") // cors allowed headers OPTION(rgw_cors_exposed_headers, OPT_STR, "ETag") // cors explicitely exposed headers OPTION(rgw_cors_content_disposition_header, OPT_STR, "Content-Disposition") // cors content disposition HEADER OPTION(rgw_cors_content_disposition_header_value, OPT_STR, "attachment") // cors content disposition HEADER value diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index c9d542172d0ae..8f263c23da52c 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -952,7 +952,7 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl,MD5* hash) return r; unsigned char* read_data = reinterpret_cast(bp.c_str()); if (hash && read_len) - hash->Update((const byte *)bp.c_str(),read_len); + hash->Update((const byte *)bp.c_str(),bp.length()); len = read_len; unsigned char* ciphertext = new unsigned char[read_len]; @@ -977,7 +977,7 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl,MD5* hash) int read_len; /* cio->read() expects int * */ int r = s->cio->read(bp.c_str(), cl, &read_len); if (hash && read_len) - hash->Update((const byte *)bp.c_str(),read_len); + hash->Update((const byte *)bp.c_str(),bp.length()); if (r < 0) { return r; } From afd06bb7b2ec95fa2ca9abb3df629218373b2156 Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Fri, 7 Oct 2016 22:48:32 +1100 Subject: [PATCH 54/60] Fixing memory leaks and adding debug log messages --- src/rgw/rgw_rest.cc | 3 ++- src/rgw/rgw_rest_s3.cc | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 8f263c23da52c..afc149f11c6ec 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -965,11 +965,12 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl,MD5* hash) if (ciphertext_len == -1) { dout(0) << " Error while encrypting " << dendl; + delete [] ciphertext; return -ERR_INTERNAL_ERROR; } dout(0) << "SSEINFO Encryption done " << ciphertext_len << dendl; bl.append((char*)ciphertext, len); - delete ciphertext; + delete [] ciphertext; } else { diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 989f38d4648a7..77bd5f935cd09 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -81,14 +81,16 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ map response_attrs; map::iterator riter; bufferlist metadata_bl, decrypted_bl; + if (ret) + { + dout(0) << "SSEDebug Value of ret at entrance " << ret << " "<< bl.length() << dendl; + } if (kmsdata) { unsigned char* read_data; int decryptedtext_len,left_data,iter, full_chunks ; uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size; unsigned char* decryptedtext = new unsigned char[chunk_size]; - if (ret) - goto done; const char* c_key = kmsdata->key_dec.c_str(); const char* c_iv = kmsdata->iv_dec.c_str(); @@ -117,20 +119,24 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ if (decryptedtext_len == -1) { dout(0) << " Error while decrypting " << dendl; + delete [] decryptedtext; return -ERR_INTERNAL_ERROR; } decrypted_bl.append((char*)decryptedtext, left_data); string bufferprinter = ""; decrypted_bl.copy(0, decrypted_bl.length(), bufferprinter); - dout(0) << "SSEINFO Decrypted text " << bufferprinter << dendl; + //dout(0) << "SSEINFO Decrypted text " << bufferprinter << dendl; } else if (left_data > 0) { dout(0) << "SSEINFO Not decrypting tail" << dendl; decrypted_bl.append((char*)read_data, left_data); } - delete decryptedtext; + delete [] decryptedtext; } + dout(0) << "SSEDebug done with enc dec" << dendl; + if (ret) + goto done; if (sent_header) goto send_data; @@ -233,7 +239,10 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ else r = s->cio->write(bl.c_str() + bl_ofs, bl_len); if (r < 0) + { + dout(0) << "SSEDebug failing while writing to io" << dendl; return r; + } } return 0; From fd1e8ee120177adcee52f486898df7769790338c Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Tue, 11 Oct 2016 07:17:10 +1100 Subject: [PATCH 55/60] Fixing an issue where 'x-jcs-*' headers are modified and all 'x-*' prefix in x-jcs/x-amz type headers are replaced with 'x-amz'. Because of this x-jcs headers are not passed as it is and causes an issue in creating cannonical string for authentication with IAM. Fix: 1. Added x-jcs in the mix with other x-* type headers so that it wont be ignored. 2. We are retaining original prefix and not replacing it with x-amz. --- src/rgw/rgw_common.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index ea40130501630..b00a8b40701a6 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -193,6 +193,7 @@ struct str_len { #define STR_LEN_ENTRY(s) { s, sizeof(s) - 1 } struct str_len meta_prefixes[] = { STR_LEN_ENTRY("HTTP_X_AMZ"), + STR_LEN_ENTRY("HTTP_X_JCS"), STR_LEN_ENTRY("HTTP_X_GOOG"), STR_LEN_ENTRY("HTTP_X_DHO"), STR_LEN_ENTRY("HTTP_X_RGW"), @@ -223,7 +224,7 @@ void req_info::init_meta_info(bool *found_bad_meta) *found_bad_meta = true; char name_low[meta_prefixes[0].len + name_len + 1]; - snprintf(name_low, meta_prefixes[0].len - 5 + name_len + 1, "%s%s", meta_prefixes[0].str + 5 /* skip HTTP_ */, name); // normalize meta prefix + snprintf(name_low, meta_prefixes[prefix_num].len - 5 + name_len + 1, "%s%s", meta_prefixes[prefix_num].str + 5 /* skip HTTP_ */, name); // normalize meta prefix but retain the original 'x-*' prefix. We need both amz and jcs type headers for now. int j; for (j = 0; name_low[j]; j++) { if (name_low[j] != '_') From 0a8bc3c555beefeb4b465728aa3021de2a8cfc0d Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Wed, 12 Oct 2016 21:55:40 +1100 Subject: [PATCH 56/60] Logging Source IP of user and sending it to IAM --- src/rgw/rgw_rest_s3.cc | 15 +++++++++++---- src/rgw/rgw_rest_s3.h | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 989f38d4648a7..fd897393ad86c 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1244,10 +1244,13 @@ int RGWPostObj_ObjStore_S3::get_policy() } dout(1) << "DSS API LOGGING: Action="<< resource_info.getAction() <<" Resource="<< resource_info.getResourceName() << " Tenant=" << resource_info.getTenantName() << dendl; + string source_ip = s->info.env->get("HTTP_X_FORWARDED_FOR","0.0.0.0"); + keystone_validator.append_header("X-Forwarded-For",source_ip); if (isTokenBasedAuth) { keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), resource_info.getTenantName(), + source_ip, false, /* Is sign auth */ false, /* Is copy */ false, /* Is cross account */ @@ -1265,6 +1268,7 @@ int RGWPostObj_ObjStore_S3::get_policy() keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), resource_info.getTenantName(), + source_ip, true, /* Is sign auth */ false, /* Is copy */ false, /* Is cross account */ @@ -2436,6 +2440,7 @@ int RGWHandler_ObjStore_S3::init(RGWRados *store, struct req_state *s, RGWClient int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, const string& resource_name, const string& tenant_name, + const string& source_ip, const bool& is_sign_auth, const bool& is_copy, const bool& is_cross_account, @@ -2470,7 +2475,6 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, << localAction << dendl; return -ENOTRECOVERABLE; } - /* Set required headers for keystone request * Recursive calls already have headers set */ if (!is_copy && !is_cross_account) { @@ -2486,6 +2490,7 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, append_header("Content-Type", "application/json"); } + append_header("X-Forwarded-For",source_ip); /* Handle special case of copy */ bool isCopyAction = false; isCopyAction = (localAction.compare("CopyObject") == 0); @@ -2497,7 +2502,7 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, string copy_src_str = copy_src.substr(0, pos); string copy_src_tenant = copy_src.substr(pos + 1); dout(0) << "DSS INFO: Validating for copy source" << dendl; - ret = validate_request(localAction, copy_src_str, copy_src_tenant, is_sign_auth, + ret = validate_request(localAction, copy_src_str, copy_src_tenant, source_ip, is_sign_auth, true, is_cross_account, is_url_token, is_infini_url_token, copy_src, token, auth_id, auth_token, auth_sign, objectname, iamerror); if (ret < 0) { @@ -2641,7 +2646,7 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, // This case requires cross account validation. // Make recursive call with is_cross_account set to true dout(0) << "DSS INFO: Validating for cross account access" << dendl; - ret = validate_request(localAction, resource_name, tenant_name, + ret = validate_request(localAction, resource_name, tenant_name, source_ip, is_sign_auth, is_copy, true, is_url_token, is_infini_url_token, copy_src, token, auth_id, auth_token, auth_sign, objectname, iamerror); @@ -2883,16 +2888,17 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) if (s != NULL) { resource_object_name = s->object.name; } + string source_ip = s->info.env->get("HTTP_X_FORWARDED_FOR","0.0.0.0"); dout(1) << "DSS API LOGGING: Action=" << resource_info.getAction() << " Resource="<< resource_info.getResourceName() << " Tenant=" << resource_info.getTenantName() << " Object=" << resource_object_name << dendl; - if (isTokenBasedAuth) { keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), resource_info.getTenantName(), + source_ip, false, /* Is sign auth */ false, /* Is copy */ false, /* Is cross account */ @@ -2910,6 +2916,7 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), resource_info.getTenantName(), + source_ip, true, /* Is sign auth */ false, /* Is copy */ false, /* Is cross account */ diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index a792764402f10..c2b8070b7c629 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -352,6 +352,7 @@ class RGW_Auth_S3_Keystone_ValidateToken : public RGWHTTPClient { int validate_request(const string& action, const string& resource_name, const string& tenant_name, + const string& source_ip, const bool& is_sign_auth, const bool& is_copy, const bool& is_cross_account, From b395c16e8a6b0ff62690878403c9ca79367c6a18 Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Sat, 22 Oct 2016 01:01:15 +1100 Subject: [PATCH 57/60] Adding a header for encrypted objects Get/Head Ops --- src/rgw/rgw_rest_s3.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index e84bd84487e6a..5d48dc165b2dd 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -177,6 +177,9 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ } } + if (kmsdata) + s->cio->print("x-jcs-server-side-encryption: \"%s\"\r\n", "AES256"); + for (struct response_attr_param *p = resp_attr_params; p->param; p++) { bool exists; string val = s->info.args.get(p->param, &exists); From 32037428e6c3152c89c65ceff18a4da190c06187 Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Wed, 26 Oct 2016 21:17:16 +1100 Subject: [PATCH 58/60] Adding x-jcs-server-side-encryption as a special case header which will be used in signature. --- src/rgw/rgw_common.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index b00a8b40701a6..55e532a64dc13 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -193,7 +193,6 @@ struct str_len { #define STR_LEN_ENTRY(s) { s, sizeof(s) - 1 } struct str_len meta_prefixes[] = { STR_LEN_ENTRY("HTTP_X_AMZ"), - STR_LEN_ENTRY("HTTP_X_JCS"), STR_LEN_ENTRY("HTTP_X_GOOG"), STR_LEN_ENTRY("HTTP_X_DHO"), STR_LEN_ENTRY("HTTP_X_RGW"), @@ -224,7 +223,7 @@ void req_info::init_meta_info(bool *found_bad_meta) *found_bad_meta = true; char name_low[meta_prefixes[0].len + name_len + 1]; - snprintf(name_low, meta_prefixes[prefix_num].len - 5 + name_len + 1, "%s%s", meta_prefixes[prefix_num].str + 5 /* skip HTTP_ */, name); // normalize meta prefix but retain the original 'x-*' prefix. We need both amz and jcs type headers for now. + snprintf(name_low, meta_prefixes[0].len - 5 + name_len + 1, "%s%s", meta_prefixes[0].str + 5 /* skip HTTP_ */, name); // normalize meta prefix int j; for (j = 0; name_low[j]; j++) { if (name_low[j] != '_') @@ -247,6 +246,9 @@ void req_info::init_meta_info(bool *found_bad_meta) x_meta_map[name_low] = val; } } + if (strncmp(header_name.c_str(), "HTTP_X_JCS_SERVER_SIDE_ENCRYPTION", strlen(header_name.c_str())) == 0) { + x_meta_map["x-jcs-server-side-encryption"] = "AES256"; + } } } for (iter = x_meta_map.begin(); iter != x_meta_map.end(); ++iter) { From 9d2101c5d44c49b1757d2810b4b6484601c12b0c Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Thu, 27 Oct 2016 01:12:47 +1100 Subject: [PATCH 59/60] adding error code for wrong encryption header --- src/rgw/rgw_common.cc | 2 +- src/rgw/rgw_http_errors.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 55e532a64dc13..95c35ac26bf2b 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -247,7 +247,7 @@ void req_info::init_meta_info(bool *found_bad_meta) } } if (strncmp(header_name.c_str(), "HTTP_X_JCS_SERVER_SIDE_ENCRYPTION", strlen(header_name.c_str())) == 0) { - x_meta_map["x-jcs-server-side-encryption"] = "AES256"; + x_meta_map["x-jcs-server-side-encryption"] = val; } } } diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index efa431e78d04f..0cc154643144e 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -65,6 +65,7 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_RENAME_COPY_FAILED, 500, "RenameFailed", "Object copy failed during rename. Please file a bug." }, { ERR_RENAME_DATA_LOST, 500, "DataLost", "Rename operation lost the original data. Please file a bug." }, { ERR_RENAME_NEW_OBJ_DEL_FAILED, 500, "RenameFailed", "Rename operation failed. Please delete the duplicated object with name same as new name for the object, manually. Please file a bug." }, + {ERR_INVALID_ENC_ALGO, 400, "InvalidEncryptionAlgorithmError", "The encryption request you specified is not valid. The valid value is AES256." }, }; const static struct rgw_http_errors RGW_HTTP_SWIFT_ERRORS[] = { From ef1fed4d4ba2952119a0ee5f88828606e5b28006 Mon Sep 17 00:00:00 2001 From: Harshal Gupta Date: Thu, 29 Dec 2016 19:30:30 +1100 Subject: [PATCH 60/60] Overriding denial of permission by ACL --- src/rgw/rgw_acl.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc index 669a83d10ea88..ece88c15b480f 100644 --- a/src/rgw/rgw_acl.cc +++ b/src/rgw/rgw_acl.cc @@ -114,7 +114,8 @@ bool RGWAccessControlPolicy::verify_permission(string& uid, int user_perm_mask, ldout(cct, 10) << " uid=" << uid << " requested perm (type)=" << perm << ", policy perm=" << policy_perm << ", user_perm_mask=" << user_perm_mask << ", acl perm=" << acl_perm << dendl; - return (perm == acl_perm); + return true; + //return (perm == acl_perm); }