diff --git a/configure.ac b/configure.ac index 969baed8357db..8c7a65a6d431b 100644 --- a/configure.ac +++ b/configure.ac @@ -315,6 +315,13 @@ AC_SUBST(KEYUTILS_LIB) AC_CHECK_LIB([m], [pow], [true], AC_MSG_FAILURE([libm not found])) AC_CHECK_FUNCS([syncfs], AC_DEFINE([HAVE_SYS_SYNCFS], [1], [we have syncfs]), []) + +# since we are now using openssl and need to provide the additional functions +# for thread safety, we also need to link to crypto library (-lcrypto) +AC_CHECK_LIB(crypto, main, [], + AC_MSG_FAILURE(["Crypto system library not found."])) + + # Find some crypto library for us to use, while letting user to decide which one to use. AC_ARG_WITH([cryptopp], [AS_HELP_STRING([--with-cryptopp], [Use cryptographic functions from cryptopp])], @@ -872,6 +879,7 @@ AC_CHECK_LIB(boost_system-mt, main, [], [AC_CHECK_LIB(boost_system, main, [], AC_MSG_NOTICE(["Boost system library not found."]))]) + # Find the right boost_thread library. BOOST_THREAD_LIBS="" saved_LIBS="${LIBS}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 45b646e8da825..43da5e082d945 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -214,6 +214,7 @@ set(libcommon_files common/simple_spin.cc common/Thread.cc common/Formatter.cc + common/HTMLFormatter.cc common/HeartbeatMap.cc common/ceph_fs.cc common/ceph_hash.cc @@ -732,7 +733,9 @@ if(${WITH_RADOSGW}) rgw/rgw_replica_log.cc rgw/rgw_keystone.cc rgw/rgw_quota.cc - rgw/rgw_dencoder.cc) + rgw/rgw_dencoder.cc + rgw/rgw_website.cc + rgw/rgw_xml_enc.cc) add_library(rgw_a STATIC ${rgw_a_srcs}) diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index 7c166ef09865f..eec5943a6733e 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -18,6 +18,7 @@ #include "assert.h" #include "Formatter.h" +#include "HTMLFormatter.h" #include "common/escape.h" #include @@ -83,6 +84,10 @@ Formatter *Formatter::create(const std::string &type, return new TableFormatter(); else if (mytype == "table-kv") return new TableFormatter(true); + else if (mytype == "html") + return new HTMLFormatter(false); + else if (mytype == "html-pretty") + return new HTMLFormatter(true); else if (fallback != "") return create(fallback, "", ""); else @@ -126,8 +131,6 @@ void JSONFormatter::flush(std::ostream& os) { finish_pending_string(); os << m_ss.str(); - if (m_pretty) - os << "\n"; m_ss.clear(); m_ss.str(""); } @@ -317,8 +320,11 @@ XMLFormatter::XMLFormatter(bool pretty) void XMLFormatter::flush(std::ostream& os) { finish_pending_string(); - os << m_ss.str(); - if (m_pretty) + std::string m_ss_str = m_ss.str(); + os << m_ss_str; + /* There is a small catch here. If the rest of the formatter had NO output, + * we should NOT output a newline. This primarily triggers on HTTP redirects */ + if (m_pretty && !m_ss_str.empty()) os << "\n"; m_ss.clear(); m_ss.str(""); @@ -332,6 +338,24 @@ void XMLFormatter::reset() m_pending_string.str(""); m_sections.clear(); m_pending_string_name.clear(); + m_header_done = false; +} + +void XMLFormatter::output_header() +{ + if(!m_header_done) { + m_header_done = true; + write_raw_data(XMLFormatter::XML_1_DTD);; + if (m_pretty) + m_ss << "\n"; + } +} + +void XMLFormatter::output_footer() +{ + while(!m_sections.empty()) { + close_section(); + } } void XMLFormatter::open_object_section(const char *name) diff --git a/src/common/Formatter.h b/src/common/Formatter.h index c61138dac3672..d8eef64b730cd 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -50,6 +50,10 @@ namespace ceph { } virtual void reset() = 0; + virtual void set_status(const char* status, const char* status_name) = 0; + virtual void output_header() = 0; + virtual void output_footer() = 0; + virtual void open_array_section(const char *name) = 0; virtual void open_array_section_in_ns(const char *name, const char *ns) = 0; virtual void open_object_section(const char *name) = 0; @@ -88,7 +92,9 @@ namespace ceph { class JSONFormatter : public Formatter { public: JSONFormatter(bool p = false); - + virtual void set_status(const char* status, const char* status_name) {}; + virtual void output_header() {}; + virtual void output_footer() {}; void flush(std::ostream& os); void reset(); virtual void open_array_section(const char *name); @@ -130,6 +136,10 @@ namespace ceph { static const char *XML_1_DTD; XMLFormatter(bool pretty = false); + virtual void set_status(const char* status, const char* status_name) {}; + virtual void output_header(); + virtual void output_footer(); + void flush(std::ostream& os); void reset(); void open_array_section(const char *name); @@ -150,7 +160,7 @@ namespace ceph { void open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs); void open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs); void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs); - private: + protected: void open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs); void finish_pending_string(); void print_spaces(); @@ -161,12 +171,16 @@ namespace ceph { std::deque m_sections; bool m_pretty; std::string m_pending_string_name; + bool m_header_done; }; class TableFormatter : public Formatter { public: TableFormatter(bool keyval = false); - + + virtual void set_status(const char* status, const char* status_name) {}; + virtual void output_header() {}; + virtual void output_footer() {}; void flush(std::ostream& os); void reset(); virtual void open_array_section(const char *name); diff --git a/src/common/HTMLFormatter.cc b/src/common/HTMLFormatter.cc new file mode 100644 index 0000000000000..60d7e5d8415b9 --- /dev/null +++ b/src/common/HTMLFormatter.cc @@ -0,0 +1,165 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#define LARGE_SIZE 1024 + +#include "include/int_types.h" + +#include "assert.h" +#include "Formatter.h" +#include "HTMLFormatter.h" +#include "common/escape.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ----------------------- +namespace ceph { + +HTMLFormatter::HTMLFormatter(bool pretty) +: XMLFormatter(pretty), m_header_done(false), m_status(NULL), m_status_name(NULL) +{ +} + +HTMLFormatter::~HTMLFormatter() +{ + if (m_status) { + free((void*)m_status); + m_status = NULL; + } + if (m_status_name) { + free((void*)m_status_name); + m_status_name = NULL; + } +} + +void HTMLFormatter::reset() +{ + XMLFormatter::reset(); + m_header_done = false; + if (m_status) { + free((void*)m_status); + m_status = NULL; + } + if (m_status_name) { + free((void*)m_status_name); + m_status_name = NULL; + } +} + +void HTMLFormatter::set_status(const char* status, const char* status_name) +{ + assert(status != NULL); // new status must not be NULL + assert(m_status == NULL); // status should NOT be set multiple times + m_status = strdup(status); + if (status_name) + m_status_name = strdup(status_name); +}; + +void HTMLFormatter::output_header() { + if (!m_header_done) { + m_header_done = true; + assert(m_status != NULL); // it should be set by this point + std::string status_line(m_status); + if (m_status_name) { + status_line += " "; + status_line += m_status_name; + } + open_object_section("html"); + print_spaces(); + m_ss << "" << status_line << ""; + if (m_pretty) + m_ss << "\n"; + open_object_section("body"); + print_spaces(); + m_ss << "

" << status_line << "

"; + if (m_pretty) + m_ss << "\n"; + open_object_section("ul"); + } +} + +template +void HTMLFormatter::dump_template(const char *name, T arg) +{ + print_spaces(); + m_ss << "
  • " << name << ": " << arg << "
  • "; + if (m_pretty) + m_ss << "\n"; +} + +void HTMLFormatter::dump_unsigned(const char *name, uint64_t u) +{ + dump_template(name, u); +} + +void HTMLFormatter::dump_int(const char *name, int64_t u) +{ + dump_template(name, u); +} + +void HTMLFormatter::dump_float(const char *name, double d) +{ + dump_template(name, d); +} + +void HTMLFormatter::dump_string(const char *name, const std::string& s) +{ + dump_template(name, escape_xml_str(s.c_str())); +} + +void HTMLFormatter::dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) +{ + std::string e(name); + std::string attrs_str; + get_attrs_str(&attrs, attrs_str); + print_spaces(); + m_ss << "
  • " << e << ": " << escape_xml_str(s.c_str()) << attrs_str << "
  • "; + if (m_pretty) + m_ss << "\n"; +} + +std::ostream& HTMLFormatter::dump_stream(const char *name) +{ + print_spaces(); + m_pending_string_name = "li"; + m_ss << "
  • " << name << ": "; + return m_pending_string; +} + +void HTMLFormatter::dump_format_va(const char* name, const char *ns, bool quoted, const char *fmt, va_list ap) +{ + char buf[LARGE_SIZE]; + vsnprintf(buf, LARGE_SIZE, fmt, ap); + + std::string e(name); + print_spaces(); + if (ns) { + m_ss << "
  • " << e << ": " << escape_xml_str(buf) << "
  • "; + } else { + m_ss << "
  • " << e << ": " << escape_xml_str(buf) << "
  • "; + } + + if (m_pretty) + m_ss << "\n"; +} + +} // namespace ceph diff --git a/src/common/HTMLFormatter.h b/src/common/HTMLFormatter.h new file mode 100644 index 0000000000000..de154c7b0d188 --- /dev/null +++ b/src/common/HTMLFormatter.h @@ -0,0 +1,50 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#ifndef CEPH_HTML_FORMATTER_H +#define CEPH_HTML_FORMATTER_H + +#include "include/int_types.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "include/buffer.h" +#include "Formatter.h" + +namespace ceph { + class HTMLFormatter : public XMLFormatter { + public: + HTMLFormatter(bool pretty = false); + ~HTMLFormatter(); + void reset(); + + virtual void set_status(const char* status, const char* status_name); + virtual void output_header(); + + void dump_unsigned(const char *name, uint64_t u); + void dump_int(const char *name, int64_t u); + void dump_float(const char *name, double d); + void dump_string(const char *name, const std::string& s); + std::ostream& dump_stream(const char *name); + void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap); + + /* with attrs */ + void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs); + private: + template void dump_template(const char *name, T arg); + + bool m_header_done; + + const char* m_status; + const char* m_status_name; + }; + +} + +#endif diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 620e550003591..81daa4ab18e78 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -45,6 +45,7 @@ libcommon_internal_la_SOURCES = \ common/simple_spin.cc \ common/Thread.cc \ common/Formatter.cc \ + common/HTMLFormatter.cc \ common/HeartbeatMap.cc \ common/config.cc \ common/utf8.c \ @@ -178,6 +179,7 @@ noinst_HEADERS += \ common/DecayCounter.h \ common/Finisher.h \ common/Formatter.h \ + common/HTMLFormatter.h \ common/perf_counters.h \ common/OutputDataSocket.h \ common/admin_socket.h \ diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 0e888b0c3b9c6..549d7d8e432d4 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -948,13 +948,14 @@ OPTION(rgw_enable_quota_threads, OPT_BOOL, true) OPTION(rgw_enable_gc_threads, OPT_BOOL, true) OPTION(rgw_data, OPT_STR, "/var/lib/ceph/radosgw/$cluster-$id") -OPTION(rgw_enable_apis, OPT_STR, "s3, admin") //<<<<<< DSS stopped swift +OPTION(rgw_enable_apis, OPT_STR, "s3, s3website, admin") OPTION(rgw_cache_enabled, OPT_BOOL, true) // rgw cache enabled OPTION(rgw_cache_lru_size, OPT_INT, 10000) // num of entries in rgw cache OPTION(rgw_socket_path, OPT_STR, "") // path to unix domain socket, if not specified, rgw will not run as external fcgi OPTION(rgw_host, OPT_STR, "") // host for radosgw, can be an IP, default is 0.0.0.0 OPTION(rgw_port, OPT_STR, "") // port to listen, format as "8080" "5000", if not specified, rgw will not run external fcgi -OPTION(rgw_dns_name, OPT_STR, "") +OPTION(rgw_dns_name, OPT_STR, "") // hostname suffix on buckets +OPTION(rgw_dns_s3website_name, OPT_STR, "") // hostname suffix on buckets for s3-website endpoint OPTION(rgw_content_length_compat, OPT_BOOL, false) // Check both HTTP_CONTENT_LENGTH and CONTENT_LENGTH in fcgi env OPTION(rgw_script_uri, OPT_STR, "") // alternative value for SCRIPT_URI if not set in request OPTION(rgw_request_uri, OPT_STR, "") // alternative value for REQUEST_URI if not set in request @@ -965,6 +966,8 @@ OPTION(rgw_swift_auth_entry, OPT_STR, "auth") // entry point for which a url is OPTION(rgw_swift_tenant_name, OPT_STR, "") // tenant name to use for swift access OPTION(rgw_swift_enforce_content_length, OPT_BOOL, false) // enforce generation of Content-Length even in cost of performance or scalability OPTION(rgw_keystone_url, OPT_STR, "") // url for keystone server +OPTION(rgw_kms_encrypt_url, OPT_STR, "") // url for kms encrypt +OPTION(rgw_kms_decrypt_url, OPT_STR, "") // url for kms decrypt OPTION(rgw_keystone_admin_token, OPT_STR, "") // keystone admin token (shared secret) OPTION(rgw_keystone_admin_user, OPT_STR, "") // keystone admin user name OPTION(rgw_keystone_admin_password, OPT_STR, "") // keystone admin user password @@ -1063,16 +1066,23 @@ OPTION(rgw_user_max_buckets, OPT_U32, 1000) // global option to set max buckets OPTION(rgw_enable_cors_response_headers, OPT_BOOL, true) // send cors response headers in case of a token based request OPTION(rgw_cors_allowed_origin, OPT_STR, "https://console.jiocloudservices.com, http://console.jiocloudservices.com, http://consolepreprod.jiocloudservices.com, https://consolepreprod.jiocloudservices.com, http://console.staging.jiocloudservices.com, https://console.staging.jiocloudservices.com")// cors allowed domains OPTION(rgw_cors_allowed_methods, OPT_STR, "GET, PUT, HEAD, POST, DELETE, COPY, OPTIONS") // cors allowed methods -OPTION(rgw_cors_allowed_headers, OPT_STR, "X-Auth-Token, Content-Disposition, Content-Type") // cors allowed headers +OPTION(rgw_cors_allowed_headers, OPT_STR, "X-Auth-Token, Content-Disposition, Content-Type, X-Jcs-Server-Side-Encryption") // cors allowed headers +OPTION(rgw_cors_exposed_headers, OPT_STR, "ETag") // cors explicitely exposed headers OPTION(rgw_cors_content_disposition_header, OPT_STR, "Content-Disposition") // cors content disposition HEADER OPTION(rgw_cors_content_disposition_header_value, OPT_STR, "attachment") // cors content disposition HEADER value OPTION(rgw_enable_token_based_presigned_url, OPT_BOOL, true) // enable token based presigned url +OPTION(rgw_enable_infinite_token_based_presigned_url, OPT_BOOL, true) // enable infinite token based presigned url + OPTION(rgw_disable_acl_api, OPT_BOOL, true) // disable all acl getters and setters OPTION(rgw_keystone_sign_api, OPT_STR, "v3/sign-auth") // api to validate signature based authentication requests OPTION(rgw_keystone_token_api, OPT_STR, "v3/token-auth") // api to validate token based authentication requests OPTION(rgw_keystone_url_token_api, OPT_STR, "url-auth") // api to validate presigned token URL based authentication requests +OPTION(rgw_keystone_infinite_url_token_api, OPT_STR, "preauth-token-auth") // api to validate infinite time presigned token URL OPTION(dss_regional_url, OPT_STR, "https://dss.ind-west-1.staging.jiocloudservices.com") // URL to be returned in XMLNS during anonymous list all buckets calls +OPTION(rgw_enable_rename_op, OPT_BOOL, true) // Enable the atomic rename op + +OPTION(rgw_enable_static_website, OPT_BOOL, false) // enable static website feature OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf counter diff --git a/src/global/Makefile.am b/src/global/Makefile.am index 79a7ffff689cd..00a7e373cad29 100644 --- a/src/global/Makefile.am +++ b/src/global/Makefile.am @@ -4,6 +4,9 @@ libglobal_la_SOURCES = \ global/pidfile.cc \ global/signal_handler.cc libglobal_la_LIBADD = $(LIBCOMMON) +if WITH_LTTNG +libglobal_la_LIBADD += -ldl -llttng-ust +endif noinst_LTLIBRARIES += libglobal.la noinst_HEADERS += \ diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc index 69d6be05a4953..8481a7017dd20 100644 --- a/src/mds/MDS.cc +++ b/src/mds/MDS.cc @@ -23,6 +23,7 @@ #include "common/signal.h" #include "common/ceph_argparse.h" #include "common/errno.h" +#include "common/Formatter.h" #include "msg/Messenger.h" #include "mon/MonClient.h" diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 1470f81218e85..e0c5b3d6b8b96 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -6561,6 +6561,7 @@ bool OSD::advance_pg( continue; } + dout(0) << __func__ << " " << *pg << " DSS Info: Calculating new acting" << dendl; vector newup, newacting; int up_primary, acting_primary; nextmap->pg_to_up_acting_osds( diff --git a/src/osd/PG.cc b/src/osd/PG.cc index df5b34d3f4b5d..23904a8a0086a 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -612,7 +612,7 @@ bool PG::needs_recovery() const } if (!ret) - dout(10) << __func__ << " is recovered" << dendl; + dout(0) << __func__ << " DSS Info is recovered" << dendl; return ret; } @@ -2011,7 +2011,7 @@ unsigned PG::get_backfill_priority() void PG::finish_recovery(list& tfin) { - dout(10) << "finish_recovery" << dendl; + dout(0) << "DSS Info finish_recovery" << dendl; assert(info.last_complete == info.last_update); clear_recovery_state(); @@ -2031,7 +2031,7 @@ void PG::_finish_recovery(Context *c) return; } if (c == finish_sync_event) { - dout(10) << "_finish_recovery" << dendl; + dout(0) << "DSS Info _finish_recovery" << dendl; finish_sync_event = 0; purge_strays(); @@ -2188,7 +2188,7 @@ void PG::split_into(pg_t child_pgid, PG *child, unsigned split_bits) void PG::clear_recovery_state() { - dout(10) << "clear_recovery_state" << dendl; + dout(0) << "DSS Info clear_recovery_state" << dendl; pg_log.reset_recovery_pointers(); finish_sync_event = 0; @@ -5293,7 +5293,7 @@ void PG::handle_advance_map( { assert(lastmap->get_epoch() == osdmap_ref->get_epoch()); assert(lastmap == osdmap_ref); - dout(10) << "handle_advance_map " + dout(0) << "DSS Info handle_advance_map " << newup << "/" << newacting << " -- " << up_primary << "/" << acting_primary << dendl; @@ -7489,7 +7489,7 @@ void PG::RecoveryState::WaitUpThru::exit() void PG::RecoveryState::RecoveryMachine::log_enter(const char *state_name) { - dout(5) << "enter " << state_name << dendl; + dout(0) << "DSS Info enter " << state_name << dendl; pg->osd->pg_recovery_stats.log_enter(state_name); } diff --git a/src/rbd_replay/actions.hpp b/src/rbd_replay/actions.hpp index ea46a883362e6..9a05285ca4f26 100644 --- a/src/rbd_replay/actions.hpp +++ b/src/rbd_replay/actions.hpp @@ -141,7 +141,7 @@ class TypedAction : public Action { virtual std::ostream& dump(std::ostream& o) const { o << get_action_name() << ": "; - ceph::JSONFormatter formatter(false); + JSONFormatter formatter(false); formatter.open_object_section(""); m_action.dump(&formatter); formatter.close_section(); diff --git a/src/rgw/Makefile.am b/src/rgw/Makefile.am index 7620d73b053d1..a4004d6681d9e 100644 --- a/src/rgw/Makefile.am +++ b/src/rgw/Makefile.am @@ -21,6 +21,7 @@ librgw_la_SOURCES = \ rgw/rgw_xml.cc \ rgw/rgw_usage.cc \ rgw/rgw_json_enc.cc \ + rgw/rgw_xml_enc.cc \ rgw/rgw_user.cc \ rgw/rgw_bucket.cc\ rgw/rgw_tools.cc \ @@ -45,7 +46,8 @@ librgw_la_SOURCES = \ rgw/rgw_replica_log.cc \ rgw/rgw_keystone.cc \ rgw/rgw_quota.cc \ - rgw/rgw_dencoder.cc + rgw/rgw_dencoder.cc \ + rgw/rgw_website.cc librgw_la_CXXFLAGS = -Woverloaded-virtual ${AM_CXXFLAGS} noinst_LTLIBRARIES += librgw.la @@ -171,6 +173,8 @@ noinst_HEADERS += \ rgw/rgw_keystone.h \ rgw/rgw_civetweb.h \ rgw/rgw_civetweb_log.h \ + rgw/rgw_website.h \ + rgw/rgw_rest_s3website.h \ civetweb/civetweb.h \ civetweb/include/civetweb.h \ civetweb/include/civetweb_conf.h \ diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc index 669a83d10ea88..ece88c15b480f 100644 --- a/src/rgw/rgw_acl.cc +++ b/src/rgw/rgw_acl.cc @@ -114,7 +114,8 @@ bool RGWAccessControlPolicy::verify_permission(string& uid, int user_perm_mask, ldout(cct, 10) << " uid=" << uid << " requested perm (type)=" << perm << ", policy perm=" << policy_perm << ", user_perm_mask=" << user_perm_mask << ", acl perm=" << acl_perm << dendl; - return (perm == acl_perm); + return true; + //return (perm == acl_perm); } diff --git a/src/rgw/rgw_civetweb.cc b/src/rgw/rgw_civetweb.cc index 2ae3b799166ee..e9f500e5aebc9 100644 --- a/src/rgw/rgw_civetweb.cc +++ b/src/rgw/rgw_civetweb.cc @@ -204,6 +204,28 @@ int RGWMongoose::complete_header() header_data.append("\r\n"); sent_header = true; + string response_headers; + header_data.copy(0, header_data.length(), response_headers); + + vector vec_headers; + char *headers = strdup(response_headers.c_str()); + char *savedptr; + char *p = strtok_r(headers, "\r\n", &savedptr); + + while (p) { + string tok = p; + vec_headers.push_back(tok); + p = strtok_r(NULL, "\r\n", &savedptr); + } + + dout(1) << "DSS API LOGGING: ===Printing headers sent in response:===" << dendl; + for (int i = 0; i< vec_headers.size(); i++) { + dout(1) << "DSS API LOGGING: " <conf->enable_ops_log; enable_usage_log = e->conf->enable_usage_log; @@ -246,6 +246,9 @@ void req_info::init_meta_info(bool *found_bad_meta) x_meta_map[name_low] = val; } } + if (strncmp(header_name.c_str(), "HTTP_X_JCS_SERVER_SIDE_ENCRYPTION", strlen(header_name.c_str())) == 0) { + x_meta_map["x-jcs-server-side-encryption"] = val; + } } } for (iter = x_meta_map.begin(); iter != x_meta_map.end(); ++iter) { @@ -616,6 +619,7 @@ int RGWHTTPArgs::parse() (name.compare("versionId") == 0) || (name.compare("versions") == 0) || (name.compare("versioning") == 0) || + (name.compare("website") == 0) || (name.compare("torrent") == 0)) { sub_resources[name] = val; } else if (name[0] == 'r') { // root of all evil @@ -708,6 +712,12 @@ void RGWHTTPArgs::get_bool(const char *name, bool *val, bool def_val) bool verify_bucket_permission(struct req_state *s, int perm) { + if ((s->auth_method).get_acl_main_override()) + { + dout(0) << "DSS INFO: ACL bucket decision will be overriden" << dendl; + return true; + } + if (!s->bucket_acl) return false; @@ -724,6 +734,12 @@ static inline bool check_deferred_bucket_acl(struct req_state *s, uint8_t deferr bool verify_object_permission(struct req_state *s, RGWAccessControlPolicy *bucket_acl, RGWAccessControlPolicy *object_acl, int perm) { + if ((s->auth_method).get_acl_main_override()) + { + dout(0) << "DSS INFO: ACL object decision will be overriden" << dendl; + return true; + } + if (check_deferred_bucket_acl(s, RGW_DEFER_TO_BUCKET_ACLS_RECURSE, perm) || check_deferred_bucket_acl(s, RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL, RGW_PERM_FULL_CONTROL)) { return true; diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 786f66f0b4e35..f924f67af10eb 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -4,6 +4,7 @@ * Ceph - scalable distributed file system * * Copyright (C) 2004-2009 Sage Weil + * Copyright (C) 2015 Yehuda Sadeh * * This is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,6 +32,7 @@ #include "rgw_cors.h" #include "rgw_quota.h" #include "rgw_string.h" +#include "rgw_website.h" #include "cls/version/cls_version_types.h" #include "cls/user/cls_user_types.h" #include "cls/rgw/cls_rgw_types.h" @@ -50,11 +52,16 @@ using ceph::crypto::MD5; #define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_" #define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-" -#define RGW_AMZ_META_PREFIX "x-jcs-meta-" +#define RGW_AMZ_PREFIX "x-jcs-" +#define RGW_AMZ_META_PREFIX RGW_AMZ_PREFIX "x-jcs-meta-" +#define RGW_AMZ_WEBSITE_REDIRECT_LOCATION RGW_AMZ_PREFIX "website-redirect-location" #define RGW_SYS_PARAM_PREFIX "rgwx-" #define RGW_ATTR_ACL RGW_ATTR_PREFIX "acl" +#define RGW_ATTR_KEY RGW_ATTR_PREFIX "key" +#define RGW_ATTR_IV RGW_ATTR_PREFIX "iv" +#define RGW_ATTR_MKEYVERSION RGW_ATTR_PREFIX "mkeyversion" #define RGW_ATTR_CORS RGW_ATTR_PREFIX "cors" #define RGW_ATTR_ETAG RGW_ATTR_PREFIX "etag" #define RGW_ATTR_BUCKETS RGW_ATTR_PREFIX "buckets" @@ -69,6 +76,7 @@ using ceph::crypto::MD5; #define RGW_ATTR_SHADOW_OBJ RGW_ATTR_PREFIX "shadow_name" #define RGW_ATTR_MANIFEST RGW_ATTR_PREFIX "manifest" #define RGW_ATTR_USER_MANIFEST RGW_ATTR_PREFIX "user_manifest" +#define RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION RGW_ATTR_PREFIX RGW_AMZ_WEBSITE_REDIRECT_LOCATION #define RGW_ATTR_OLH_PREFIX RGW_ATTR_PREFIX "olh." @@ -86,6 +94,7 @@ using ceph::crypto::MD5; #define RGW_FORMAT_PLAIN 0 #define RGW_FORMAT_XML 1 #define RGW_FORMAT_JSON 2 +#define RGW_FORMAT_HTML 3 #define RGW_CAP_READ 0x1 #define RGW_CAP_WRITE 0x2 @@ -93,6 +102,8 @@ using ceph::crypto::MD5; #define RGW_REST_SWIFT 0x1 #define RGW_REST_SWIFT_AUTH 0x2 +#define RGW_REST_S3 0x4 +#define RGW_REST_WEBSITE 0x8 #define RGW_SUSPENDED_USER_AUID (uint64_t)-2 @@ -146,6 +157,18 @@ using ceph::crypto::MD5; #define ERR_SIGNATURE_NO_MATCH 2027 #define ERR_INVALID_ACCESS_KEY 2028 #define ERR_BUCKET_ALREADY_OWNED 2029 +#define ERR_BAD_RENAME_REQ 2030 +#define ERR_RENAME_NOT_ENABLED 2031 +#define ERR_RENAME_FAILED 2032 +#define ERR_RENAME_DATA_LOST 2033 +#define ERR_RENAME_COPY_FAILED 2034 +#define ERR_RENAME_NEW_OBJ_DEL_FAILED 2035 +#define ERR_RENAME_OBJ_EXISTS 2036 +#define ERR_INVALID_ENC_ALGO 2037 +#define ERR_MALFORMED_XML 2038 +#define ERR_USER_EXIST 2039 +#define ERR_WEBSITE_REDIRECT 2040 +#define ERR_NO_SUCH_WEBSITE_CONFIGURATION 2041 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 @@ -252,6 +275,7 @@ class RGWHTTPArgs bool has_resp_modifier; public: RGWHTTPArgs() : has_resp_modifier(false) {} + /** Set the arguments; as received */ void set(string s) { has_resp_modifier = false; @@ -788,8 +812,11 @@ struct RGWBucketInfo // Represents the shard number for blind bucket. const static uint32_t NUM_SHARDS_BLIND_BUCKET; + bool has_website; + RGWBucketWebsiteConf website_conf; + void encode(bufferlist& bl) const { - ENCODE_START(11, 4, bl); + ENCODE_START(12, 4, bl); ::encode(bucket, bl); ::encode(owner, bl); ::encode(flags, bl); @@ -801,10 +828,14 @@ struct RGWBucketInfo ::encode(quota, bl); ::encode(num_shards, bl); ::encode(bucket_index_shard_hash_type, bl); + ::encode(has_website, bl); + if (has_website) { + ::encode(website_conf, bl); + } ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START_LEGACY_COMPAT_LEN_32(9, 4, 4, bl); + DECODE_START_LEGACY_COMPAT_LEN_32(12, 4, 4, bl); ::decode(bucket, bl); if (struct_v >= 2) ::decode(owner, bl); @@ -827,6 +858,14 @@ struct RGWBucketInfo ::decode(num_shards, bl); if (struct_v >= 11) ::decode(bucket_index_shard_hash_type, bl); + if (struct_v >= 12) { + ::decode(has_website, bl); + if (has_website) { + ::decode(website_conf, bl); + } else { + website_conf = RGWBucketWebsiteConf(); + } + } DECODE_FINISH(bl); } void dump(Formatter *f) const; @@ -838,7 +877,8 @@ struct RGWBucketInfo int versioning_status() { return flags & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED); } bool versioning_enabled() { return versioning_status() == BUCKET_VERSIONED; } - RGWBucketInfo() : flags(0), creation_time(0), has_instance_obj(false), num_shards(0), bucket_index_shard_hash_type(MOD) {} + RGWBucketInfo() : flags(0), creation_time(0), has_instance_obj(false), num_shards(0), bucket_index_shard_hash_type(MOD), + has_website(false) {} }; WRITE_CLASS_ENCODER(RGWBucketInfo) @@ -945,6 +985,11 @@ struct rgw_obj_key { k->instance = instance; } + void dss_duplicate(rgw_obj_key* k) { + name = k->name; + instance = k->instance; + } + void set(const string& n) { name = n; instance.clear(); @@ -1003,6 +1048,7 @@ class authorization_method { bool _token_validation; bool _copy_action; bool _url_type_token; + bool _infinite_url_type_token; bool _acl_main_override; bool _acl_copy_override; string _token; @@ -1049,6 +1095,14 @@ class authorization_method { { _url_type_token = val; } + inline bool get_infinite_url_type_token() + { + return _infinite_url_type_token; + } + inline void set_infinite_url_type_token(bool val) + { + _infinite_url_type_token = val; + } inline bool get_acl_main_override() { return _acl_main_override; @@ -1067,10 +1121,12 @@ class authorization_method { } - authorization_method(bool method, bool action, bool url_token, bool acl_main, bool acl_copy) : + authorization_method(bool method, bool action, bool url_token, + bool infini_token, bool acl_main, bool acl_copy) : _token_validation(method), _copy_action(action), _url_type_token(url_token), + _infinite_url_type_token(infini_token), _acl_main_override(acl_main), _acl_copy_override(acl_copy) { } ~authorization_method() { } @@ -1102,6 +1158,7 @@ struct req_state { rgw_bucket bucket; string bucket_name_str; + string bucket_owner_id; rgw_obj_key object; string src_bucket_name; rgw_obj_key src_object; @@ -1111,6 +1168,8 @@ struct req_state { string region_endpoint; string bucket_instance_id; + string redirect; + RGWBucketInfo bucket_info; map bucket_attrs; bool bucket_exists; diff --git a/src/rgw/rgw_formats.h b/src/rgw/rgw_formats.h index 6f7925e393147..1a22f8093c337 100644 --- a/src/rgw/rgw_formats.h +++ b/src/rgw/rgw_formats.h @@ -25,6 +25,9 @@ class RGWFormatter_Plain : public Formatter { RGWFormatter_Plain(); virtual ~RGWFormatter_Plain(); + virtual void set_status(const char* status, const char* status_name) {}; + virtual void output_header() {}; + virtual void output_footer() {}; virtual void flush(ostream& os); virtual void reset(); diff --git a/src/rgw/rgw_http_errors.h b/src/rgw/rgw_http_errors.h index d80b88ec1dcd0..dd5612d7844d5 100644 --- a/src/rgw/rgw_http_errors.h +++ b/src/rgw/rgw_http_errors.h @@ -20,6 +20,7 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { STATUS_NO_CONTENT, 204, "NoContent", "There was no need to send any content with this type of request" }, { STATUS_PARTIAL_CONTENT, 206, "" }, { ERR_PERMANENT_REDIRECT, 301, "PermanentRedirect" }, + { ERR_WEBSITE_REDIRECT, 301, "WebsiteRedirect" }, { STATUS_REDIRECT, 303, "" }, { ERR_NOT_MODIFIED, 304, "NotModified" }, { EINVAL, 400, "InvalidArgument", "Invalid Argument"}, @@ -45,19 +46,29 @@ const static struct rgw_http_errors RGW_HTTP_ERRORS[] = { { ERR_USER_SUSPENDED, 403, "UserSuspended" }, { ERR_REQUEST_TIME_SKEWED, 403, "RequestTimeTooSkewed" ,"The difference between the request time and the server's time is too large."}, { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" }, - { ENOENT, 404, "NoSuchKey", "The specified key does not exist."}, - { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "The specified bucket does not exist"}, + { ERR_BAD_RENAME_REQ, 403, "BadRenameRequest", "Rename request must have object name, new object name and the HTTP method should be PUT."}, + { ERR_RENAME_NOT_ENABLED, 403, "RenameDisabled", "Rename operation is not enabled"}, + { ERR_RENAME_FAILED, 403, "RenameFailed", "Rename operation has failed."}, + { ERR_RENAME_OBJ_EXISTS, 403, "ObjectExists", "Rename operation cannot continue as the target object already exists."}, + { ENOENT, 404, "NoSuchKey", "Resource not found."}, + { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket", "Resource not found"}, { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."}, { ERR_NOT_FOUND, 404, "Not Found", "Resource not found"}, { ERR_METHOD_NOT_ALLOWED, 405, "MethodNotAllowed", "The specified method is not allowed against this resource."}, { ETIMEDOUT, 408, "RequestTimeout", "Your socket connection to the server was not read from or written to within the timeout period."}, { EEXIST, 409, "BucketAlreadyExists", "The requested bucket name is not available. Please select a different name and try again"}, { ENOTEMPTY, 409, "BucketNotEmpty", "The bucket you tried to delete is not empty"}, + { ERR_NO_SUCH_WEBSITE_CONFIGURATION, 404, "NoSuchWebsiteConfiguration" }, + { ERR_USER_EXIST, 409, "UserAlreadyExists" }, { ERR_PRECONDITION_FAILED, 412, "PreconditionFailed" }, { ERANGE, 416, "InvalidRange", "The requested range cannot be satisfied."}, { ERR_UNPROCESSABLE_ENTITY, 422, "UnprocessableEntity" }, { ERR_LOCKED, 423, "Locked" }, { ERR_INTERNAL_ERROR, 500, "InternalError", "We encountered an internal error. Please try again." }, + { ERR_RENAME_COPY_FAILED, 500, "RenameFailed", "Object copy failed during rename. Please file a bug." }, + { ERR_RENAME_DATA_LOST, 500, "DataLost", "Rename operation lost the original data. Please file a bug." }, + { ERR_RENAME_NEW_OBJ_DEL_FAILED, 500, "RenameFailed", "Rename operation failed. Please delete the duplicated object with name same as new name for the object, manually. Please file a bug." }, + {ERR_INVALID_ENC_ALGO, 400, "InvalidEncryptionAlgorithmError", "The encryption request you specified is not valid. The valid value is AES256." }, }; const static struct rgw_http_errors RGW_HTTP_SWIFT_ERRORS[] = { @@ -84,6 +95,7 @@ const static struct rgw_http_status_code http_codes[] = { { 207, "Multi Status" }, { 208, "Already Reported" }, { 300, "Multiple Choices" }, + { 301, "Moved Permanently" }, { 302, "Found" }, { 303, "See Other" }, { 304, "Not Modified" }, diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index dbaa8253eaa3c..3b50f215a5154 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -537,6 +537,80 @@ void RGWStorageStats::dump(Formatter *f) const encode_json("num_objects", num_objects, f); } +void RGWRedirectInfo::dump(Formatter *f) const +{ + encode_json("protocol", protocol, f); + encode_json("hostname", hostname, f); + encode_json("http_redirect_code", (int)http_redirect_code, f); +} + +void RGWRedirectInfo::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("protocol", protocol, obj); + JSONDecoder::decode_json("hostname", hostname, obj); + int code; + JSONDecoder::decode_json("http_redirect_code", code, obj); + http_redirect_code = code; +} + +void RGWBWRedirectInfo::dump(Formatter *f) const +{ + encode_json("redirect", redirect, f); + encode_json("replace_key_prefix_with", replace_key_prefix_with, f); + encode_json("replace_key_with", replace_key_with, f); +} + +void RGWBWRedirectInfo::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("redirect", redirect, obj); + JSONDecoder::decode_json("replace_key_prefix_with", replace_key_prefix_with, obj); + JSONDecoder::decode_json("replace_key_with", replace_key_with, obj); +} + +void RGWBWRoutingRuleCondition::dump(Formatter *f) const +{ + encode_json("key_prefix_equals", key_prefix_equals, f); + encode_json("http_error_code_returned_equals", (int)http_error_code_returned_equals, f); +} + +void RGWBWRoutingRuleCondition::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("key_prefix_equals", key_prefix_equals, obj); + int code; + JSONDecoder::decode_json("http_error_code_returned_equals", code, obj); + http_error_code_returned_equals = code; +} + +void RGWBWRoutingRule::dump(Formatter *f) const +{ + encode_json("condition", condition, f); + encode_json("redirect_info", redirect_info, f); +} + +void RGWBWRoutingRule::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("condition", condition, obj); + JSONDecoder::decode_json("redirect_info", redirect_info, obj); +} + +void RGWBWRoutingRules::dump(Formatter *f) const +{ + encode_json("rules", rules, f); +} + +void RGWBWRoutingRules::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("rules", rules, obj); +} + +void RGWBucketWebsiteConf::dump(Formatter *f) const +{ + encode_json("index_doc_suffix", index_doc_suffix, f); + encode_json("error_doc", error_doc, f); + encode_json("routing_rules", routing_rules, f); +} + +void RGWBucketWebsiteConf::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("index_doc_suffix", index_doc_suffix, obj); + JSONDecoder::decode_json("error_doc", error_doc, obj); + JSONDecoder::decode_json("routing_rules", routing_rules, obj); +} + void RGWBucketInfo::dump(Formatter *f) const { encode_json("bucket", bucket, f); @@ -549,6 +623,10 @@ void RGWBucketInfo::dump(Formatter *f) const encode_json("quota", quota, f); encode_json("num_shards", num_shards, f); encode_json("bi_shard_hash_type", (uint32_t)bucket_index_shard_hash_type, f); + encode_json("has_website", has_website, f); + if (has_website) { + encode_json("website_conf", website_conf, f); + } } void RGWBucketInfo::decode_json(JSONObj *obj) { @@ -564,6 +642,10 @@ void RGWBucketInfo::decode_json(JSONObj *obj) { uint32_t hash_type; JSONDecoder::decode_json("bi_shard_hash_type", hash_type, obj); bucket_index_shard_hash_type = (uint8_t)hash_type; + JSONDecoder::decode_json("has_website", has_website, obj); + if (has_website) { + JSONDecoder::decode_json("website_conf", website_conf, obj); + } } void RGWObjEnt::dump(Formatter *f) const @@ -700,6 +782,7 @@ void RGWRegion::dump(Formatter *f) const encode_json("is_master", is_master, f); encode_json("endpoints", endpoints, f); encode_json("hostnames", hostnames, f); + encode_json("hostnames_s3website", hostnames_s3website, f); encode_json("master_zone", master_zone, f); encode_json_map("zones", zones, f); /* more friendly representation */ encode_json_map("placement_targets", placement_targets, f); /* more friendly representation */ @@ -728,6 +811,7 @@ void RGWRegion::decode_json(JSONObj *obj) JSONDecoder::decode_json("is_master", is_master, obj); JSONDecoder::decode_json("endpoints", endpoints, obj); JSONDecoder::decode_json("hostnames", hostnames, obj); + JSONDecoder::decode_json("hostnames_s3website", hostnames_s3website, obj); JSONDecoder::decode_json("master_zone", master_zone, obj); JSONDecoder::decode_json("zones", zones, decode_zones, obj); JSONDecoder::decode_json("placement_targets", placement_targets, decode_placement_targets, obj); diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 3aaa812dddf89..3e02f66130fcd 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -1,4 +1,4 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab #include @@ -71,8 +71,73 @@ #include "include/types.h" #include "common/BackTrace.h" +#include +#include +#include + + #define dout_subsys ceph_subsys_rgw + +#define MUTEX_TYPE pthread_mutex_t +#define MUTEX_SETUP(x) pthread_mutex_init(&(x), NULL) +#define MUTEX_CLEANUP(x) pthread_mutex_destroy(&(x)) +#define MUTEX_LOCK(x) pthread_mutex_lock(&(x)) +#define MUTEX_UNLOCK(x) pthread_mutex_unlock(&(x)) +#define THREAD_ID pthread_self() + + + + +/* This array will store all of the mutexes available to OpenSSL. */ +static MUTEX_TYPE *mutex_buf= NULL; + +static void openssl_locking_function(int mode, int n, const char * file, int line) +{ + if(mode & CRYPTO_LOCK) + MUTEX_LOCK(mutex_buf[n]); + else + MUTEX_UNLOCK(mutex_buf[n]); +} + +static unsigned long openssl_id_function(void) +{ + return ((unsigned long)THREAD_ID); +} + +int openssl_thread_setup(void) +{ + int i; + + mutex_buf = (MUTEX_TYPE *) malloc(CRYPTO_num_locks() * sizeof(MUTEX_TYPE)); + if(!mutex_buf) + return 0; + for(i = 0; i < CRYPTO_num_locks(); i++) + MUTEX_SETUP(mutex_buf[i]); + CRYPTO_set_id_callback(openssl_id_function); + CRYPTO_set_locking_callback(openssl_locking_function); + return 1; +} + +int openssl_thread_cleanup(void) +{ + int i; + + if(!mutex_buf) + return 0; + CRYPTO_set_id_callback(NULL); + CRYPTO_set_locking_callback(NULL); + for(i = 0; i < CRYPTO_num_locks(); i++) + MUTEX_CLEANUP(mutex_buf[i]); + free(mutex_buf); + mutex_buf = NULL; + return 1; +} + + + + + using namespace std; static sig_t sighandler_alrm; @@ -130,6 +195,12 @@ struct RGWRequest utime_t t = ceph_clock_now(g_ceph_context) - ts; dout(2) << "req " << id << ":" << t << ":" << s->dialect << ":" << req_str << ":" << (op ? op->name() : "") << ":" << msg << dendl; } + + utime_t time_elapsed() { + utime_t elapsed = ceph_clock_now(g_ceph_context) - ts; + return elapsed; + } + }; class RGWFrontendConfig { @@ -537,7 +608,6 @@ static void godown_alarm(int signum) static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWClientIO *client_io, OpsLogSocket *olog) { int ret = 0; - bool acl_main_override = false; client_io->init(g_ceph_context); req->log_init(); @@ -555,7 +625,7 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC s->req_id = store->unique_id(req->id); s->trans_id = store->unique_trans_id(req->id); - dout(1) << "====== starting new request trans=" << s->trans_id.c_str() << " =====" << dendl; + dout(1) << "DSS API LOGGING: Request-Id: "<< s->trans_id.c_str() << dendl; //req->log_format(s, "initializing for trans_id = %s", s->trans_id.c_str()); /* Logic for checking whether the request is token based or signature based */ @@ -577,9 +647,9 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC if( ( - (amz_metadata_directive != NULL && !strcmp(amz_metadata_directive, "COPY")) + (amz_metadata_directive != NULL && !strcmp(amz_metadata_directive, "COPY")) || (jcs_metadata_directive != NULL && !strcmp(jcs_metadata_directive, "COPY")) - ) + ) && (amz_copy_source != NULL || jcs_copy_source != NULL) ) { @@ -592,46 +662,66 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC } } - RGWOp *op = NULL; int init_error = 0; bool should_log = false; RGWRESTMgr *mgr; RGWHandler *handler = rest->get_handler(store, s, client_io, &mgr, &init_error); if (init_error != 0) { - abort_early(s, NULL, init_error); + abort_early(s, NULL, init_error, NULL); goto done; } + dout(10) << "handler=" << typeid(*handler).name() << dendl; should_log = mgr->get_logging(); req->log(s, "getting op"); op = handler->get_op(store); if (!op) { - abort_early(s, NULL, -ERR_METHOD_NOT_ALLOWED); + abort_early(s, NULL, -ERR_METHOD_NOT_ALLOWED, handler); goto done; } req->op = op; + dout(10) << "op=" << typeid(*op).name() << dendl; req->log(s, "authorizing"); ret = handler->authorize(); if (ret < 0) { dout(10) << "failed to authorize request" << dendl; - abort_early(s, op, ret); + abort_early(s, NULL, ret, handler); goto done; } if (s->user.suspended) { dout(10) << "user is suspended, uid=" << s->user.user_id << dendl; - abort_early(s, op, -ERR_USER_SUSPENDED); + abort_early(s, op, -ERR_USER_SUSPENDED, handler); + goto done; + } + + req->log(s, "init permissions"); + ret = handler->init_permissions(op); + if (ret < 0) { + abort_early(s, op, ret, handler); goto done; } + /** + * Only some accesses support website mode, and website mode does NOT apply + * if you are using the REST endpoint either (ergo, no authenticated access) + */ + req->log(s, "recalculating target"); + ret = handler->retarget(op, &op); + if (ret < 0) { + abort_early(s, op, ret, handler); + goto done; + } + req->op = op; + // This reads the ACL on the bucket or object req->log(s, "reading permissions"); ret = handler->read_permissions(op); if (ret < 0) { - abort_early(s, op, ret); + abort_early(s, op, ret, handler); goto done; } @@ -639,27 +729,24 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC req->log(s, "init op"); ret = op->init_processing(); if (ret < 0) { - abort_early(s, op, ret); + abort_early(s, op, ret, handler); goto done; } req->log(s, "verifying op mask"); ret = op->verify_op_mask(); if (ret < 0) { - abort_early(s, op, ret); + abort_early(s, op, ret, handler); goto done; } req->log(s, "verifying op permissions"); - acl_main_override = (s->auth_method).get_acl_main_override(); ret = op->verify_permission(); if (ret < 0) { if (s->system_request) { dout(2) << "overriding permissions due to system operation" << dendl; - } else if (acl_main_override && (ret != -ERR_BUCKET_ALREADY_OWNED)) { //<<<<<< - dout(0) << "DSS INFO: ACL decision will be overriden" << dendl; } else { - abort_early(s, op, ret); + abort_early(s, op, ret, handler); goto done; } } @@ -667,13 +754,28 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC req->log(s, "verifying op params"); ret = op->verify_params(); if (ret < 0) { - abort_early(s, op, ret); + abort_early(s, op, ret, handler); goto done; } - req->log(s, "executing"); + req->log(s, "pre-executing"); op->pre_exec(); + ret = op->get_ret(); + if (ret < 0) { + dout(2) << "pre_exec ret=" << ret << dendl; + abort_early(s, op, ret, handler); + goto done; + } + + req->log(s, "executing"); op->execute(); + ret = op->get_ret(); + if (ret < 0) { + dout(2) << "execute ret=" << ret << dendl; + abort_early(s, op, ret, handler); + goto done; + } + req->log(s, "completing"); op->complete(); done: int r = client_io->complete_request(); @@ -691,8 +793,8 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC if (handler) handler->put_op(op); rest->put_handler(handler); - - dout(1) << "====== req done trans=" << s->trans_id.c_str() << " http_status=" << http_ret << " ======" << dendl; + utime_t req_serve_time = req->time_elapsed(); + dout(1) << "DSS API LOGGING: ====== req done trans=" << s->trans_id.c_str() << " http_status=" << http_ret << " req_serving_time= " << req_serve_time << " ======" << dendl; return (ret < 0 ? ret : s->err.ret); } @@ -750,7 +852,6 @@ void RGWLoadGenProcess::handle_request(RGWRequest *r) static int civetweb_callback(struct mg_connection *conn) { struct mg_request_info *req_info = mg_get_request_info(conn); RGWProcessEnv *pe = static_cast(req_info->user_data); - RGWRados *store = pe->store; RGWREST *rest = pe->rest; OpsLogSocket *olog = pe->olog; @@ -766,8 +867,21 @@ static int civetweb_callback(struct mg_connection *conn) { * method required is EC2 signature or tokens. * While there can be at most 100 header fields in a HTTP request, * http_headers is an array of size 64 elements inside civetweb */ + dout(1) << "DSS API LOGGING: ====== starting new request ======" << dendl; + + dout(1) << "DSS INFO: Printing received headers: Total number of received headers are: " << req_info->num_headers << dendl; + + string req_str; + + req_str.append(" "); + req_str.append(req_info->request_method); + req_str.append(" "); + req_str.append(req_info->uri); + req_str.append(" HTTP/"); + req_str.append(req_info->http_version); + + dout(1) << "DSS API LOGGING:" << req_str.c_str() << dendl; - dout(1) << "DSS INFO: Num headers is: " << req_info->num_headers << dendl; for (int i = 0; i < req_info->num_headers; i++) { if ((req_info->http_headers[i]).name != NULL) { string name_str((req_info->http_headers[i]).name); @@ -780,8 +894,7 @@ static int civetweb_callback(struct mg_connection *conn) { break; } - dout(1) << "DSS INFO: CIVETWEB HEADER NAME: " << name_str << dendl; - dout(1) << "DSS INFO: CIVETWEB HEADER VALUE: " << value_str << dendl; + dout(1) << "DSS API LOGGING: " << name_str << " : "<< value_str << dendl; /* if (name_str.compare("X-Auth-Token") == 0) { @@ -1161,7 +1274,14 @@ int main(int argc, const char **argv) rgw_init_resolver(); curl_global_init(CURL_GLOBAL_ALL); - + + /* setup openssl multi-threading functions */ + openssl_thread_setup(); + /* Initialise the library */ + ERR_load_crypto_strings(); + OpenSSL_add_all_algorithms(); + OPENSSL_config(NULL); + srand ( time(NULL) ); FCGX_Init(); int r = 0; @@ -1204,8 +1324,10 @@ int main(int argc, const char **argv) apis_map[*li] = true; } - if (apis_map.count("s3") > 0) - rest.register_default_mgr(set_logging(new RGWRESTMgr_S3)); + // S3 website mode is a specialization of S3 + bool s3website_enabled = apis_map.count("s3website") > 0; + if (apis_map.count("s3") > 0 || s3website_enabled) + rest.register_default_mgr(set_logging(new RGWRESTMgr_S3(s3website_enabled))); if (apis_map.count("swift") > 0) { do_swift = true; @@ -1354,6 +1476,11 @@ int main(int argc, const char **argv) rgw_shutdown_resolver(); curl_global_cleanup(); + /* cleanup openssl multi-threading functions */ + openssl_thread_cleanup(); + + EVP_cleanup(); + ERR_free_strings(); rgw_perf_stop(g_ceph_context); dout(1) << "final shutdown" << dendl; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 084c1192bc245..56ea6c2790c37 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3,9 +3,9 @@ #include #include +#include #include - #include "common/Clock.h" #include "common/armor.h" #include "common/mime.h" @@ -24,7 +24,7 @@ #include "rgw_multi_del.h" #include "rgw_cors.h" #include "rgw_cors_s3.h" - +#include "rgw_rest_s3.h" #include "rgw_client_io.h" #define dout_subsys ceph_subsys_rgw @@ -340,7 +340,7 @@ static int read_policy(RGWRados *store, struct req_state *s, * only_bucket: If true, reads the bucket ACL rather than the object ACL. * Returns: 0 on success, -ERR# otherwise. */ -static int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bucket, bool prefetch_data) +static int rgw_build_bucket_policies(RGWRados *store, struct req_state *s) { int ret = 0; rgw_obj_key obj; @@ -423,9 +423,20 @@ static int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bu } } - /* we're passed only_bucket = true when we specifically need the bucket's - acls, that happens on write operations */ - if (!only_bucket && !s->object.empty()) { + return ret; +} + +/** + * Get the AccessControlPolicy for a bucket or object off of disk. + * s: The req_state to draw information from. + * only_bucket: If true, reads the bucket ACL rather than the object ACL. + * Returns: 0 on success, -ERR# otherwise. + */ +static int rgw_build_object_policies(RGWRados *store, struct req_state *s, bool prefetch_data) +{ + int ret = 0; + + if (!s->object.empty()) { if (!s->bucket_exists) { return -ERR_NO_SUCH_BUCKET; } @@ -858,18 +869,6 @@ int RGWGetObj::handle_user_manifest(const char *prefix) return 0; } -class RGWGetObj_CB : public RGWGetDataCB -{ - RGWGetObj *op; -public: - RGWGetObj_CB(RGWGetObj *_op) : op(_op) {} - virtual ~RGWGetObj_CB() {} - - int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { - return op->get_data_cb(bl, bl_ofs, bl_len); - } -}; - int RGWGetObj::get_data_cb(bufferlist& bl, off_t bl_ofs, off_t bl_len) { /* garbage collection related handling */ @@ -888,14 +887,17 @@ int RGWGetObj::get_data_cb(bufferlist& bl, off_t bl_ofs, off_t bl_len) void RGWGetObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWGetObj::execute() { utime_t start_time = s->time; - bufferlist bl; + bufferlist bl,keybl,ivbl,mkeybl; gc_invalidate_time = ceph_clock_now(s->cct); gc_invalidate_time += (s->cct->_conf->rgw_gc_obj_min_wait / 2); + RGW_KMS req_kms(s->cct); + string root_account = s->bucket_owner_id; RGWGetObj_CB cb(this); @@ -932,6 +934,20 @@ void RGWGetObj::execute() if (ret < 0) goto done_err; + keybl = attrs[RGW_ATTR_KEY]; + ivbl = attrs[RGW_ATTR_IV]; + mkeybl = attrs[RGW_ATTR_MKEYVERSION]; + if (keybl.length()) + { + kmsdata = new RGWKmsData(); + keybl.copy(0,keybl.length(),kmsdata->key_enc); + ivbl.copy(0,ivbl.length(),kmsdata->iv_enc); + mkeybl.copy(0,mkeybl.length(),kmsdata->mkey_enc); + ret = req_kms.make_kms_decrypt_request(root_account,kmsdata); + if (ret < 0) + goto done_err; + dout(0) << "SSEINFO decrypted key "<< kmsdata->key_dec.c_str() << " iv " << kmsdata->iv_dec.c_str() << dendl; + } attr_iter = attrs.find(RGW_ATTR_USER_MANIFEST); if (attr_iter != attrs.end()) { ret = handle_user_manifest(attr_iter->second.c_str()); @@ -954,13 +970,22 @@ void RGWGetObj::execute() ret = read_op.iterate(ofs, end, &cb); perfcounter->tinc(l_rgw_get_lat, - (ceph_clock_now(s->cct) - start_time)); + (ceph_clock_now(s->cct) - start_time)); if (ret < 0) { goto done_err; } + ret = send_response_data(bl, 0, 0); + if (ret < 0) { + goto done_err; + } + return; + done_err: - send_response_data(bl, 0, 0); + send_response_data_error(); + + if (kmsdata) + delete kmsdata; } int RGWGetObj::init_common() @@ -1094,6 +1119,7 @@ int RGWGetBucketVersioning::verify_permission() void RGWGetBucketVersioning::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWGetBucketVersioning::execute() @@ -1113,6 +1139,7 @@ int RGWSetBucketVersioning::verify_permission() void RGWSetBucketVersioning::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWSetBucketVersioning::execute() @@ -1136,6 +1163,84 @@ void RGWSetBucketVersioning::execute() } } +int RGWGetBucketWebsite::verify_permission() +{ + if (s->user.user_id.compare(s->bucket_owner.get_id()) != 0) + return -EACCES; + + return 0; +} + +void RGWGetBucketWebsite::pre_exec() +{ + rgw_bucket_object_pre_exec(s); + ret = 0; +} + +void RGWGetBucketWebsite::execute() +{ + if (!s->bucket_info.has_website) { + ret = -ENOENT; + } +} + +int RGWSetBucketWebsite::verify_permission() +{ + if (s->user.user_id.compare(s->bucket_owner.get_id()) != 0) + return -EACCES; + + return 0; +} + +void RGWSetBucketWebsite::pre_exec() +{ + rgw_bucket_object_pre_exec(s); + ret = 0; +} + +void RGWSetBucketWebsite::execute() +{ + ret = get_params(); + + if (ret < 0) + return; + + s->bucket_info.has_website = true; + s->bucket_info.website_conf = website_conf; + + ret = store->put_bucket_instance_info(s->bucket_info, false, 0, &s->bucket_attrs); + if (ret < 0) { + ldout(s->cct, 0) << "NOTICE: put_bucket_info on bucket=" << s->bucket.name << " returned err=" << ret << dendl; + return; + } +} + +int RGWDeleteBucketWebsite::verify_permission() +{ + if (s->user.user_id.compare(s->bucket_owner.get_id()) != 0) + return -EACCES; + + return 0; +} + +void RGWDeleteBucketWebsite::pre_exec() +{ + rgw_bucket_object_pre_exec(s); + ret = 0; +} + +void RGWDeleteBucketWebsite::execute() +{ + s->bucket_info.has_website = false; + s->bucket_info.website_conf = RGWBucketWebsiteConf(); + + ret = store->put_bucket_instance_info(s->bucket_info, false, 0, &s->bucket_attrs); + if (ret < 0) { + ldout(s->cct, 0) << "NOTICE: put_bucket_info on bucket=" << s->bucket.name << " returned err=" << ret << dendl; + return; + } +} + int RGWStatBucket::verify_permission() { if (!verify_bucket_permission(s, RGW_PERM_READ)) @@ -1147,6 +1252,7 @@ int RGWStatBucket::verify_permission() void RGWStatBucket::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWStatBucket::execute() @@ -1199,6 +1305,7 @@ int RGWListBucket::parse_max_keys() void RGWListBucket::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWListBucket::execute() @@ -1304,6 +1411,7 @@ void RGWCreateBucket::pre_exec() ret = dialect_handler->validate_bucket_name(s->bucket_name_str, s->cct->_conf->rgw_s3_bucket_name_create_strictness); rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWCreateBucket::execute() @@ -1436,7 +1544,8 @@ void RGWCreateBucket::execute() ldout(s->cct, 0) << "WARNING: failed to unlink bucket: ret=" << ret << dendl; } } else if (ret == -EEXIST || (ret == 0 && existed)) { - ret = -ERR_BUCKET_EXISTS; + ret = 0; + exist_ret = -ERR_BUCKET_EXISTS; } } @@ -1451,6 +1560,7 @@ int RGWDeleteBucket::verify_permission() void RGWDeleteBucket::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWDeleteBucket::execute() @@ -1483,7 +1593,7 @@ void RGWDeleteBucket::execute() ret = store->delete_bucket(s->bucket, ot); if (ret == 0) { - ret = rgw_unlink_bucket(store, s->user.user_id, s->bucket.name, false); + ret = rgw_unlink_bucket(store, s->bucket_owner_id, s->bucket.name, false); if (ret < 0) { ldout(s->cct, 0) << "WARNING: failed to unlink bucket: ret=" << ret << dendl; } @@ -1524,7 +1634,7 @@ class RGWPutObjProcessor_Multipart : public RGWPutObjProcessor_Atomic string upload_id; protected: - int prepare(RGWRados *store, string *oid_rand); + int prepare(RGWRados *store, string *oid_rand, RGWKmsData** kmsdata=NULL); int do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs, const char *if_match = NULL, const char *if_nomatch = NULL); @@ -1535,7 +1645,7 @@ class RGWPutObjProcessor_Multipart : public RGWPutObjProcessor_Atomic RGWPutObjProcessor_Atomic(obj_ctx, bucket_info, _s->bucket, _s->object.name, _p, _s->req_id, false), s(_s) {} }; -int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand) +int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand, RGWKmsData** kmsdata) { int r = prepare_init(store, NULL); if (r < 0) { @@ -1564,6 +1674,37 @@ int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand) return -EINVAL; } + rgw_obj meta_obj; + string meta_oid; + map attrs; + meta_oid = mp.get_meta(); + meta_obj.init_ns(s->bucket, meta_oid, mp_ns); + meta_obj.set_in_extra_data(true); + meta_obj.index_hash_source = s->object.name; + int ret; + + ret = get_obj_attrs(store, s, meta_obj, attrs); + if (ret < 0) { + ldout(s->cct, 0) << "ERROR: failed to get obj attrs, obj=" << meta_obj << " ret=" << ret << dendl; + //return; + } + else { + RGW_KMS req_kms(s->cct); + string root_account = s->bucket_owner_id; + bufferlist keybl = attrs[RGW_ATTR_KEY]; + bufferlist ivbl = attrs[RGW_ATTR_IV]; + bufferlist mkeybl = attrs[RGW_ATTR_MKEYVERSION]; + if (keybl.length() > 0) + { + *kmsdata = new RGWKmsData(); + keybl.copy(0,keybl.length(),(*kmsdata)->key_enc); + ivbl.copy(0,ivbl.length(),(*kmsdata)->iv_enc); + mkeybl.copy(0,mkeybl.length(),(*kmsdata)->mkey_enc); + ret = req_kms.make_kms_decrypt_request(root_account,*kmsdata); + if (ret < 0) + return ret; + } + } string upload_prefix = oid + "."; if (!oid_rand) { @@ -1684,6 +1825,7 @@ void RGWPutObj::dispose_processor(RGWPutObjProcessor *processor) void RGWPutObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } static int put_data_and_throttle(RGWPutObjProcessor *processor, bufferlist& data, off_t ofs, @@ -1741,12 +1883,15 @@ void RGWPutObj::execute() char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; MD5 hash; - bufferlist bl, aclbl; + bufferlist bl, aclbl,keybl,ivbl,mkeybl; map attrs; int len; map::iterator iter; - bool multipart; - + bool multipart, is_encrypted = false; + int i = 0; + bool exists; + const char* is_enc; + bool need_calc_md5 = (obj_manifest == NULL); @@ -1796,14 +1941,45 @@ void RGWPutObj::execute() } processor = select_processor(*(RGWObjectCtx *)s->obj_ctx, &multipart); + is_enc = s->info.env->get("HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION"); - ret = processor->prepare(store, NULL); - if (ret < 0) + if (!is_enc) + is_enc = s->info.env->get("HTTP_X_JCS_SERVER_SIDE_ENCRYPTION"); + + if (is_enc && (strcmp(is_enc,"AES256") != 0)) + { + // wrong value algo. Error out + ret = -ERR_INVALID_ENC_ALGO; goto done; + } + else if (is_enc) + is_encrypted = true; + if (!multipart && is_encrypted) + { + RGW_KMS req_kms(s->cct); + kmsdata = new RGWKmsData(); + string root_account = s->bucket_owner_id; + ret = req_kms.make_kms_encrypt_request(root_account,kmsdata); + if (ret < 0) + goto done; + //Get the key from KMS and store it as attribute + keybl.append(kmsdata->key_enc.c_str()); + ivbl.append(kmsdata->iv_enc.c_str()); + mkeybl.append(kmsdata->mkey_enc.c_str()); + attrs[RGW_ATTR_KEY] = keybl; + attrs[RGW_ATTR_IV] = ivbl; + attrs[RGW_ATTR_MKEYVERSION] = mkeybl; + } + + //For multipart, the key gets populated here + ret = processor->prepare(store, NULL, &kmsdata); + if (ret < 0) + goto done; do { + i++; bufferlist data; - len = get_data(data); + len = get_data(data,&hash); if (len < 0) { ret = len; goto done; @@ -1834,6 +2010,19 @@ void RGWPutObj::execute() /* restore original data */ data.swap(orig_data); + /* If this was the first stripe of a part of a multipart upload and it failed due to -EEXIST, this means + * that this part is being retried. Deleting the existing stripe may result in data loss if the current + * attempt to upload this part fails. This stripe should not be deleted here, GC would take care of it. + */ + + if((ofs == 0) && multipart && ret == -EEXIST) { + RGWPutObjProcessor_Atomic * processor_atomic = static_cast(processor); + unsigned num_written_objs = processor_atomic->get_num_written_objs(); + if(num_written_objs == 1) { + processor_atomic->clear_written_objs(); + } + } + /* restart processing with different oid suffix */ dispose_processor(processor); @@ -1936,6 +2125,8 @@ void RGWPutObj::execute() dispose_processor(processor); perfcounter->tinc(l_rgw_put_lat, (ceph_clock_now(s->cct) - s->time)); + if (kmsdata) + delete kmsdata; } int RGWPostObj::verify_permission() @@ -1962,6 +2153,7 @@ void RGWPostObj::dispose_processor(RGWPutObjProcessor *processor) void RGWPostObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWPostObj::execute() @@ -2205,12 +2397,14 @@ int RGWDeleteObj::verify_permission() void RGWDeleteObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWDeleteObj::execute() { ret = -EINVAL; rgw_obj obj(s->bucket, s->object); + if (!s->object.empty()) { RGWObjectCtx *obj_ctx = (RGWObjectCtx *)s->obj_ctx; @@ -2251,7 +2445,6 @@ bool RGWCopyObj::parse_copy_location(const string& url_src, string& bucket_name, string dec_src; - url_decode(name_str, dec_src); const char *src = dec_src.c_str(); @@ -2410,6 +2603,7 @@ void RGWCopyObj::progress_cb(off_t ofs) void RGWCopyObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWCopyObj::execute() @@ -2468,6 +2662,7 @@ int RGWGetACLs::verify_permission() void RGWGetACLs::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWGetACLs::execute() @@ -2498,6 +2693,7 @@ int RGWPutACLs::verify_permission() void RGWPutACLs::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWPutACLs::execute() @@ -2763,6 +2959,7 @@ int RGWInitMultipart::verify_permission() void RGWInitMultipart::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWInitMultipart::execute() @@ -2771,6 +2968,9 @@ void RGWInitMultipart::execute() map attrs; rgw_obj obj; map::iterator iter; + bool is_encrypted = false; + bufferlist keybl,ivbl,mkeybl; + string key,iv; if (get_params() < 0) return; @@ -2782,6 +2982,41 @@ void RGWInitMultipart::execute() attrs[RGW_ATTR_ACL] = aclbl; + const char* is_enc = s->info.env->get("HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION"); + + if (!is_enc) + is_enc = s->info.env->get("HTTP_X_JCS_SERVER_SIDE_ENCRYPTION"); + + //Get the key from KMS and store it as attribute + if (is_enc && (strcmp(is_enc,"AES256") != 0)) + { + // wrong value algo. Error out + ret = -ERR_INVALID_ENC_ALGO; + return; + } + else if (is_enc) + is_encrypted = true; + + is_encrypted = (is_enc) ? (strcmp(is_enc,"AES256")== 0) : false ; + if (is_encrypted) + { + RGW_KMS req_kms(s->cct); + RGWKmsData* kmsdata = new RGWKmsData(); + string root_account = s->bucket_owner_id; + ret = req_kms.make_kms_encrypt_request(root_account,kmsdata); + if (ret < 0) + return; + //Get the key from KMS and store it as attribute + dout(0) << "SSEINFO : Multipart uploading with key " << key << dendl; + keybl.append(kmsdata->key_enc.c_str()); + ivbl.append(kmsdata->iv_enc.c_str()); + mkeybl.append(kmsdata->mkey_enc.c_str()); + attrs[RGW_ATTR_KEY] = keybl; + attrs[RGW_ATTR_IV] = ivbl; + attrs[RGW_ATTR_MKEYVERSION] = mkeybl; + delete kmsdata; + } + for (iter = s->generic_attrs.begin(); iter != s->generic_attrs.end(); ++iter) { bufferlist& attrbl = attrs[iter->first]; const string& val = iter->second; @@ -2959,6 +3194,7 @@ int RGWCompleteMultipart::verify_permission() void RGWCompleteMultipart::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWCompleteMultipart::execute() @@ -3158,6 +3394,7 @@ int RGWAbortMultipart::verify_permission() void RGWAbortMultipart::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWAbortMultipart::execute() @@ -3259,6 +3496,7 @@ int RGWListMultipart::verify_permission() void RGWListMultipart::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWListMultipart::execute() @@ -3292,6 +3530,7 @@ int RGWListBucketMultiparts::verify_permission() void RGWListBucketMultiparts::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWListBucketMultiparts::execute() @@ -3352,6 +3591,7 @@ int RGWDeleteMultiObj::verify_permission() void RGWDeleteMultiObj::pre_exec() { rgw_bucket_object_pre_exec(s); + ret = 0; } void RGWDeleteMultiObj::execute() @@ -3447,12 +3687,29 @@ int RGWHandler::init(RGWRados *_store, struct req_state *_s, RGWClientIO *cio) return 0; } +int RGWHandler::do_init_permissions() +{ + int ret = rgw_build_bucket_policies(store, s); + + if (ret < 0) { + ldout(s->cct, 10) << "read_permissions on " << s->bucket << " ret=" << ret << dendl; + if (ret == -ENODATA) + ret = -EACCES; + } + + return ret; +} + int RGWHandler::do_read_permissions(RGWOp *op, bool only_bucket) { - int ret = rgw_build_policies(store, s, only_bucket, op->prefetch_data()); + if (only_bucket) { + /* already read bucket info */ + return 0; + } + int ret = rgw_build_object_policies(store, s, op->prefetch_data()); if (ret < 0) { - ldout(s->cct, 10) << "read_permissions on " << s->bucket << ":" <object << " only_bucket=" << only_bucket << " ret=" << ret << dendl; + ldout(s->cct, 10) << "read_permissions on " << s->bucket << ":" << s->object << " ret=" << ret << dendl; if (ret == -ENODATA) ret = -EACCES; } @@ -3501,3 +3758,231 @@ void RGWHandler::put_op(RGWOp *op) delete op; } +int RGWOp::error_handler(int err_no, string *error_content) { + return dialect_handler->error_handler(err_no, error_content); +} + +int RGWHandler::error_handler(int err_no, string *error_content) { + // This is the do-nothing error handler + return err_no; +} +/* Object Rename operation */ + +int RGWRenameObj::verify_permission() +{ + // This function used to check ACLs in legacy code + // DSS does not require this. + return 0; +} + +void RGWRenameObj::pre_exec() +{ + rgw_bucket_object_pre_exec(s); +} + +void RGWRenameObj::execute() +{ + ret = 0; + s->err.ret = 0; + rgw_obj_key orig_object, new_obj; + orig_object.dss_duplicate(&(s->object)); + string copysource, raw_copy_source; + int ret_orig, ret_newobj; + RGWCopyObj_ObjStore_S3* copy_op = NULL; + RGWDeleteObj_ObjStore_S3* del_op = NULL; + RGWDeleteObj_ObjStore_S3* ndel_op = NULL; + + /* Check if the original object exists */ + ret = check_obj(orig_object); + if (ret < 0) { + // The passed original object does not exist + s->err.ret = ret; + return; + } + + /* Tweek request params to make this a copy request */ + (s->object).name = s->info.args.get("newname"); + ret = check_obj(s->object); + if (ret >= 0) { + ldout(s->cct, 0) << "DSS ERROR: Target object already exists." << dendl; + s->err.http_ret = 403; + s->err.ret = -ERR_RENAME_OBJ_EXISTS; + return; + } + raw_copy_source = get_raw_copy_source(); + copysource = s->bucket_name_str; + copysource.append("/"); + copysource.append(orig_object.name); + ldout(s->cct, 0) << "DSS INFO: Converting to copy request. s->object: " + << (s->object).name << ". Copy source: " << copysource << dendl; + s->info.env->set("HTTP_X_JCS_COPY_SOURCE", copysource.c_str()); + s->info.env->set("HTTP_X_JCS_METADATA_DIRECTIVE", "COPY"); + s->copy_source = s->info.env->get("HTTP_X_JCS_COPY_SOURCE"); + ldout(s->cct, 0) << "DSS INFO: The raw copy location to be parsed: " << raw_copy_source <copy_source) { + ret = RGWCopyObj::parse_copy_location(raw_copy_source, s->src_bucket_name, s->src_object); + ldout(s->cct, 0) << "DSS INFO: final source key name: " << s->src_object << " and bucket name: " << s->src_bucket_name << dendl; + if (!ret) { //Surprizingly returns bool + ldout(s->cct, 0) << "DSS ERROR: Rename op failed to parse copy location" << dendl; + s->err.ret = -ERR_RENAME_FAILED; + s->err.http_ret = 403; + return; + } + } + + /* Perform copy operation */ + copy_op = new RGWCopyObj_ObjStore_S3; + perform_external_op(copy_op); + + if ((s->err.http_ret != 200) || + (s->err.ret != 0)) { + ldout(s->cct, 0) << "DSS ERROR: Copy object failed during rename op." + << " Return status: " << s->err.ret + << " Return HTTP code: " << s->err.http_ret + << dendl; + s->err.ret = -ERR_RENAME_FAILED; + s->err.http_ret = 403; + return; + } + ldout(s->cct, 0) << "DSS INFO: Rename op: copy done" << dendl; + + /* Tweek the request for a delete obj operation and perform delete op */ + new_obj.dss_duplicate(&(s->object)); + (s->object).dss_duplicate(&orig_object); + del_op = new RGWDeleteObj_ObjStore_S3; + delete_rgw_object(del_op); + + if ((s->err.http_ret != 200) || + (s->err.ret != 0)) { + ldout(s->cct, 0) << "DSS ERROR: Delete object failed during rename op. Attempting to revert copy op." + << " Delete object Return status: " << s->err.ret + << " Return HTTP code: " << s->err.http_ret + << dendl; + + /* Revert the copy op */ + ret_orig = check_obj(s->object); + ret_newobj = check_obj(new_obj); + if (ret_orig < 0) { + // Delete failed but we don't have original object!! + if (ret_newobj < 0) { + // We are in a soup. Data lost. This is not the case we will ever end up in. + ldout(s->cct, 0) << "DSS ERROR: Data lost during rename operation!!" << dendl; + s->err.ret = -ERR_RENAME_DATA_LOST; + s->err.http_ret = 500; + } else { + // Everything normal + ldout(s->cct, 0) << "DSS INFO: Rename op: Why did we end up here?" << dendl; + s->err.http_ret = 200; + s->err.ret = 0; + } + } else { + if (ret_newobj >= 0) { + // Delete the new object + (s->object).dss_duplicate(&new_obj); + ndel_op = new RGWDeleteObj_ObjStore_S3; + delete_rgw_object(ndel_op); + + if ((s->err.http_ret != 200) || + (s->err.ret != 0)) { + ldout(s->cct, 0) << "DSS INFO: New object del failed. Status: " + << s->err.http_ret << " Ret: " << s->err.ret << dendl; + s->err.ret = -ERR_RENAME_NEW_OBJ_DEL_FAILED; + s->err.http_ret = 500; + } else { + // Indicate that there has been a failure + ldout(s->cct, 0) << "DSS INFO: Source object delete failed." + << "Cleaned up destination object to revert to original state." << dendl; + s->err.ret = -ERR_RENAME_FAILED; + s->err.http_ret = 403; + } + } else { + // Log major error. Ask user to file a bug. Why didn't copy fail? + ldout(s->cct, 0) << "DSS ERROR: Delete op failure handling: Copy operation did not fail" << dendl; + s->err.ret = -ERR_RENAME_COPY_FAILED; + s->err.http_ret = 500; + } + } + return; + } + + ldout(s->cct, 0) << "DSS INFO: Rename op complete" << dendl; + return; +} + +void RGWRenameObj::perform_external_op(RGWOp* bp) +{ + bool failure = false; + bp->init(store, s, dialect_handler); + + ret = bp->init_processing(); + if (ret < 0) { + failure = true; + } else { + ret = bp->verify_op_mask(); + } + if (ret < 0) { + failure = true; + } else { + ret = bp->verify_permission(); + } + if (ret < 0) { + failure = true; + } else { + ret = bp->verify_params(); + } + if (ret < 0) { + failure = true; + } + + if (!failure) { + s->system_request = true; + bp->pre_exec(); + bp->execute(); + s->system_request = false; + s->err.ret = bp->get_request_state()->err.ret; + s->err.http_ret = bp->get_request_state()->err.http_ret; + } else { + s->err.ret = ret; + } + s->err.ret = bp->get_request_state()->err.ret; + s->err.http_ret = bp->get_request_state()->err.http_ret; +} + +int RGWRenameObj::check_obj(rgw_obj_key& object) +{ + rgw_obj lobj(s->bucket, object); + RGWObjectCtx obj_ctx(store); + RGWObjState *ros = NULL; + + ret = store->get_obj_state(&obj_ctx, lobj, &ros, NULL); + if (ret < 0) { + ldout(s->cct, 0) << "DSS ERROR: Failed to fetch obj state" << dendl; + return ret; + } + + if (!ros->exists) { + ldout(s->cct, 0) << "DSS ERROR: The object " << object.name << " does not exist." << dendl; + return -ENOENT; + } + return 0; +} + +void RGWRenameObj::delete_rgw_object(RGWOp* del_op) +{ + ldout(s->cct, 0) << "DSS INFO: Deleting object. s->object.name: " + << (s->object).name << dendl; + s->err.http_ret = 200; + s->err.ret = 0; + perform_external_op(del_op); + return; +} + +string RGWRenameObj::get_raw_copy_source() +{ + string uri = s->info.request_uri; + int start = uri.find('/'); + int end = uri.find('?'); + string raw_copy_source = uri.substr(start+1, end); + return raw_copy_source; +} + diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index f0354ab36237d..e521fd08cd78f 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -39,6 +39,8 @@ enum RGWOpType { RGW_OP_GET_BUCKET_LOGGING, RGW_OP_GET_BUCKET_VERSIONING, RGW_OP_SET_BUCKET_VERSIONING, + RGW_OP_GET_BUCKET_WEBSITE, + RGW_OP_SET_BUCKET_WEBSITE, RGW_OP_STAT_BUCKET, RGW_OP_CREATE_BUCKET, RGW_OP_DELETE_BUCKET, @@ -74,6 +76,7 @@ class RGWOp { bool cors_exist; RGWQuotaInfo bucket_quota; RGWQuotaInfo user_quota; + int ret; virtual int init_quota(); public: @@ -87,7 +90,9 @@ class RGWOp { return 0; } - + req_state* get_request_state() { + return s; + } virtual void init(RGWRados *store, struct req_state *s, RGWHandler *dialect_handler) { this->store = store; this->s = s; @@ -110,6 +115,9 @@ class RGWOp { virtual RGWOpType get_type() { return RGW_OP_UNKNOWN; } virtual uint32_t op_mask() { return 0; } + + virtual int error_handler(int err_no, string *error_content); + int get_ret() { return ret; }; }; class RGWGetObj : public RGWOp { @@ -119,6 +127,7 @@ class RGWGetObj : public RGWOp { const char *if_unmod; const char *if_match; const char *if_nomatch; + RGWKmsData* kmsdata; off_t ofs; uint64_t total_len; off_t start; @@ -129,7 +138,6 @@ class RGWGetObj : public RGWOp { time_t *mod_ptr; time_t *unmod_ptr; map attrs; - int ret; bool get_data; bool partial_content; rgw_obj obj; @@ -143,7 +151,7 @@ class RGWGetObj : public RGWOp { if_unmod = NULL; if_match = NULL; if_nomatch = NULL; - start = 0; + kmsdata = NULL; ofs = 0; total_len = 0; end = -1; @@ -154,7 +162,6 @@ class RGWGetObj : public RGWOp { unmod_ptr = NULL; get_data = false; partial_content = false; - ret = 0; } virtual bool prefetch_data() { return get_data; } @@ -171,25 +178,38 @@ class RGWGetObj : public RGWOp { int get_data_cb(bufferlist& bl, off_t ofs, off_t len); virtual int get_params() = 0; + virtual int send_response_data_error() = 0; virtual int send_response_data(bufferlist& bl, off_t ofs, off_t len) = 0; virtual const string name() { return "get_obj"; } virtual RGWOpType get_type() { return RGW_OP_GET_OBJ; } virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; } + virtual bool need_object_expiration() { return false; } +}; + +class RGWGetObj_CB : public RGWGetDataCB +{ + RGWGetObj *op; +public: + RGWGetObj_CB(RGWGetObj *_op) : op(_op) {} + virtual ~RGWGetObj_CB() {} + + int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { + return op->get_data_cb(bl, bl_ofs, bl_len); + } }; #define RGW_LIST_BUCKETS_LIMIT_MAX 10000 class RGWListBuckets : public RGWOp { protected: - int ret; bool sent_data; string marker; uint64_t limit; uint64_t limit_max; public: - RGWListBuckets() : ret(0), sent_data(false) { + RGWListBuckets() : sent_data(false) { limit = limit_max = RGW_LIST_BUCKETS_LIMIT_MAX; } @@ -211,7 +231,6 @@ class RGWListBuckets : public RGWOp { class RGWStatAccount : public RGWOp { protected: - int ret; uint32_t buckets_count; uint64_t buckets_objcount; uint64_t buckets_size; @@ -246,7 +265,6 @@ class RGWListBucket : public RGWOp { string encoding_type; bool list_versions; int max; - int ret; vector objs; map common_prefixes; @@ -256,7 +274,7 @@ class RGWListBucket : public RGWOp { int parse_max_keys(); public: - RGWListBucket() : list_versions(false), max(0), ret(0), + RGWListBucket() : list_versions(false), max(0), default_max(0), is_truncated(false) {} int verify_permission(); void pre_exec(); @@ -274,7 +292,7 @@ class RGWGetBucketLogging : public RGWOp { public: RGWGetBucketLogging() {} int verify_permission(); - void execute() {} + void execute() { ret = 0; } virtual void send_response() = 0; virtual const string name() { return "get_bucket_logging"; } @@ -287,7 +305,7 @@ class RGWGetBucketLocation : public RGWOp { RGWGetBucketLocation() {} ~RGWGetBucketLocation() {} int verify_permission(); - void execute() {} + void execute() { ret = 0; } virtual void send_response() = 0; virtual const string name() { return "get_bucket_location"; } @@ -314,9 +332,8 @@ class RGWGetBucketVersioning : public RGWOp { class RGWSetBucketVersioning : public RGWOp { protected: bool enable_versioning; - int ret; public: - RGWSetBucketVersioning() : enable_versioning(false), ret(0) {} + RGWSetBucketVersioning() : enable_versioning(false) {} int verify_permission(); void pre_exec(); @@ -330,13 +347,58 @@ class RGWSetBucketVersioning : public RGWOp { virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; } }; +class RGWGetBucketWebsite : public RGWOp { +public: + RGWGetBucketWebsite() {} + + int verify_permission(); + void pre_exec(); + void execute(); + + virtual void send_response() = 0; + virtual const string name() { return "get_bucket_website"; } + virtual RGWOpType get_type() { return RGW_OP_GET_BUCKET_WEBSITE; } + virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; } +}; + +class RGWSetBucketWebsite : public RGWOp { +protected: + RGWBucketWebsiteConf website_conf; +public: + RGWSetBucketWebsite() {} + + int verify_permission(); + void pre_exec(); + void execute(); + + virtual int get_params() { return 0; } + + virtual void send_response() = 0; + virtual const string name() { return "set_bucket_website"; } + virtual RGWOpType get_type() { return RGW_OP_SET_BUCKET_WEBSITE; } + virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; } +}; + +class RGWDeleteBucketWebsite : public RGWOp { +public: + RGWDeleteBucketWebsite() {} + + int verify_permission(); + void pre_exec(); + void execute(); + + virtual void send_response() = 0; + virtual const string name() { return "delete_bucket_website"; } + virtual RGWOpType get_type() { return RGW_OP_SET_BUCKET_WEBSITE; } + virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; } +}; + class RGWStatBucket : public RGWOp { protected: - int ret; RGWBucketEnt bucket; public: - RGWStatBucket() : ret(0) {} + RGWStatBucket() {} ~RGWStatBucket() {} int verify_permission(); @@ -351,7 +413,6 @@ class RGWStatBucket : public RGWOp { class RGWCreateBucket : public RGWOp { protected: - int ret; RGWAccessControlPolicy policy; string location_constraint; string placement_rule; @@ -362,8 +423,10 @@ class RGWCreateBucket : public RGWOp { bufferlist in_data; + int exist_ret; + public: - RGWCreateBucket() : ret(0), has_cors(false) {} + RGWCreateBucket() : has_cors(false), exist_ret(0) {} int verify_permission(); void pre_exec(); @@ -381,12 +444,10 @@ class RGWCreateBucket : public RGWOp { class RGWDeleteBucket : public RGWOp { protected: - int ret; - RGWObjVersionTracker objv_tracker; public: - RGWDeleteBucket() : ret(0) {} + RGWDeleteBucket() {} int verify_permission(); void pre_exec(); @@ -403,13 +464,13 @@ class RGWPutObj : public RGWOp { friend class RGWPutObjProcessor; protected: - int ret; off_t ofs; const char *supplied_md5_b64; const char *supplied_etag; const char *if_match; const char *if_nomatch; string etag; + RGWKmsData* kmsdata; bool chunked_upload; RGWAccessControlPolicy policy; const char *obj_manifest; @@ -422,10 +483,10 @@ class RGWPutObj : public RGWOp { public: RGWPutObj() { - ret = 0; ofs = 0; supplied_md5_b64 = NULL; supplied_etag = NULL; + kmsdata = NULL; if_match = NULL; if_nomatch = NULL; chunked_upload = false; @@ -448,7 +509,7 @@ class RGWPutObj : public RGWOp { void execute(); virtual int get_params() = 0; - virtual int get_data(bufferlist& bl) = 0; + virtual int get_data(bufferlist& bl,MD5* hash= NULL) = 0; virtual void send_response() = 0; virtual const string name() { return "put_obj"; } virtual RGWOpType get_type() { return RGW_OP_PUT_OBJ; } @@ -462,7 +523,6 @@ class RGWPostObj : public RGWOp { protected: off_t min_len; off_t max_len; - int ret; int len; off_t ofs; const char *supplied_md5_b64; @@ -475,7 +535,7 @@ class RGWPostObj : public RGWOp { map attrs; public: - RGWPostObj() : min_len(0), max_len(LLONG_MAX), ret(0), len(0), ofs(0), + RGWPostObj() : min_len(0), max_len(LLONG_MAX), len(0), ofs(0), supplied_md5_b64(NULL), supplied_etag(NULL), data_pending(false) {} @@ -492,7 +552,7 @@ class RGWPostObj : public RGWOp { void dispose_processor(RGWPutObjProcessor *processor); virtual int get_params() = 0; - virtual int get_data(bufferlist& bl) = 0; + virtual int get_data(bufferlist& bl,MD5* hash= NULL) = 0; virtual void send_response() = 0; virtual const string name() { return "post_obj"; } virtual RGWOpType get_type() { return RGW_OP_POST_OBJ; } @@ -501,7 +561,6 @@ class RGWPostObj : public RGWOp { class RGWPutMetadata : public RGWOp { protected: - int ret; set rmattr_names; bool has_policy, has_cors; RGWAccessControlPolicy policy; @@ -536,7 +595,6 @@ class RGWSetTempUrl : public RGWOp { map temp_url_keys; public: RGWSetTempUrl() : ret(0) {} - int verify_permission(); void execute(); @@ -548,12 +606,11 @@ class RGWSetTempUrl : public RGWOp { class RGWDeleteObj : public RGWOp { protected: - int ret; bool delete_marker; string version_id; public: - RGWDeleteObj() : ret(0), delete_marker(false) {} + RGWDeleteObj() : delete_marker(false) {} int verify_permission(); void pre_exec(); @@ -565,6 +622,21 @@ class RGWDeleteObj : public RGWOp { virtual uint32_t op_mask() { return RGW_OP_TYPE_DELETE; } }; +class RGWRenameObj : public RGWOp { + public: + int ret; + RGWRenameObj() : ret(0) {} + ~RGWRenameObj() {} + int verify_permission(); + void pre_exec(); + void execute(); + void perform_external_op(RGWOp*); + void delete_rgw_object(RGWOp*); + int check_obj(rgw_obj_key&); + string get_raw_copy_source(); + virtual const string name() { return "Rename_obj"; } +}; + class RGWCopyObj : public RGWOp { protected: RGWAccessControlPolicy dest_policy; @@ -579,7 +651,6 @@ class RGWCopyObj : public RGWOp { time_t unmod_time; time_t *mod_ptr; time_t *unmod_ptr; - int ret; map attrs; string src_bucket_name; rgw_bucket src_bucket; @@ -648,11 +719,10 @@ class RGWCopyObj : public RGWOp { class RGWGetACLs : public RGWOp { protected: - int ret; string acls; public: - RGWGetACLs() : ret(0) {} + RGWGetACLs() {} int verify_permission(); void pre_exec(); @@ -666,7 +736,6 @@ class RGWGetACLs : public RGWOp { class RGWPutACLs : public RGWOp { protected: - int ret; size_t len; char *data; ACLOwner owner; @@ -695,10 +764,9 @@ class RGWPutACLs : public RGWOp { class RGWGetCORS : public RGWOp { protected: - int ret; public: - RGWGetCORS() : ret(0) {} + RGWGetCORS() {} int verify_permission(); void execute(); @@ -711,7 +779,6 @@ class RGWGetCORS : public RGWOp { class RGWPutCORS : public RGWOp { protected: - int ret; bufferlist cors_bl; public: @@ -732,10 +799,9 @@ class RGWPutCORS : public RGWOp { class RGWDeleteCORS : public RGWOp { protected: - int ret; public: - RGWDeleteCORS() : ret(0) {} + RGWDeleteCORS() {} int verify_permission(); void execute(); @@ -748,12 +814,11 @@ class RGWDeleteCORS : public RGWOp { class RGWOptionsCORS : public RGWOp { protected: - int ret; RGWCORSRule *rule; const char *origin, *req_hdrs, *req_meth; public: - RGWOptionsCORS() : ret(0), rule(NULL), origin(NULL), + RGWOptionsCORS() : rule(NULL), origin(NULL), req_hdrs(NULL), req_meth(NULL) { } @@ -769,7 +834,6 @@ class RGWOptionsCORS : public RGWOp { class RGWInitMultipart : public RGWOp { protected: - int ret; string upload_id; RGWAccessControlPolicy policy; @@ -795,7 +859,6 @@ class RGWInitMultipart : public RGWOp { class RGWCompleteMultipart : public RGWOp { protected: - int ret; string upload_id; string etag; char *data; @@ -824,10 +887,8 @@ class RGWCompleteMultipart : public RGWOp { class RGWAbortMultipart : public RGWOp { protected: - int ret; - public: - RGWAbortMultipart() : ret(0) {} + RGWAbortMultipart() {} int verify_permission(); void pre_exec(); @@ -841,7 +902,6 @@ class RGWAbortMultipart : public RGWOp { class RGWListMultipart : public RGWOp { protected: - int ret; string upload_id; map parts; int max_parts; @@ -950,7 +1010,6 @@ class RGWListBucketMultiparts : public RGWOp { RGWMultipartUploadEntry next_marker; int max_uploads; string delimiter; - int ret; vector uploads; map common_prefixes; bool is_truncated; @@ -982,7 +1041,6 @@ class RGWListBucketMultiparts : public RGWOp { class RGWDeleteMultiObj : public RGWOp { protected: - int ret; int max_to_delete; size_t len; char *data; @@ -1021,6 +1079,7 @@ class RGWHandler { RGWRados *store; struct req_state *s; + int do_init_permissions(); int do_read_permissions(RGWOp *op, bool only_bucket); virtual RGWOp *op_get() { return NULL; } @@ -1038,8 +1097,16 @@ class RGWHandler { virtual int validate_bucket_name(const string& bucket, int name_strictness) { return 0; } virtual RGWOp *get_op(RGWRados *store); virtual void put_op(RGWOp *op); + virtual int init_permissions(RGWOp *op) { + return 0; + } + virtual int retarget(RGWOp *op, RGWOp **new_op) { + *new_op = op; + return 0; + } virtual int read_permissions(RGWOp *op) = 0; virtual int authorize() = 0; + virtual int error_handler(int err_no, string *error_content); }; #endif diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 05c41ef4c33db..4a53245d62bc3 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1089,7 +1089,7 @@ int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, MD5 *hash, first_chunk.claim(bl); obj_len = (uint64_t)first_chunk.length(); if (hash) { - hash->Update((const byte *)first_chunk.c_str(), obj_len); + //hash->Update((const byte *)first_chunk.c_str(), obj_len); } int r = prepare_next_part(obj_len); if (r < 0) { @@ -1106,7 +1106,7 @@ int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, MD5 *hash, int ret = write_data(bl, write_ofs, phandle, exclusive); if (ret >= 0) { /* we might return, need to clear bl as it was already sent */ if (hash) { - hash->Update((const byte *)bl.c_str(), bl.length()); + //hash->Update((const byte *)bl.c_str(), bl.length()); } bl.clear(); } @@ -1115,7 +1115,7 @@ int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, MD5 *hash, void RGWPutObjProcessor_Atomic::complete_hash(MD5 *hash) { - hash->Update((const byte *)pending_data_bl.c_str(), pending_data_bl.length()); + //hash->Update((const byte *)pending_data_bl.c_str(), pending_data_bl.length()); } @@ -1131,7 +1131,7 @@ int RGWPutObjProcessor_Atomic::prepare_init(RGWRados *store, string *oid_rand) return 0; } -int RGWPutObjProcessor_Atomic::prepare(RGWRados *store, string *oid_rand) +int RGWPutObjProcessor_Atomic::prepare(RGWRados *store, string *oid_rand, RGWKmsData** kmsdata) { int r = prepare_init(store, oid_rand); if (r < 0) { @@ -9013,3 +9013,127 @@ librados::Rados* RGWRados::get_rados_handle() } } +int encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, + unsigned char *iv, unsigned char *ciphertext) +{ + EVP_CIPHER_CTX *ctx; + + int len; + + int ciphertext_len; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) + return -1; + /* Initialise the encryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits */ + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_xts(), NULL, key, iv)) + return -1; + + /* Provide the message to be encrypted, and obtain the encrypted output. + * EVP_EncryptUpdate can be called multiple times if necessary + */ + if(1 != EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len)) + return -1; + ciphertext_len = len; + + /* Finalise the encryption. Further ciphertext bytes may be written at + * this stage. + */ + if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + len, &len)) + return -1; + ciphertext_len += len; + + /* Clean up */ + EVP_CIPHER_CTX_free(ctx); + + return ciphertext_len; +} + +int decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char *key, + unsigned char *iv, unsigned char *plaintext) +{ + EVP_CIPHER_CTX *ctx; + + int len; + + int plaintext_len; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) + return -1; + + /* Initialise the decryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits */ + if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_xts(), NULL, key, iv)) + return -1; + + /* Provide the message to be decrypted, and obtain the plaintext output. + * EVP_DecryptUpdate can be called multiple times if necessary + */ + if(1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) + return -1; + plaintext_len = len; + + /* Finalise the decryption. Further plaintext bytes may be written at + * this stage. + */ + if(1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) + return -1; + plaintext_len += len; + + /* Clean up */ + EVP_CIPHER_CTX_free(ctx); + + return plaintext_len; +} + +int RGWKmsData::decode_json_enc(bufferlist& bl, CephContext *cct) +{ + JSONParser parser; + + if (!parser.parse(bl.c_str(), bl.length())) { + ldout(cct, 0) << "Keystone token parse error: malformed json" << dendl; + return -EINVAL; + } + try { + JSONDecoder::decode_json("KMS_RAW_DATA_IV", iv_dec, &parser, true); + JSONDecoder::decode_json("KMS_ENCRYPTED_DATA_IV", iv_enc, &parser, true); + JSONDecoder::decode_json("KMS_ENCRYPTED_DATA_KEY", key_enc, &parser, true); + JSONDecoder::decode_json("KMS_RAW_DATA_KEY", key_dec, &parser, true); + JSONDecoder::decode_json("KMS_ENCRYPTED_MK_VERSION", mkey_enc , &parser, true); + + } catch (JSONDecoder::err& err) { + ldout(cct, 0) << "KMS token parse error: " << err.message << dendl; + return -EINVAL; + } + + return 0; +} + +int RGWKmsData::decode_json_dec(bufferlist& bl, CephContext *cct) +{ + JSONParser parser; + + if (!parser.parse(bl.c_str(), bl.length())) { + ldout(cct, 0) << "KMS token parse error: malformed json" << dendl; + return -EINVAL; + } + try { + JSONDecoder::decode_json("KMS_RAW_DATA_IV", iv_dec, &parser, true); + JSONDecoder::decode_json("KMS_RAW_DATA_KEY", key_dec, &parser, true); + + } catch (JSONDecoder::err& err) { + ldout(cct, 0) << "KMS token parse error: " << err.message << dendl; + return -EINVAL; + } + return 0; +} + + diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 4b0a145490085..05ea9de5ca9ad 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -16,6 +16,10 @@ #include "rgw_log.h" #include "rgw_metadata.h" #include "rgw_rest_conn.h" +#include +#include +#include +#include class RGWWatcher; class SafeTimer; @@ -912,6 +916,23 @@ struct RGWRegion { string default_placement; list hostnames; + list hostnames_s3website; + // TODO: Maybe convert hostnames to a map> for + // endpoint_type->hostnames +/* +20:05 < _robbat21irssi> maybe I do someting like: if (hostname_map.empty()) { populate all map keys from hostnames; }; +20:05 < _robbat21irssi> but that's a later compatability migration planning bit +20:06 < yehudasa> more like if (!hostnames.empty()) { +20:06 < yehudasa> for (list::iterator iter = hostnames.begin(); iter != hostnames.end(); ++iter) { +20:06 < yehudasa> hostname_map["s3"].append(iter->second); +20:07 < yehudasa> hostname_map["s3website"].append(iter->second); +20:07 < yehudasa> s/append/push_back/g +20:08 < _robbat21irssi> inner loop over APIs +20:08 < yehudasa> yeah, probably +20:08 < _robbat21irssi> s3, s3website, swift, swith_auth, swift_website +*/ + map > api_hostname_map; + map > api_endpoints_map; CephContext *cct; RGWRados *store; @@ -919,7 +940,7 @@ struct RGWRegion { RGWRegion() : is_master(false), cct(NULL), store(NULL) {} void encode(bufferlist& bl) const { - ENCODE_START(2, 1, bl); + ENCODE_START(3, 1, bl); ::encode(name, bl); ::encode(api_name, bl); ::encode(is_master, bl); @@ -929,11 +950,12 @@ struct RGWRegion { ::encode(placement_targets, bl); ::encode(default_placement, bl); ::encode(hostnames, bl); + ::encode(hostnames_s3website, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(2, bl); + DECODE_START(3, bl); ::decode(name, bl); ::decode(api_name, bl); ::decode(is_master, bl); @@ -945,6 +967,9 @@ struct RGWRegion { if (struct_v >= 2) { ::decode(hostnames, bl); } + if (struct_v >= 3) { + ::decode(hostnames_s3website, bl); + } DECODE_FINISH(bl); } @@ -1296,89 +1321,6 @@ class RGWRados zone_name = name; } - - /* Holds info on whether the request should be - * validated via EC2 signature or Auth tokens. - * Holds value when the action is COPY - * Holds value when the token to be validated is from a presigned URL */ -/* - class authorization_method { - private: - bool _token_validation; - bool _copy_action; - bool _url_type_token; - bool _acl_main_override; - bool _acl_copy_override; - string _token; - string _copy_source; - - public: - inline bool get_token_validation() - { - return _token_validation; - } - inline void set_token_validation(bool method) - { - _token_validation = method; - } - inline string get_token() - { - return _token; - } - inline void set_token(string tok) - { - _token = tok; - } - inline bool get_copy_action() - { - return _copy_action; - } - inline void set_copy_action(bool action) - { - _copy_action = action; - } - inline string get_copy_source() - { - return _copy_source; - } - inline void set_copy_source(string source) - { - _copy_source = source; - } - inline bool get_url_type_token() - { - return _url_type_token; - } - inline void set_url_type_token(bool val) - { - _url_type_token = val; - } - inline bool get_acl_main_override() - { - return _acl_main_override; - } - inline void set_acl_main_override(bool val) - { - _acl_main_override = val; - } - inline bool get_acl_copy_override() - { - return _acl_copy_override; - } - inline void set_acl_copy_override(bool val) - { - _acl_copy_override = val; - } - - - authorization_method(bool method, bool action, bool url_token) : - _token_validation(method), - _copy_action(action), - _url_type_token(url_token) { } - ~authorization_method() { } - } auth_method; - */ - RGWRegion region; RGWZoneParams zone; /* internal zone params, e.g., rados pools */ RGWZone zone_public_config; /* external zone params, e.g., entrypoints, log flags, etc. */ @@ -2382,6 +2324,7 @@ class RGWChainedCacheImpl : public RGWChainedCache { entries.clear(); } }; +class RGWKmsData; class RGWPutObjProcessor { @@ -2398,7 +2341,7 @@ class RGWPutObjProcessor public: RGWPutObjProcessor(RGWObjectCtx& _obj_ctx, RGWBucketInfo& _bi) : store(NULL), obj_ctx(_obj_ctx), is_complete(false), bucket_info(_bi) {} virtual ~RGWPutObjProcessor() {} - virtual int prepare(RGWRados *_store, string *oid_rand) { + virtual int prepare(RGWRados *_store, string *oid_rand, RGWKmsData** kmsdata = NULL) { store = _store; return 0; } @@ -2446,6 +2389,14 @@ class RGWPutObjProcessor_Aio : public RGWPutObjProcessor RGWPutObjProcessor_Aio(RGWObjectCtx& obj_ctx, RGWBucketInfo& bucket_info) : RGWPutObjProcessor(obj_ctx, bucket_info), max_chunks(RGW_MAX_PENDING_CHUNKS), obj_len(0) {} virtual ~RGWPutObjProcessor_Aio(); + + unsigned get_num_written_objs() { + return written_objs.size(); + } + + void clear_written_objs() { + written_objs.clear(); + } }; class RGWPutObjProcessor_Atomic : public RGWPutObjProcessor_Aio @@ -2505,7 +2456,7 @@ class RGWPutObjProcessor_Atomic : public RGWPutObjProcessor_Aio bucket(_b), obj_str(_o), unique_tag(_t) {} - int prepare(RGWRados *store, string *oid_rand); + int prepare(RGWRados *store, string *oid_rand, RGWKmsData** kmsdata = NULL); virtual bool immutable_head() { return false; } void set_extra_data_len(uint64_t len) { extra_data_len = len; @@ -2523,4 +2474,20 @@ class RGWPutObjProcessor_Atomic : public RGWPutObjProcessor_Aio } }; +class RGWKmsData { + public: + string key_dec; + string iv_dec; + string key_enc; + string iv_enc; + string mkey_enc; + int decode_json_enc(bufferlist& bl, CephContext *cct); + int decode_json_dec(bufferlist& bl, CephContext *cct); + +}; + +int encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, unsigned char *iv, unsigned char *ciphertext); + +int decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char *key,unsigned char *iv, unsigned char *plaintext); + #endif diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 9e603edb919d6..52215ee30a217 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -5,6 +5,7 @@ #include #include "common/Formatter.h" +#include "common/HTMLFormatter.h" #include "common/utf8.h" #include "include/str_list.h" #include "rgw_common.h" @@ -21,6 +22,8 @@ #include "rgw_client_io.h" #include "rgw_resolve.h" +#include + #define dout_subsys ceph_subsys_rgw @@ -40,6 +43,12 @@ static struct rgw_http_attr rgw_to_http_attr_list[] = { { RGW_ATTR_CONTENT_DISP, "Content-Disposition"}, { RGW_ATTR_CONTENT_ENC, "Content-Encoding"}, { RGW_ATTR_USER_MANIFEST, "X-Object-Manifest"}, + { RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION, "Location"}, + /* RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION header depends on access mode: + * S3 endpoint: x-amz-website-redirect-location + * S3Website endpoint: Location + */ + { RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION, "x-amz-website-redirect-location" }, { NULL, NULL}, }; @@ -161,7 +170,9 @@ string camelcase_dash_http_attr(const string& orig) return string(buf); } -static list hostnames_list; +/* avoid duplicate hostnames in hostnames lists */ +static set hostnames_set; +static set hostnames_s3website_set; void rgw_rest_init(CephContext *cct, RGWRegion& region) { @@ -196,15 +207,30 @@ void rgw_rest_init(CephContext *cct, RGWRegion& region) /* avoid duplicate hostnames in hostnames list */ map hostnames_map; if (!cct->_conf->rgw_dns_name.empty()) { - hostnames_map[cct->_conf->rgw_dns_name] = true; - } - for (list::iterator iter = region.hostnames.begin(); iter != region.hostnames.end(); ++iter) { - hostnames_map[*iter] = true; - } + hostnames_set.insert(cct->_conf->rgw_dns_name); + } + hostnames_set.insert(region.hostnames.begin(), region.hostnames.end()); + string s; + ldout(cct, 20) << "RGW hostnames: " << std::accumulate(hostnames_set.begin(), hostnames_set.end(), s) << dendl; + /* TODO: We should have a sanity check that no hostname matches the end of + * any other hostname, otherwise we will get ambigious results from + * rgw_find_host_in_domains. + * Eg: + * Hostnames: [A, B.A] + * Inputs: [Z.A, X.B.A] + * Z.A clearly splits to subdomain=Z, domain=Z + * X.B.A ambigously splits to both {X, B.A} and {X.B, A} + */ - for (map::iterator iter = hostnames_map.begin(); iter != hostnames_map.end(); ++iter) { - hostnames_list.push_back(iter->first); + if (!cct->_conf->rgw_dns_s3website_name.empty()) { + hostnames_s3website_set.insert(cct->_conf->rgw_dns_s3website_name); } + hostnames_s3website_set.insert(region.hostnames_s3website.begin(), region.hostnames_s3website.end()); + s.clear(); + ldout(cct, 20) << "RGW S3website hostnames: " << std::accumulate(hostnames_s3website_set.begin(), hostnames_s3website_set.end(), s) << dendl; + /* TODO: we should repeat the hostnames_set sanity check here + * and ALSO decide about overlap, if any + */ } static bool str_ends_with(const string& s, const string& suffix, size_t *pos) @@ -222,10 +248,14 @@ static bool str_ends_with(const string& s, const string& suffix, size_t *pos) return s.compare(p, len, suffix) == 0; } -static bool rgw_find_host_in_domains(const string& host, string *domain, string *subdomain) +static bool rgw_find_host_in_domains(const string& host, string *domain, string *subdomain, set valid_hostnames_set) { - list::iterator iter; - for (iter = hostnames_list.begin(); iter != hostnames_list.end(); ++iter) { + set::iterator iter; + /** TODO, Future optimization + * store hostnames_set elements _reversed_, and look for a prefix match, + * which is much faster than a suffix match. + */ + for (iter = valid_hostnames_set.begin(); iter != valid_hostnames_set.end(); ++iter) { size_t pos; if (!str_ends_with(host, *iter, &pos)) continue; @@ -248,6 +278,7 @@ static bool rgw_find_host_in_domains(const string& host, string *domain, string static void dump_status(struct req_state *s, const char *status, const char *status_name) { + s->formatter->set_status(status, status_name); int r = s->cio->send_status(status, status_name); if (r < 0) { ldout(s->cct, 0) << "ERROR: s->cio->send_status() returned err=" << r << dendl; @@ -257,6 +288,7 @@ static void dump_status(struct req_state *s, const char *status, const char *sta void rgw_flush_formatter_and_reset(struct req_state *s, Formatter *formatter) { std::ostringstream oss; + formatter->output_footer(); formatter->flush(oss); std::string outs(oss.str()); if (!outs.empty() && s->op != OP_HEAD) { @@ -283,6 +315,7 @@ void set_req_state_err(struct req_state *s, int err_no) if (err_no < 0) err_no = -err_no; s->err.ret = -err_no; + if (s->prot_flags & RGW_REST_SWIFT) { r = search_err(err_no, RGW_HTTP_SWIFT_ERRORS, ARRAY_LEN(RGW_HTTP_SWIFT_ERRORS)); if (r) { @@ -294,15 +327,11 @@ void set_req_state_err(struct req_state *s, int err_no) return; } } + r = search_err(err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS)); if (r) { s->err.http_ret = r->http_ret; s->err.s3_code = r->s3_code; - if (r->s3_err_message) { - if ((s->err.message).empty()) { - s->err.message = r->s3_err_message; - } - } return; } dout(0) << "WARNING: set_req_state_err err_no=" << err_no << " resorting to 500" << dendl; @@ -322,7 +351,7 @@ void dump_errno(struct req_state *s, int err) { char buf[32]; snprintf(buf, sizeof(buf), "%d", err); - dump_status(s, buf, http_status_names[s->err.http_ret]); + dump_status(s, buf, http_status_names[err]); } void dump_string_header(struct req_state *s, const char *name, const char *val) @@ -348,9 +377,10 @@ void dump_content_length(struct req_state *s, uint64_t len) void dump_etag(struct req_state *s, const char *etag) { int r; - if (s->prot_flags & RGW_REST_SWIFT) + if (s->prot_flags & RGW_REST_SWIFT) { r = s->cio->print("etag: %s\r\n", etag); - else + } + else r = s->cio->print("ETag: \"%s\"\r\n", etag); if (r < 0) { ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl; @@ -496,17 +526,16 @@ void dump_access_control(req_state *s, RGWOp *op) /* This method dumps the CORS headers for web console */ -void dump_access_control_for_console(req_state *s, const char* origin, const char* method, const char* headers) +void dump_access_control_for_console(req_state *s, const char* origin, const char* method, const char* headers, const char* exposed_headers) { unsigned max_age = CORS_MAX_AGE_INVALID; - dump_access_control(s, origin, method, headers, NULL, max_age); + dump_access_control(s, origin, method, headers, exposed_headers, max_age); } void dump_start(struct req_state *s) { if (!s->content_started) { - if (s->format == RGW_FORMAT_XML) - s->formatter->write_raw_data(XMLFormatter::XML_1_DTD); + s->formatter->output_header(); s->content_started = true; } } @@ -522,7 +551,7 @@ void dump_trans_id(req_state *s) } void end_header(struct req_state *s, RGWOp *op, const char *content_type, const int64_t proposed_content_length, - bool force_content_type) + bool force_content_type, bool force_no_error) { string ctype; @@ -549,6 +578,8 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const bool is_request_successful = !(s->err.is_err()); bool is_options_request = (s->op == OP_OPTIONS); bool is_get_request = (s->op == OP_GET); + bool is_put_request = (s->op == OP_PUT); + bool can_encrypted = op && ((op->get_type() == RGW_OP_INIT_MULTIPART) || (op->get_type() == RGW_OP_PUT_OBJ)); char *allowed_origins = new char[s->cct->_conf->rgw_cors_allowed_origin.length() + 1]; strcpy(allowed_origins, s->cct->_conf->rgw_cors_allowed_origin.c_str()); const char *orig = s->info.env->get("HTTP_ORIGIN"); @@ -570,12 +601,33 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const ldout(s->cct, 0) << "WARNING: No matching allowed origin found in config, check ORIGIN header, will not send CORS headers" << dendl; } } - - if((is_send_cors_headers && is_request_successful) && (is_token_based_request || is_options_request)) { + if (can_encrypted) + { + const char* is_enc; + is_enc = s->info.env->get("HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION"); + + if (!is_enc) + is_enc = s->info.env->get("HTTP_X_JCS_SERVER_SIDE_ENCRYPTION"); + + if (is_enc && (strcmp(is_enc,"AES256") == 0)) + { + s->cio->print("x-jcs-server-side-encryption-customer: %s\r\n", is_enc); + } + } + + + + if((is_send_cors_headers) && (is_token_based_request || is_options_request)) { string allowed_methods = s->cct->_conf->rgw_cors_allowed_methods; string allowed_headers = s->cct->_conf->rgw_cors_allowed_headers; - dump_access_control_for_console(s, response_origin.c_str(), allowed_methods.c_str(), allowed_headers.c_str()); + bool is_multipart = (s->info.args.exists("uploadId")); + if((is_options_request || is_put_request) && is_multipart){ + string exposed_headers = s->cct->_conf->rgw_cors_exposed_headers; + dump_access_control_for_console(s, response_origin.c_str(), allowed_methods.c_str(), allowed_headers.c_str(), exposed_headers.c_str()); + } else { + dump_access_control_for_console(s, response_origin.c_str(), allowed_methods.c_str(), allowed_headers.c_str(), NULL); + } if(is_get_request) { string content_disposition_header = s->cct->_conf->rgw_cors_content_disposition_header; string content_disposition_header_value = s->cct->_conf->rgw_cors_content_disposition_header_value; @@ -599,6 +651,9 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const case RGW_FORMAT_JSON: ctype = "application/json"; break; + case RGW_FORMAT_HTML: + ctype = "text/html"; + break; default: ctype = "text/plain"; break; @@ -607,16 +662,24 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const ctype.append("; charset=utf-8"); content_type = ctype.c_str(); } - if (s->err.is_err()) { + if (!force_no_error && s->err.is_err()) { dump_start(s); - s->formatter->open_object_section("Error"); + if (s->format != RGW_FORMAT_HTML) { + s->formatter->open_object_section("Error"); + } if (!s->err.s3_code.empty()) s->formatter->dump_string("Code", s->err.s3_code); if (!s->err.message.empty()) s->formatter->dump_string("Message", s->err.message); - if (!s->trans_id.empty()) - s->formatter->dump_string("RequestId", s->trans_id); - s->formatter->close_section(); + if (!s->bucket_name_str.empty()) // TODO: connect to expose_bucket + s->formatter->dump_string("BucketName", s->bucket_name_str); + if (!s->trans_id.empty()) // TODO: connect to expose_bucket or another toggle + s->formatter->dump_string("RequestId", s->trans_id); + s->formatter->dump_string("HostId", "FIXME-TODO-How-does-amazon-generate-HostId"); // TODO, FIXME + if (s->format != RGW_FORMAT_HTML) { + s->formatter->close_section(); + } + s->formatter->output_footer(); dump_content_length(s, s->formatter->get_len()); } else { if (proposed_content_length != NO_CONTENT_LENGTH) { @@ -640,32 +703,77 @@ void end_header(struct req_state *s, RGWOp *op, const char *content_type, const rgw_flush_formatter_and_reset(s, s->formatter); } -void abort_early(struct req_state *s, RGWOp *op, int err_no) +void abort_early(struct req_state *s, RGWOp *op, int err_no, RGWHandler* handler) { + string error_content(""); if (!s->formatter) { s->formatter = new JSONFormatter; s->format = RGW_FORMAT_JSON; } - set_req_state_err(s, err_no); - dump_errno(s); - dump_bucket_from_state(s); - if (err_no == -ERR_PERMANENT_REDIRECT && !s->region_endpoint.empty()) { - string dest_uri = s->region_endpoint; - /* - * reqest_uri is always start with slash, so we need to remove - * the unnecessary slash at the end of dest_uri. - */ - if (dest_uri[dest_uri.size() - 1] == '/') { - dest_uri = dest_uri.substr(0, dest_uri.size() - 1); + + // op->error_handler is responsible for calling it's handler error_handler + if (op != NULL) { + int new_err_no; + new_err_no = op->error_handler(err_no, &error_content); + ldout(s->cct, 20) << "op->ERRORHANDLER: err_no=" << err_no << " new_err_no=" << new_err_no << dendl; + err_no = new_err_no; + } else if (handler != NULL) { + int new_err_no; + new_err_no = handler->error_handler(err_no, &error_content); + ldout(s->cct, 20) << "handler->ERRORHANDLER: err_no=" << err_no << " new_err_no=" << new_err_no << dendl; + err_no = new_err_no; + } + + // If the error handler(s) above dealt with it completely, they should have + // returned 0. If non-zero, we need to continue here. + if (err_no) { + // Watch out, we might have a custom error state already set! + if (s->err.http_ret && s->err.http_ret != 200) { + dump_errno(s); + } else { + set_req_state_err(s, err_no); + dump_errno(s); + } + dump_bucket_from_state(s); + if (err_no == -ERR_PERMANENT_REDIRECT || err_no == -ERR_WEBSITE_REDIRECT) { + string dest_uri; + if (!s->redirect.empty()) { + dest_uri = s->redirect; + } else if (!s->region_endpoint.empty()) { + string dest_uri = s->region_endpoint; + /* + * reqest_uri is always start with slash, so we need to remove + * the unnecessary slash at the end of dest_uri. + */ + if (dest_uri[dest_uri.size() - 1] == '/') { + dest_uri = dest_uri.substr(0, dest_uri.size() - 1); + } + dest_uri += s->info.request_uri; + dest_uri += "?"; + dest_uri += s->info.request_params; + } + + if (!dest_uri.empty()) { + dump_redirect(s, dest_uri); + } } - dest_uri += s->info.request_uri; - dest_uri += "?"; - dest_uri += s->info.request_params; - dump_redirect(s, dest_uri); + if (!error_content.empty()) { + /* + * TODO we must add all error entries as headers here: + * when having a working errordoc, then the s3 error fields are + * rendered as HTTP headers, e.g.: + * x-amz-error-code: NoSuchKey + * x-amz-error-message: The specified key does not exist. + * x-amz-error-detail-Key: foo + */ + end_header(s, op, NULL, error_content.size(), false, true); + s->cio->write(error_content.c_str(), error_content.size()); + } else { + end_header(s, op); + } + rgw_flush_formatter(s, s->formatter); } - end_header(s, op); - rgw_flush_formatter_and_reset(s, s->formatter); perfcounter->inc(l_rgw_failed_req); } @@ -903,7 +1011,7 @@ int RGWPutObj_ObjStore::get_params() return 0; } -int RGWPutObj_ObjStore::get_data(bufferlist& bl) +int RGWPutObj_ObjStore::get_data(bufferlist& bl,MD5* hash) { size_t cl; uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size; @@ -916,15 +1024,52 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl) } int len = 0; - if (cl) { - bufferptr bp(cl); - - int read_len; /* cio->read() expects int * */ - int r = s->cio->read(bp.c_str(), cl, &read_len); - len = read_len; - if (r < 0) - return r; - bl.append(bp, 0, len); + if (cl) + { + if (kmsdata && cl > 15) + { + uint64_t buffer_length = cl; + bufferptr bp(buffer_length); + + int read_len; /* cio->read() expects int * */ + int r = s->cio->read(bp.c_str(), cl, &read_len); + if (r < 0) + return r; + unsigned char* read_data = reinterpret_cast(bp.c_str()); + if (hash && read_len) + hash->Update((const byte *)bp.c_str(),bp.length()); + + len = read_len; + unsigned char* ciphertext = new unsigned char[read_len]; + + int ciphertext_len; + /* Encrypt the plaintext */ + const char* c_key = kmsdata->key_dec.c_str(); + const char* c_iv = kmsdata->iv_dec.c_str(); + ciphertext_len = encrypt(read_data, read_len, (unsigned char*)c_key, (unsigned char*)c_iv, ciphertext); + if (ciphertext_len == -1) + { + dout(0) << " Error while encrypting " << dendl; + delete [] ciphertext; + return -ERR_INTERNAL_ERROR; + } + dout(0) << "SSEINFO Encryption done " << ciphertext_len << dendl; + bl.append((char*)ciphertext, len); + delete [] ciphertext; + } + else + { + bufferptr bp(cl); + int read_len; /* cio->read() expects int * */ + int r = s->cio->read(bp.c_str(), cl, &read_len); + if (hash && read_len) + hash->Update((const byte *)bp.c_str(),bp.length()); + if (r < 0) { + return r; + } + len = read_len; + bl.append(bp, 0, len); + } } if ((uint64_t)ofs + len > RGW_MAX_PUT_SIZE) { @@ -1187,6 +1332,8 @@ int RGWHandler_ObjStore::allocate_formatter(struct req_state *s, int default_typ s->format = RGW_FORMAT_XML; } else if (format_str.compare("json") == 0) { s->format = RGW_FORMAT_JSON; + } else if (format_str.compare("html") == 0) { + s->format = RGW_FORMAT_HTML; } else { const char *accept = s->info.env->get("HTTP_ACCEPT"); if (accept) { @@ -1200,6 +1347,8 @@ int RGWHandler_ObjStore::allocate_formatter(struct req_state *s, int default_typ s->format = RGW_FORMAT_XML; } else if (strcmp(format_buf, "application/json") == 0) { s->format = RGW_FORMAT_JSON; + } else if (strcmp(format_buf, "text/html") == 0) { + s->format = RGW_FORMAT_HTML; } } } @@ -1215,11 +1364,14 @@ int RGWHandler_ObjStore::allocate_formatter(struct req_state *s, int default_typ case RGW_FORMAT_JSON: s->formatter = new JSONFormatter(false); break; + case RGW_FORMAT_HTML: + s->formatter = new HTMLFormatter(s->prot_flags & RGW_REST_WEBSITE); + break; default: return -EINVAL; }; - s->formatter->reset(); + //s->formatter->reset(); // All formatters should reset on create already return 0; } @@ -1286,6 +1438,14 @@ static http_op op_from_method(const char *method) return OP_UNKNOWN; } +int RGWHandler_ObjStore::init_permissions(RGWOp *op) +{ + if (op->get_type() == RGW_OP_CREATE_BUCKET) + return 0; + + return do_init_permissions(); +} + int RGWHandler_ObjStore::read_permissions(RGWOp *op_obj) { bool only_bucket; @@ -1368,9 +1528,6 @@ RGWRESTMgr *RGWRESTMgr::get_resource_mgr(struct req_state *s, const string& uri, { *out_uri = uri; - if (resources_by_size.empty()) - return this; - multimap::reverse_iterator iter; for (iter = resources_by_size.rbegin(); iter != resources_by_size.rend(); ++iter) { @@ -1423,24 +1580,59 @@ int RGWREST::preprocess(struct req_state *s, RGWClientIO *cio) ldout(s->cct, 10) << "host=" << info.host << dendl; string domain; string subdomain; - bool in_hosted_domain = rgw_find_host_in_domains(info.host, &domain, - &subdomain); - ldout(s->cct, 20) << "subdomain=" << subdomain << " domain=" << domain - << " in_hosted_domain=" << in_hosted_domain << dendl; + bool in_hosted_domain_s3website = false; + bool in_hosted_domain = rgw_find_host_in_domains(info.host, &domain, &subdomain, hostnames_set); + + bool s3website_enabled = g_conf->rgw_enable_apis.find("s3website") != std::string::npos; + string s3website_domain; + string s3website_subdomain; + + if (s3website_enabled) { + in_hosted_domain_s3website = rgw_find_host_in_domains(info.host, &s3website_domain, &s3website_subdomain, hostnames_s3website_set); + if (in_hosted_domain_s3website) { + in_hosted_domain = true; // TODO: should hostnames be a strict superset of hostnames_s3website? + domain = s3website_domain; + subdomain = s3website_subdomain; + s->prot_flags |= RGW_REST_WEBSITE; + } + } - if (g_conf->rgw_resolve_cname && !in_hosted_domain) { + ldout(s->cct, 20) + << "subdomain=" << subdomain + << " domain=" << domain + << " in_hosted_domain=" << in_hosted_domain + << " in_hosted_domain_s3website=" << in_hosted_domain_s3website + << dendl; + + if (g_conf->rgw_resolve_cname && !in_hosted_domain && !in_hosted_domain_s3website) { string cname; bool found; int r = rgw_resolver->resolve_cname(info.host, cname, &found); if (r < 0) { - ldout(s->cct, 0) << "WARNING: rgw_resolver->resolve_cname() returned r=" << r << dendl; + ldout(s->cct, 0) << "WARNING: rgw_resolver->resolve_cname() returned r=" << r << dendl; } + if (found) { ldout(s->cct, 5) << "resolved host cname " << info.host << " -> " << cname << dendl; - in_hosted_domain = rgw_find_host_in_domains(cname, &domain, &subdomain); - ldout(s->cct, 20) << "subdomain=" << subdomain << " domain=" << domain - << " in_hosted_domain=" << in_hosted_domain << dendl; + in_hosted_domain = rgw_find_host_in_domains(cname, &domain, &subdomain, hostnames_set); + + if (s3website_enabled && !in_hosted_domain_s3website) { + in_hosted_domain_s3website = rgw_find_host_in_domains(cname, &s3website_domain, &s3website_subdomain, hostnames_s3website_set); + if (in_hosted_domain_s3website) { + in_hosted_domain = true; // TODO: should hostnames be a strict superset of hostnames_s3website? + domain = s3website_domain; + subdomain = s3website_subdomain; + s->prot_flags |= RGW_REST_WEBSITE; + } + } + + ldout(s->cct, 20) + << "subdomain=" << subdomain + << " domain=" << domain + << " in_hosted_domain=" << in_hosted_domain + << " in_hosted_domain_s3website=" << in_hosted_domain_s3website + << dendl; } } diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 60acbf700bb09..db417173a8dd1 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -151,7 +151,7 @@ class RGWPutObj_ObjStore : public RGWPutObj virtual int verify_params(); virtual int get_params(); - int get_data(bufferlist& bl); + int get_data(bufferlist& bl,MD5* hash= NULL); }; class RGWPostObj_ObjStore : public RGWPostObj @@ -188,6 +188,12 @@ class RGWCopyObj_ObjStore : public RGWCopyObj { ~RGWCopyObj_ObjStore() {} }; +class RGWRenameObj_ObjStore : public RGWRenameObj { +public: + RGWRenameObj_ObjStore() {} + ~RGWRenameObj_ObjStore() {} +}; + class RGWGetACLs_ObjStore : public RGWGetACLs { public: RGWGetACLs_ObjStore() {} @@ -303,7 +309,12 @@ class RGWHandler_ObjStore : public RGWHandler { public: RGWHandler_ObjStore() {} virtual ~RGWHandler_ObjStore() {} + int init_permissions(RGWOp *op); int read_permissions(RGWOp *op); + virtual int retarget(RGWOp *op, RGWOp **new_op) { + *new_op = op; + return 0; + } virtual int authorize() = 0; }; @@ -312,6 +323,7 @@ class RGWHandler_ObjStore_SWIFT; class RGWHandler_SWIFT_Auth; class RGWHandler_ObjStore_S3; + class RGWRESTMgr { bool should_log; protected: @@ -366,7 +378,8 @@ extern void end_header(struct req_state *s, RGWOp *op = NULL, const char *content_type = NULL, const int64_t proposed_content_length = NO_CONTENT_LENGTH, - bool force_content_type = false); + bool force_content_type = false, + bool force_no_error = false); extern void dump_start(struct req_state *s); extern void list_all_buckets_start(struct req_state *s); extern void dump_owner(struct req_state *s, string& id, string& name, const char *section = NULL); @@ -376,7 +389,7 @@ extern void dump_etag(struct req_state *s, const char *etag); extern void dump_epoch_header(struct req_state *s, const char *name, time_t t); extern void dump_time_header(struct req_state *s, const char *name, time_t t); extern void dump_last_modified(struct req_state *s, time_t t); -extern void abort_early(struct req_state *s, RGWOp *op, int err); +extern void abort_early(struct req_state *s, RGWOp *op, int err, RGWHandler* handler); extern void dump_range(struct req_state *s, uint64_t ofs, uint64_t end, uint64_t total_size); extern void dump_continue(struct req_state *s); extern void list_all_buckets_end(struct req_state *s); diff --git a/src/rgw/rgw_rest_metadata.cc b/src/rgw/rgw_rest_metadata.cc index 1068076a7761e..995585746f686 100644 --- a/src/rgw/rgw_rest_metadata.cc +++ b/src/rgw/rgw_rest_metadata.cc @@ -104,7 +104,7 @@ void RGWOp_Metadata_List::execute() { http_ret = 0; } -int RGWOp_Metadata_Put::get_data(bufferlist& bl) { +int RGWOp_Metadata_Put::get_data(bufferlist& bl,MD5* hash) { size_t cl = 0; char *data; int read_len; diff --git a/src/rgw/rgw_rest_metadata.h b/src/rgw/rgw_rest_metadata.h index 7f3cf1f2207ae..3affb1bdb27b9 100644 --- a/src/rgw/rgw_rest_metadata.h +++ b/src/rgw/rgw_rest_metadata.h @@ -39,7 +39,7 @@ class RGWOp_Metadata_Get : public RGWRESTOp { }; class RGWOp_Metadata_Put : public RGWRESTOp { - int get_data(bufferlist& bl); + int get_data(bufferlist& bl,MD5* hash= NULL); string update_status; obj_version ondisk_version; public: diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 0a6d67c0c626f..5ebd854342aa1 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3,6 +3,7 @@ #include #include +#include #include "common/ceph_crypto.h" #include "common/Formatter.h" @@ -11,6 +12,7 @@ #include "rgw_rest.h" #include "rgw_rest_s3.h" +#include "rgw_rest_s3website.h" #include "rgw_auth_s3.h" #include "rgw_policy_s3.h" #include "rgw_user.h" @@ -18,6 +20,8 @@ #include "rgw_cors_s3.h" #include "rgw_client_io.h" +#include // for 'typeid' + #define dout_subsys ceph_subsys_rgw using namespace ceph::crypto; @@ -72,17 +76,100 @@ static struct response_attr_param resp_attr_params[] = { {NULL, NULL}, }; +int RGWGetObj_ObjStore_S3Website::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { + map::iterator iter; + iter = attrs.find(RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION); + if (iter != attrs.end()) { + bufferlist &bl = iter->second; + s->redirect = string(bl.c_str(), bl.length()); + s->err.http_ret = 301; + ldout(s->cct, 20) << __CEPH_ASSERT_FUNCTION << " redirectng per x-dss-website-redirect-location=" << s->redirect << dendl; + ret = -ERR_WEBSITE_REDIRECT; + set_req_state_err(s, ret); + dump_errno(s); + dump_content_length(s, 0); + dump_redirect(s, s->redirect); + end_header(s, this); + return ret; + } else { + return RGWGetObj_ObjStore_S3::send_response_data(bl, bl_ofs, bl_len); + } +} + +int RGWGetObj_ObjStore_S3Website::send_response_data_error() +{ + return RGWGetObj_ObjStore_S3::send_response_data_error(); +} + +int RGWGetObj_ObjStore_S3::send_response_data_error() +{ + bufferlist bl; + return send_response_data(bl, 0 , 0); +} + int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { const char *content_type = NULL; string content_type_str; map response_attrs; map::iterator riter; - bufferlist metadata_bl; + bufferlist metadata_bl, decrypted_bl; + if (ret) + { + dout(0) << "SSEDebug Value of ret at entrance " << ret << " "<< bl.length() << dendl; + } + if (kmsdata) + { + unsigned char* read_data; + int decryptedtext_len,left_data,iter, full_chunks ; + uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size; + unsigned char* decryptedtext = new unsigned char[chunk_size]; + + const char* c_key = kmsdata->key_dec.c_str(); + const char* c_iv = kmsdata->iv_dec.c_str(); + + read_data = reinterpret_cast(bl.c_str()); + full_chunks = floor(((double)bl_len)/chunk_size); + dout(0) << "SSEINFO Total part number " << full_chunks << dendl; + for (iter=0; iter < full_chunks ; iter++) + { + decryptedtext_len = decrypt(read_data, chunk_size, (unsigned char*)c_key,(unsigned char*)c_iv,decryptedtext); + if (decryptedtext_len == -1) + { + dout(0) << " Error while decrypting " << dendl; + return -ERR_INTERNAL_ERROR; + } + read_data += chunk_size; + decrypted_bl.append((char*)decryptedtext, chunk_size); + dout(0) << "SSEINFO Doing for part number " << iter << dendl; + } + left_data = bl_len % chunk_size; + if (left_data > 15) + { + dout(0) << "SSEINFO Length of Encrypted text " << bl_len << " and key is " << c_key << " " << strlen(c_key) << " and iv " << c_iv << " " << strlen(c_iv) << dendl; + decryptedtext_len = decrypt(read_data, left_data, (unsigned char*)c_key,(unsigned char*)c_iv,decryptedtext); + if (decryptedtext_len == -1) + { + dout(0) << " Error while decrypting " << dendl; + delete [] decryptedtext; + return -ERR_INTERNAL_ERROR; + } + decrypted_bl.append((char*)decryptedtext, left_data); + string bufferprinter = ""; + decrypted_bl.copy(0, decrypted_bl.length(), bufferprinter); + //dout(0) << "SSEINFO Decrypted text " << bufferprinter << dendl; + } + else if (left_data > 0) + { + dout(0) << "SSEINFO Not decrypting tail" << dendl; + decrypted_bl.append((char*)read_data, left_data); + } + delete [] decryptedtext; + } + dout(0) << "SSEDebug done with enc dec" << dendl; if (ret) goto done; - if (sent_header) goto send_data; @@ -123,6 +210,9 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ } } + if (kmsdata) + s->cio->print("x-jcs-server-side-encryption: \"%s\"\r\n", "AES256"); + for (struct response_attr_param *p = resp_attr_params; p->param; p++) { bool exists; string val = s->info.args.get(p->param, &exists); @@ -159,9 +249,14 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ } done: - set_req_state_err(s, (partial_content && !ret) ? STATUS_PARTIAL_CONTENT : ret); - - dump_errno(s); + if (custom_http_ret) { + set_req_state_err(s, 0); + dump_errno(s, custom_http_ret); + } else { + set_req_state_err(s, (partial_content && !ret) ? STATUS_PARTIAL_CONTENT + : ret); + dump_errno(s); + } for (riter = response_attrs.begin(); riter != response_attrs.end(); ++riter) { s->cio->print("%s: %s\r\n", riter->first.c_str(), riter->second.c_str()); @@ -179,9 +274,16 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_ send_data: if (get_data && !ret) { - int r = s->cio->write(bl.c_str() + bl_ofs, bl_len); + int r; + if (kmsdata) + r = s->cio->write(decrypted_bl.c_str() + bl_ofs, bl_len); + else + r = s->cio->write(bl.c_str() + bl_ofs, bl_len); if (r < 0) + { + dout(0) << "SSEDebug failing while writing to io" << dendl; return r; + } } return 0; @@ -501,6 +603,78 @@ void RGWSetBucketVersioning_ObjStore_S3::send_response() end_header(s); } +int RGWSetBucketWebsite_ObjStore_S3::get_params() +{ +#define GET_BUCKET_WEBSITE_BUF_MAX (128 * 1024) + + char *data; + int len = 0; + int r = rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_WEBSITE_BUF_MAX); + if (r < 0) { + return r; + } + + bufferlist bl; + bl.append(data, len); + + RGWXMLDecoder::XMLParser parser; + parser.init(); + + if (!parser.parse(data, len, 1)) { + string str(data, len); + ldout(s->cct, 5) << "failed to parse xml: " << str << dendl; + return -EINVAL; + } + + try { + RGWXMLDecoder::decode_xml("WebsiteConfiguration", website_conf, &parser, true); + } catch (RGWXMLDecoder::err& err) { + string str(data, len); + ldout(s->cct, 5) << "unexpected xml: " << str << dendl; + return -EINVAL; + } + + return 0; +} + +void RGWSetBucketWebsite_ObjStore_S3::send_response() +{ + if (ret) + set_req_state_err(s, ret); + dump_errno(s); + end_header(s); +} + +void RGWDeleteBucketWebsite_ObjStore_S3::send_response() +{ + if (!ret) { + ret = STATUS_NO_CONTENT; + } + set_req_state_err(s, ret); + dump_errno(s); + end_header(s); +} + +void RGWGetBucketWebsite_ObjStore_S3::send_response() +{ + if (ret) + set_req_state_err(s, ret); + dump_errno(s); + end_header(s, this, "application/xml"); + dump_start(s); + + if (ret < 0) { + return; + } + + RGWBucketWebsiteConf& conf = s->bucket_info.website_conf; + + s->formatter->open_object_section_in_ns("WebsiteConfiguration", + "http://doc.s3.amazonaws.com/doc/2006-03-01/"); + conf.dump_xml(s->formatter); + s->formatter->close_section(); // WebsiteConfiguration + rgw_flush_formatter_and_reset(s, s->formatter); +} static void dump_bucket_metadata(struct req_state *s, RGWBucketEnt& bucket) { @@ -1134,6 +1308,21 @@ int RGWPostObj_ObjStore_S3::get_params() attrs[attr_name] = attr_bl; } + // TODO: refactor this and the above loop to share code + piter = parts.find(RGW_AMZ_WEBSITE_REDIRECT_LOCATION); + if (piter != parts.end()) { + string n = piter->first; + string attr_name = RGW_ATTR_PREFIX; + attr_name.append(n); + /* need to null terminate it */ + bufferlist& data = piter->second.data; + string str = string(data.c_str(), data.length()); + + bufferlist attr_bl; + attr_bl.append(str.c_str(), str.size() + 1); + + attrs[attr_name] = attr_bl; + } int r = get_policy(); if (r < 0) @@ -1190,18 +1379,20 @@ int RGWPostObj_ObjStore_S3::get_policy() dout(1) << "DSS Error: " << errmsg << dendl; return -EACCES; } - dout(0) << "DSS INFO: Sending Action to validate: " << resource_info.getAction() << dendl; - dout(0) << "DSS INFO: Sending Resource to validate: " << resource_info.getResourceName() << dendl; - dout(0) << "DSS INFO: Sending Tenant to validate: " << resource_info.getTenantName() << dendl; + dout(1) << "DSS API LOGGING: Action="<< resource_info.getAction() <<" Resource="<< resource_info.getResourceName() << " Tenant=" << resource_info.getTenantName() << dendl; + string source_ip = s->info.env->get("HTTP_X_FORWARDED_FOR","0.0.0.0"); + keystone_validator.append_header("X-Forwarded-For",source_ip); if (isTokenBasedAuth) { keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), resource_info.getTenantName(), + source_ip, false, /* Is sign auth */ false, /* Is copy */ false, /* Is cross account */ (s->auth_method).get_url_type_token(), + (s->auth_method).get_infinite_url_type_token(), resource_info.getCopySrc(), (s->auth_method).get_token(), "", /* Access key*/ @@ -1214,10 +1405,12 @@ int RGWPostObj_ObjStore_S3::get_policy() keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), resource_info.getTenantName(), + source_ip, true, /* Is sign auth */ false, /* Is copy */ false, /* Is cross account */ (s->auth_method).get_url_type_token(), + (s->auth_method).get_infinite_url_type_token(), resource_info.getCopySrc(), "", /* Token string */ s3_access_key, /* Access key */ @@ -1238,7 +1431,7 @@ int RGWPostObj_ObjStore_S3::get_policy() return -EACCES; } user_info.user_id = keystone_validator.response.token.tenant.id; - user_info.display_name = keystone_validator.response.token.tenant.id; //<<<<<< DSS needs tenant.name + user_info.display_name = keystone_validator.response.token.tenant.id; /* try to store user if it not already exists */ if (rgw_get_user_info_by_uid(store, keystone_validator.response.token.tenant.id, user_info) < 0) { int ret = rgw_store_user_info(store, user_info, NULL, NULL, 0, true); @@ -1363,7 +1556,7 @@ int RGWPostObj_ObjStore_S3::complete_get_params() return 0; } -int RGWPostObj_ObjStore_S3::get_data(bufferlist& bl) +int RGWPostObj_ObjStore_S3::get_data(bufferlist& bl,MD5* hash) { bool boundary; bool done; @@ -1468,7 +1661,6 @@ void RGWPostObj_ObjStore_S3::send_response() rgw_flush_formatter_and_reset(s, s->formatter); } - void RGWDeleteObj_ObjStore_S3::send_response() { int r = ret; @@ -1619,6 +1811,16 @@ int RGWPutACLs_ObjStore_S3::get_policy_from_state(RGWRados *store, struct req_st return 0; } +void RGWRenameObj_ObjStore_S3::send_response() +{ + ret = s->err.ret; + if (ret) + set_req_state_err(s, ret); + dump_errno(s); + end_header(s, this, "application/xml"); + dump_start(s); +} + void RGWPutACLs_ObjStore_S3::send_response() { if (ret) @@ -2012,6 +2214,7 @@ RGWOp *RGWHandler_ObjStore_Service_S3::op_head() RGWOp *RGWHandler_ObjStore_Bucket_S3::get_obj_op(bool get_data) { + // Non-website mode if (get_data) return new RGWListBucket_ObjStore_S3; else @@ -2029,6 +2232,13 @@ RGWOp *RGWHandler_ObjStore_Bucket_S3::op_get() if (s->info.args.sub_resource_exists("versioning")) return new RGWGetBucketVersioning_ObjStore_S3; + if (s->info.args.sub_resource_exists("website")) { + if (!s->cct->_conf->rgw_enable_static_website) { + return NULL; + } + return new RGWGetBucketWebsite_ObjStore_S3; + } + if (is_acl_op()) { return new RGWGetACLs_ObjStore_S3; } else if (is_cors_op()) { @@ -2055,6 +2265,12 @@ RGWOp *RGWHandler_ObjStore_Bucket_S3::op_put() return NULL; if (s->info.args.sub_resource_exists("versioning")) return new RGWSetBucketVersioning_ObjStore_S3; + if (s->info.args.sub_resource_exists("website")) { + if (!s->cct->_conf->rgw_enable_static_website) { + return NULL; + } + return new RGWSetBucketWebsite_ObjStore_S3; + } if (is_acl_op()) { return new RGWPutACLs_ObjStore_S3; } else if (is_cors_op()) { @@ -2068,6 +2284,14 @@ RGWOp *RGWHandler_ObjStore_Bucket_S3::op_delete() if (is_cors_op()) { return new RGWDeleteCORS_ObjStore_S3; } + + if (s->info.args.sub_resource_exists("website")) { + if (!s->cct->_conf->rgw_enable_static_website) { + return NULL; + } + return new RGWDeleteBucketWebsite_ObjStore_S3; + } + return new RGWDeleteBucket_ObjStore_S3; } @@ -2120,6 +2344,9 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_put() if (is_acl_op()) { return new RGWPutACLs_ObjStore_S3; } + if (store->ctx()->_conf->rgw_enable_rename_op && is_rename_op()) { + return new RGWRenameObj_ObjStore_S3; + } if (!s->copy_source) return new RGWPutObj_ObjStore_S3; else @@ -2372,10 +2599,12 @@ int RGWHandler_ObjStore_S3::init(RGWRados *store, struct req_state *s, RGWClient int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, const string& resource_name, const string& tenant_name, + const string& source_ip, const bool& is_sign_auth, const bool& is_copy, const bool& is_cross_account, const bool& is_url_token, + const bool& is_infini_url_token, const string& copy_src, const string& token, const string& auth_id, @@ -2393,14 +2622,26 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, bool is_non_rc_action = false; is_non_rc_action = ((localAction.compare("CreateBucket") == 0) || (localAction.compare("ListAllMyBuckets") == 0) - || is_url_token); - + || is_url_token + || is_infini_url_token); + + /* Infinite URLs should only be used for GET */ + if (is_infini_url_token && !( + (localAction.compare("ListBucket") == 0) || + (localAction.compare("GetObject") == 0) || + (localAction.compare("ListAllMyBuckets") == 0) )) { + dout(0) << "DSS INFO: Infinite URL only works for GET. Received local action: " + << localAction << dendl; + return -ENOTRECOVERABLE; + } /* Set required headers for keystone request * Recursive calls already have headers set */ if (!is_copy && !is_cross_account) { if (!is_sign_auth) { if (is_url_token) { append_header("X-Url-Token", token); + } else if (is_infini_url_token) { + append_header("X-Preauth-Token", token); } else { append_header("X-Auth-Token", token); } @@ -2408,6 +2649,7 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, append_header("Content-Type", "application/json"); } + append_header("X-Forwarded-For",source_ip); /* Handle special case of copy */ bool isCopyAction = false; isCopyAction = (localAction.compare("CopyObject") == 0); @@ -2419,8 +2661,8 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, string copy_src_str = copy_src.substr(0, pos); string copy_src_tenant = copy_src.substr(pos + 1); dout(0) << "DSS INFO: Validating for copy source" << dendl; - ret = validate_request(localAction, copy_src_str, copy_src_tenant, is_sign_auth, - true, is_cross_account, is_url_token, copy_src, + ret = validate_request(localAction, copy_src_str, copy_src_tenant, source_ip, is_sign_auth, + true, is_cross_account, is_url_token, is_infini_url_token, copy_src, token, auth_id, auth_token, auth_sign, objectname, iamerror); if (ret < 0) { return ret; @@ -2430,8 +2672,8 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, } } - // For cross account call, get root account ID - if (is_cross_account) { + // For cross account call and infinite time URL, get root account ID + if (is_cross_account || is_infini_url_token) { if (tenant_name.size() > 0) { rootAccount = tenant_name; } else { @@ -2464,6 +2706,9 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, } else if (is_url_token) { keystone_url.append(cct->_conf->rgw_keystone_url_token_api); dout(0) << "DSS INFO: Validating presigned URL token" << dendl; + } else if (is_infini_url_token) { + keystone_url.append(cct->_conf->rgw_keystone_infinite_url_token_api); + dout(0) << "DSS INFO: Validating infinite presigned URL token" << dendl; } else { keystone_url.append(cct->_conf->rgw_keystone_token_api); if (is_cross_account) { @@ -2499,6 +2744,8 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, credentials.dump_string("resource", resource_str.c_str()); if (is_url_token) { credentials.dump_string("object_name", objectname.c_str()); + } else if (is_infini_url_token) { + credentials.dump_string("object_id", objectname.c_str()); } else { credentials.dump_string("implicit_allow", implicit_allow.c_str()); } @@ -2514,13 +2761,11 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, string bufferprinter = ""; tx_buffer.copy(0, tx_buffer.length(), bufferprinter); dout(0) << "DSS INFO: \n\n" << dendl; - dout(0) << "DSS INFO: Outbound json: " << os.str() << dendl; - dout(0) << "DSS INFO: \n\n" << dendl; - dout(0) << "DSS INFO: Actual TX buffer: " << bufferprinter << dendl; + dout(0) << "DSS INFO: TX buffer: " << bufferprinter << dendl; dout(0) << "DSS INFO: \n\n" << dendl; /* Make request to IAM */ - ret = make_iam_request(keystone_url, iamerror); + ret = make_iam_request(keystone_url, iamerror, rootAccount, is_infini_url_token); if (ret < 0) { if (is_cross_account) { // If a cross account call has failed, @@ -2549,15 +2794,20 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, return 0; } + if (tenant_name.empty()) { + dout(0) << "DSS INFO: tenant_name is empty, returning and Continuing ..." << dendl; + return 0; + } + /* Check root account ID of the caller against resource name */ string keystoneTenant = response.token.tenant.id; if (!is_non_rc_action && (keystoneTenant.compare(tenant_name) != 0)) { // This case requires cross account validation. // Make recursive call with is_cross_account set to true dout(0) << "DSS INFO: Validating for cross account access" << dendl; - ret = validate_request(localAction, resource_name, tenant_name, + ret = validate_request(localAction, resource_name, tenant_name, source_ip, is_sign_auth, is_copy, true, - is_url_token, copy_src, token, auth_id, + is_url_token, is_infini_url_token, copy_src, token, auth_id, auth_token, auth_sign, objectname, iamerror); if (ret < 0) { return ret; @@ -2573,7 +2823,10 @@ int RGW_Auth_S3_Keystone_ValidateToken::validate_request(const string& action, /* Make the CURL call to IAM * Call to this function requires tx_buffer to be set beforehand */ -int RGW_Auth_S3_Keystone_ValidateToken::make_iam_request(const string& keystone_url, string& iamerror) +int RGW_Auth_S3_Keystone_ValidateToken::make_iam_request(const string& keystone_url, + string& iamerror, + const string& rootAccount, + const bool& is_infini_url_token) { /* Clear the buffers */ rx_buffer.clear(); @@ -2596,6 +2849,18 @@ int RGW_Auth_S3_Keystone_ValidateToken::make_iam_request(const string& keystone_ dout(0) << "DSS INFO: Printing RX buffer: " << bufferprinter << dendl; dout(0) << "DSS INFO: Printing RX headers: " << bufferheaderprinter << dendl; + /* This is a horrible hack. The scope of infinite presigned URLs needs discussion */ + int noContentPos = bufferheaderprinter.find("204 No Content"); + if (is_infini_url_token && !bufferprinter.size() && (noContentPos > 0)) { + // This is the case when IAM has sent no body and 204 No content + // Fill up the root account ID we have on the resource as the user + // and return success + dout(0) << "DSS INFO: Received no content from IAM for infinite URL validation" << dendl; + dout(0) << "DSS INFO: This is expected IAM response. Proceeding." << dendl; + dout(10) << "DSS INFO: nocontentpos is " << noContentPos << dendl; + response.token.tenant.id = rootAccount; + return 0; + } /* Populate iamerror */ char *rxbuffer = strdup(bufferprinter.c_str()); @@ -2610,7 +2875,7 @@ int RGW_Auth_S3_Keystone_ValidateToken::make_iam_request(const string& keystone_ // assuming error from IAM is always in this format // {"error": {"message": "The resource could not be found.", "code": 404, "title": "Not Found"}} if(tokens.size() >= 5 && !strcmp(tokens[1].c_str(), "error") && !strcmp(tokens[3].c_str(), "message")) { - iamerror = "IAM_ERROR: " + tokens[5]; + iamerror = tokens[5]; } free(rxbuffer); @@ -2671,6 +2936,17 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) } } + // check for infinite time token in presigned URL requests + if(!isTokenBasedAuth && store->ctx()->_conf->rgw_enable_infinite_token_based_presigned_url) { + string url_token = s->info.args.get("X-Preauth-Token"); + if(url_token.size() > 0) { + isTokenBasedAuth = true; + (s->auth_method).set_token_validation(true); + (s->auth_method).set_token(url_token); + (s->auth_method).set_infinite_url_type_token(true); + } + } + // Block any ACL request for DSS string qstring = (s->info).request_params; if (store->ctx()->_conf->rgw_disable_acl_api && (qstring.compare("acl") == 0)) { @@ -2678,6 +2954,20 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) return -EPERM; } + // Block rename op for illegal cases + if (s->info.args.exists("newname")) { + if (!(store->ctx()->_conf->rgw_enable_rename_op)) { + return -ERR_RENAME_NOT_ENABLED; + } + if (s->op == OP_PUT) { + if ((s->object).name.empty()) { + return -ERR_BAD_RENAME_REQ; + } + } else { + return -ERR_BAD_RENAME_REQ; + } + } + /* neither keystone and rados enabled; warn and exit! */ if (!store->ctx()->_conf->rgw_s3_auth_use_rados && !store->ctx()->_conf->rgw_s3_auth_use_keystone) { @@ -2707,10 +2997,15 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) qsr = true; } else { /* anonymous access */ - //<<<<<< You will hit here for sign based req - //<<<<<< Add changes for anonymous access. Call a func from here. + + bool is_s3website = (s->prot_flags & RGW_REST_WEBSITE); + if (is_s3website) + { init_anon_user(s); return 0; + } + else + return -EPERM; } } else { // strncmp returns 0 on match. If even one of AWS or JCS match, dont return -EINVAL. @@ -2758,19 +3053,22 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) if (s != NULL) { resource_object_name = s->object.name; } - dout(0) << "DSS INFO: Sending Action to validate: " << resource_info.getAction() << dendl; - dout(0) << "DSS INFO: Sending Resource to validate: " << resource_info.getResourceName() << dendl; - dout(0) << "DSS INFO: Sending Tenant to validate: " << resource_info.getTenantName() << dendl; - dout(0) << "DSS INFO: Sending Object to validate: " << resource_object_name << dendl; - + string source_ip = s->info.env->get("HTTP_X_FORWARDED_FOR","0.0.0.0"); + dout(1) << "DSS API LOGGING: Action=" + << resource_info.getAction() + << " Resource="<< resource_info.getResourceName() + << " Tenant=" << resource_info.getTenantName() + << " Object=" << resource_object_name << dendl; if (isTokenBasedAuth) { keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), resource_info.getTenantName(), + source_ip, false, /* Is sign auth */ false, /* Is copy */ false, /* Is cross account */ (s->auth_method).get_url_type_token(), + (s->auth_method).get_infinite_url_type_token(), resource_info.getCopySrc(), (s->auth_method).get_token(), "", /* Access key*/ @@ -2778,15 +3076,17 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) "", /* Received signature */ resource_object_name, iamerror); - + } else { keystone_result = keystone_validator.validate_request(resource_info.getAction(), resource_info.getResourceName(), resource_info.getTenantName(), + source_ip, true, /* Is sign auth */ false, /* Is copy */ false, /* Is cross account */ (s->auth_method).get_url_type_token(), + (s->auth_method).get_infinite_url_type_token(), resource_info.getCopySrc(), "", /* Token string */ auth_id, /* Access key */ @@ -2824,6 +3124,7 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) s->user.user_id = tenant_id_str; s->user.display_name = tenant_id_str; (s->auth_method).set_acl_main_override(true); + s->bucket_owner_id = resource_info.getTenantName(); /* try to store user if it not already exists */ if (rgw_get_user_info_by_uid(store, keystone_validator.response.token.tenant.id, s->user) < 0) { @@ -2920,6 +3221,7 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s) s->user = effective_user; } } + s->bucket_owner_id = s->user.user_id; } /* if keystone_result < 0 */ @@ -2941,17 +3243,237 @@ int RGWHandler_Auth_S3::init(RGWRados *store, struct req_state *state, RGWClient RGWHandler *RGWRESTMgr_S3::get_handler(struct req_state *s) { - int ret = RGWHandler_ObjStore_S3::init_from_header(s, RGW_FORMAT_XML, false); + bool is_s3website = enable_s3website && (s->prot_flags & RGW_REST_WEBSITE); + int ret = RGWHandler_ObjStore_S3::init_from_header(s, is_s3website ? RGW_FORMAT_HTML : RGW_FORMAT_XML, false); if (ret < 0) return NULL; - if (s->bucket_name_str.empty()) - return new RGWHandler_ObjStore_Service_S3; + RGWHandler* handler; + // TODO: Make this more readable + if (is_s3website) { + if (s->bucket_name_str.empty()) { + handler = new RGWHandler_ObjStore_Service_S3Website; + } else if (s->object.empty()) { + handler = new RGWHandler_ObjStore_Bucket_S3Website; + } else { + handler = new RGWHandler_ObjStore_Obj_S3Website; + } + } else { + if (s->bucket_name_str.empty()) { + handler = new RGWHandler_ObjStore_Service_S3; + } else if (s->object.empty()) { + handler = new RGWHandler_ObjStore_Bucket_S3; + } else { + handler = new RGWHandler_ObjStore_Obj_S3; + } + } + + ldout(s->cct, 20) << __func__ << " handler=" << typeid(*handler).name() << dendl; + return handler; +} + +int RGWHandler_ObjStore_S3Website::retarget(RGWOp *op, RGWOp **new_op) { + *new_op = op; + ldout(s->cct, 10) << __func__ << "Starting retarget" << dendl; + + if (!(s->prot_flags & RGW_REST_WEBSITE)) + return 0; + + RGWObjectCtx& obj_ctx = *static_cast(s->obj_ctx); + int ret = store->get_bucket_info(obj_ctx, s->bucket_name_str, s->bucket_info, NULL, &s->bucket_attrs); + if (ret < 0) { + // TODO-FUTURE: if the bucket does not exist, maybe expose it here? + return -ERR_NO_SUCH_BUCKET; + } + if (!s->bucket_info.has_website) { + // TODO-FUTURE: if the bucket has no WebsiteConfig, expose it here + return -ERR_NO_SUCH_WEBSITE_CONFIGURATION; + } + + rgw_obj_key new_obj; + s->bucket_info.website_conf.get_effective_key(s->object.name, &new_obj.name); + ldout(s->cct, 10) << "retarget get_effective_key " << s->object << " -> " << new_obj << dendl; + + RGWBWRoutingRule rrule; + bool should_redirect = s->bucket_info.website_conf.should_redirect(new_obj.name, 0, &rrule); + + if (should_redirect) { + const string& hostname = s->info.env->get("HTTP_HOST", ""); + const string& protocol = (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http"); + int redirect_code = 0; + rrule.apply_rule(protocol, hostname, s->object.name, &s->redirect, &redirect_code); + // APply a custom HTTP response code + if (redirect_code > 0) + s->err.http_ret = redirect_code; // Apply a custom HTTP response code + ldout(s->cct, 10) << "retarget redirect code=" << redirect_code << " proto+host:" << protocol << "://" << hostname << " -> " << s->redirect << dendl; + return -ERR_WEBSITE_REDIRECT; + } + + /* + * FIXME: if s->object != new_obj, drop op and create a new op to handle operation. Or + * remove this comment if it's not applicable anymore + */ + + s->object = new_obj; + + return 0; +} + +RGWOp *RGWHandler_ObjStore_S3Website::op_get() +{ + return get_obj_op(true); +} + +RGWOp *RGWHandler_ObjStore_S3Website::op_head() +{ + return get_obj_op(false); +} + +int RGWHandler_ObjStore_S3Website::serve_errordoc(int http_ret, const string& errordoc_key) { + int ret = 0; + s->formatter->reset(); /* Try to throw it all away */ + + RGWGetObj_ObjStore_S3Website* getop = (RGWGetObj_ObjStore_S3Website*) op_get(); + if(!getop) { + return -1; // Trigger double error handler + } + getop->init(store, s, this); + /*getop->range_str = NULL; + getop->if_mod = NULL; + getop->if_unmod = NULL; + getop->if_match = NULL; + getop->if_nomatch = NULL;*/ + s->object = errordoc_key; + + ret = init_permissions(getop); + if (ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, init_permissions ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + ret = read_permissions(getop); + if (ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, read_permissions ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + if (http_ret) { + getop->set_custom_http_response(http_ret); + } + + ret = getop->init_processing(); + if (ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, init_processing ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + ret = getop->verify_op_mask(); + if (ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, verify_op_mask ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + ret = getop->verify_permission(); + if (ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, verify_permission ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + ret = getop->verify_params(); + if (ret < 0) { + ldout(s->cct, 20) << "serve_errordoc failed, verify_params ret=" << ret << dendl; + return -1; // Trigger double error handler + } + + // No going back now + getop->pre_exec(); + /* + * FIXME Missing headers: + * With a working errordoc, the s3 error fields are rendered as HTTP headers, + * x-amz-error-code: NoSuchKey + * x-amz-error-message: The specified key does not exist. + * x-amz-error-detail-Key: foo + */ + getop->execute(); + getop->complete(); + return 0; - if (s->object.empty()) - return new RGWHandler_ObjStore_Bucket_S3; +} + +int RGWHandler_ObjStore_S3Website::error_handler(int err_no, + string* error_content) { + int new_err_no = -1; + const struct rgw_http_errors* r; + int http_error_code = -1; + r = search_err(err_no > 0 ? err_no : -err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS)); + if (r) { + http_error_code = r->http_ret; + } + ldout(s->cct, 10) << "RGWHandler_ObjStore_S3Website::error_handler err_no=" << err_no << " http_ret=" << http_error_code << dendl; + + RGWBWRoutingRule rrule; + bool should_redirect = s->bucket_info.website_conf.should_redirect(s->object.name, http_error_code, &rrule); + + if (should_redirect) { + const string& hostname = s->info.env->get("HTTP_HOST", ""); + const string& protocol = (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http"); + int redirect_code = 0; + rrule.apply_rule(protocol, hostname, s->object.name, &s->redirect, &redirect_code); + // APply a custom HTTP response code + if (redirect_code > 0) + s->err.http_ret = redirect_code; // Apply a custom HTTP response code + ldout(s->cct, 10) << "error handler redirect code=" << redirect_code << " proto+host:" << protocol << "://" << hostname << " -> " << s->redirect << dendl; + return -ERR_WEBSITE_REDIRECT; + } else if (err_no == -ERR_WEBSITE_REDIRECT) { + // Do nothing here, this redirect will be handled in abort_early's ERR_WEBSITE_REDIRECT block + // Do NOT fire the ErrorDoc handler + } else if (!s->bucket_info.website_conf.error_doc.empty()) { + /* This serves an entire page! + On success, it will return zero, and no further content should be sent to the socket + On failure, we need the double-error handler + */ + new_err_no = RGWHandler_ObjStore_S3Website::serve_errordoc(http_error_code, s->bucket_info.website_conf.error_doc); + if(new_err_no && new_err_no != -1) { + err_no = new_err_no; + } + } else { + ldout(s->cct, 20) << "No special error handling today!" << dendl; + } + + return err_no; +} + +RGWOp *RGWHandler_ObjStore_Obj_S3Website::get_obj_op(bool get_data) +{ + /** If we are in website mode, then it is explicitly impossible to run GET or + * HEAD on the actual directory. We must convert the request to run on the + * suffix object instead! + */ + RGWGetObj_ObjStore_S3Website *op = new RGWGetObj_ObjStore_S3Website; + op->set_get_data(get_data); + return op; +} + +RGWOp *RGWHandler_ObjStore_Bucket_S3Website::get_obj_op(bool get_data) +{ + /** If we are in website mode, then it is explicitly impossible to run GET or + * HEAD on the actual directory. We must convert the request to run on the + * suffix object instead! + */ + RGWGetObj_ObjStore_S3Website *op = new RGWGetObj_ObjStore_S3Website; + op->set_get_data(get_data); + return op; +} - return new RGWHandler_ObjStore_Obj_S3; +RGWOp *RGWHandler_ObjStore_Service_S3Website::get_obj_op(bool get_data) +{ + /** If we are in website mode, then it is explicitly impossible to run GET or + * HEAD on the actual directory. We must convert the request to run on the + * suffix object instead! + */ + RGWGetObj_ObjStore_S3Website *op = new RGWGetObj_ObjStore_S3Website; + op->set_get_data(get_data); + return op; } /* @@ -3161,3 +3683,114 @@ bool RGWResourceKeystoneInfo::get_bucket_public_perm(const string& action, reason = "OK"; return true; } + +// Make a kms request for encrypted as well as decrypted key/iv +// We will use encrypted keys to store in xattr +// Decrypted keys,iv will be used to encode data +int RGW_KMS::make_kms_encrypt_request(string &root_account, RGWKmsData* kmsdata) +{ + string kms_url = cct->_conf->rgw_kms_encrypt_url; + if (kms_url[kms_url.size() -1] != '/') { + kms_url.append("?"); + } + else + kms_url[kms_url.size() -1] = '?'; + + kms_url.append("user_id="); + kms_url.append(root_account); + dout(0)<< "SSEINFO Final KMS URL " << kms_url << dendl; + int ret = 1 ; + + string empty ; + set_tx_buffer(empty); + utime_t begin_time = ceph_clock_now(g_ceph_context); + ret = process("GET", kms_url.c_str()); + utime_t end_time = ceph_clock_now(g_ceph_context); + end_time = end_time - begin_time; + dout(0) << "SSEINFO: KMS Encrypt response time (milliseconds): " << end_time.to_msec() << dendl; + + if (ret < 0) + { + ret = -ERR_INTERNAL_ERROR; + dout(0) << " Unable to obtain encryped and decrypted keys from KMS "<< dendl; + return ret; + } + + //string bufferprinter = ""; + //rx_buffer.copy(0, rx_buffer.length(), bufferprinter); + //dout(0) << "SSEINFO Printing RX buffer: " << bufferprinter << dendl; + + ret = kmsdata->decode_json_enc(rx_buffer, cct); + if (ret < 0) + { + ret = -ERR_INTERNAL_ERROR; + return ret; + } + + if (kmsdata->key_dec.size() != 64 || kmsdata->iv_dec.size() != 16) + { + dout(0) << "SSEINFO KMS Key Size " << kmsdata->key_dec.size() << " or IV Size not right" << " " << kmsdata->iv_dec.size() << dendl; + ret = -ERR_INTERNAL_ERROR; + return ret; + } + + dout(0) << " SSEINFO After parsing " << kmsdata->key_dec << " & " << kmsdata->iv_dec << dendl; + return 1; +} + + +// Make a kms call for decrypted key/iv +// We extract encrypted keys from xattr +// Decrypted keys,iv will be used to decode data +int RGW_KMS::make_kms_decrypt_request(string &root_account, RGWKmsData* kmsdata) +{ + string kms_url = cct->_conf->rgw_kms_decrypt_url; + if (kms_url[kms_url.size()] -1 != '/') { + kms_url.append("?"); + } + kms_url.append("user_id="); + kms_url.append(root_account); + kms_url.append("&encrypted_data_key="); + kms_url.append(kmsdata->key_enc); + kms_url.append("&encrypted_data_iv="); + kms_url.append(kmsdata->iv_enc); + kms_url.append("&encryptedMKVersionId="); + kms_url.append(kmsdata->mkey_enc); + kms_url.append("&randomId="); + int id = rand() % 900000 + 100000; + char* cid = new char[7]; + sprintf(cid,"%d",id); + kms_url.append(cid); + dout(0)<< "SSEINFO Final URL For Decoding KMS" << kms_url << dendl; + string empty ; + set_tx_buffer(empty); + int ret = 1; + + utime_t begin_time = ceph_clock_now(g_ceph_context); + ret = process("GET", kms_url.c_str()); + utime_t end_time = ceph_clock_now(g_ceph_context); + end_time = end_time - begin_time; + + dout(0) << "SSEINFO: KMS Decrypt response time (milliseconds): " << end_time.to_msec() << dendl; + if (ret < 0) + { + ret = -ERR_INTERNAL_ERROR; + dout(0) << " Unable to obtain encryped and decrypted keys from KMS "<< dendl; + return ret; + } + + string bufferprinter = ""; + rx_buffer.copy(0, rx_buffer.length(), bufferprinter); + dout(0) << "SSEINFO Printing RX buffer: " << bufferprinter << dendl; + + ret = kmsdata->decode_json_dec(rx_buffer, cct); + if (ret < 0) + { + ret = -ERR_INTERNAL_ERROR; + return ret; + } + + dout(0) << " SSEINFO After parsing " << kmsdata->key_dec << " & " << kmsdata->iv_dec << dendl; + + return 0; +} diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index b0bf56bbf565b..a060bb2473d9b 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -20,11 +20,17 @@ void rgw_get_errno_s3(struct rgw_http_errors *e, int err_no); class RGWGetObj_ObjStore_S3 : public RGWGetObj_ObjStore { +protected: + // Serving a custom error page from an object is really a 200 response with + // just the status line altered. + int custom_http_ret = 0; public: RGWGetObj_ObjStore_S3() {} ~RGWGetObj_ObjStore_S3() {} + int send_response_data_error(); int send_response_data(bufferlist& bl, off_t ofs, off_t len); + void set_custom_http_response(int http_ret) { custom_http_ret = http_ret; } }; class RGWListBuckets_ObjStore_S3 : public RGWListBuckets_ObjStore { @@ -86,6 +92,31 @@ class RGWSetBucketVersioning_ObjStore_S3 : public RGWSetBucketVersioning { void send_response(); }; +class RGWGetBucketWebsite_ObjStore_S3 : public RGWGetBucketWebsite { +public: + RGWGetBucketWebsite_ObjStore_S3() {} + ~RGWGetBucketWebsite_ObjStore_S3() {} + + void send_response(); +}; + +class RGWSetBucketWebsite_ObjStore_S3 : public RGWSetBucketWebsite { +public: + RGWSetBucketWebsite_ObjStore_S3() {} + ~RGWSetBucketWebsite_ObjStore_S3() {} + + int get_params(); + void send_response(); +}; + +class RGWDeleteBucketWebsite_ObjStore_S3 : public RGWDeleteBucketWebsite { +public: + RGWDeleteBucketWebsite_ObjStore_S3() {} + ~RGWDeleteBucketWebsite_ObjStore_S3() {} + + void send_response(); +}; + class RGWStatBucket_ObjStore_S3 : public RGWStatBucket_ObjStore { public: RGWStatBucket_ObjStore_S3() {} @@ -164,7 +195,7 @@ class RGWPostObj_ObjStore_S3 : public RGWPostObj_ObjStore { int get_params(); int complete_get_params(); void send_response(); - int get_data(bufferlist& bl); + int get_data(bufferlist& bl,MD5* hash= NULL); }; class RGWDeleteObj_ObjStore_S3 : public RGWDeleteObj_ObjStore { @@ -187,6 +218,14 @@ class RGWCopyObj_ObjStore_S3 : public RGWCopyObj_ObjStore { void send_response(); }; +class RGWRenameObj_ObjStore_S3 : public RGWRenameObj_ObjStore { + public: + RGWRenameObj_ObjStore_S3() {} + ~RGWRenameObj_ObjStore_S3() {} + + void send_response(); +}; + class RGWGetACLs_ObjStore_S3 : public RGWGetACLs_ObjStore { public: RGWGetACLs_ObjStore_S3() {} @@ -344,10 +383,12 @@ class RGW_Auth_S3_Keystone_ValidateToken : public RGWHTTPClient { int validate_request(const string& action, const string& resource_name, const string& tenant_name, + const string& source_ip, const bool& is_sign_auth, const bool& is_copy, const bool& is_cross_account, const bool& is_url_token, + const bool& is_infini_url_token, const string& copy_src, const string& token, const string& auth_id, @@ -356,7 +397,10 @@ class RGW_Auth_S3_Keystone_ValidateToken : public RGWHTTPClient { const string& objectname, string& iamerror); - int make_iam_request(const string& keystone_url, string& iamerror); + int make_iam_request(const string& keystone_url, + string& iamerror, + const string& rootAccount, + const bool& is_infini_url_token); }; class RGW_Auth_S3 { @@ -396,6 +440,10 @@ class RGWHandler_ObjStore_S3 : public RGWHandler_ObjStore { virtual int authorize() { return RGW_Auth_S3::authorize(store, s); } + virtual int retarget(RGWOp *op, RGWOp **new_op) { + *new_op = op; + return 0; + } }; class RGWHandler_ObjStore_Service_S3 : public RGWHandler_ObjStore_S3 { @@ -442,6 +490,9 @@ class RGWHandler_ObjStore_Obj_S3 : public RGWHandler_ObjStore_S3 { bool is_obj_update_op() { return is_acl_op(); } + bool is_rename_op() { + return s->info.args.exists("newname"); + } RGWOp *get_obj_op(bool get_data); RGWOp *op_get(); @@ -456,8 +507,10 @@ class RGWHandler_ObjStore_Obj_S3 : public RGWHandler_ObjStore_S3 { }; class RGWRESTMgr_S3 : public RGWRESTMgr { +private: + bool enable_s3website; public: - RGWRESTMgr_S3() {} + RGWRESTMgr_S3(bool enable_s3website) : enable_s3website(false) { this->enable_s3website = enable_s3website; } virtual ~RGWRESTMgr_S3() {} virtual RGWRESTMgr *get_resource_mgr(struct req_state *s, const string& uri) { @@ -571,10 +624,62 @@ class RGWResourceKeystoneInfo { string& reason); }; +class RGW_KMS: public RGWHTTPClient { +private: + bufferlist rx_buffer; + bufferlist rx_headers_buffer; + bufferlist tx_buffer; + bufferlist::iterator tx_buffer_it; + +private: + void set_tx_buffer(const string& d) { + tx_buffer.clear(); + tx_buffer.append(d); + tx_buffer_it = tx_buffer.begin(); + set_send_length(tx_buffer.length()); + } + +public: + RGW_KMS(CephContext *_cct) + : RGWHTTPClient(_cct) { + } + + int receive_header(void *ptr, size_t len) { + rx_headers_buffer.append((char *)ptr, len); + return 0; + } + int receive_data(void *ptr, size_t len) { + rx_buffer.append((char *)ptr, len); + return 0; + } + + int send_data(void *ptr, size_t len) { + if (!tx_buffer_it.get_remaining()) + return 0; // nothing left to send + + int l = MIN(tx_buffer_it.get_remaining(), len); + memcpy(ptr, tx_buffer_it.get_current_ptr().c_str(), l); + try { + tx_buffer_it.advance(l); + } catch (buffer::end_of_buffer &e) { + assert(0); + } + + return l; + } + + int make_kms_encrypt_request(string &root_acount, RGWKmsData* kmsdata); + int make_kms_decrypt_request(string &root_acount, RGWKmsData* kmsdata);; + +}; + + class dss_endpoint { public: static string endpoint; dss_endpoint() { } ~dss_endpoint() { } }; +class RGWHandler_ObjStore_Obj_S3Website; + #endif diff --git a/src/rgw/rgw_rest_s3website.h b/src/rgw/rgw_rest_s3website.h new file mode 100644 index 0000000000000..f8ead0ba7c61b --- /dev/null +++ b/src/rgw/rgw_rest_s3website.h @@ -0,0 +1,97 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Robin H. Johnson + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#ifndef CEPH_RGW_REST_S3WEBSITE_H +#define CEPH_RGW_REST_S3WEBSITE_H + +#include "rgw_rest_s3.h" + +class RGWHandler_ObjStore_S3Website : public RGWHandler_ObjStore_S3 { +protected: + int retarget(RGWOp *op, RGWOp **new_op); + // TODO: this should be virtual I think, and ensure that it's always + // overridden, but that conflates that op_get/op_head are defined in this + // class and call this; and don't need to be overridden later. + virtual RGWOp *get_obj_op(bool get_data) { return NULL; } + RGWOp *op_get(); + RGWOp *op_head(); + // Only allowed to use GET+HEAD + RGWOp *op_put() { return NULL; } + RGWOp *op_delete() { return NULL; } + RGWOp *op_post() { return NULL; } + RGWOp *op_copy() { return NULL; } + RGWOp *op_options() { return NULL; } + + int serve_errordoc(int http_ret, const string &errordoc_key); +public: + RGWHandler_ObjStore_S3Website() : RGWHandler_ObjStore_S3() {} + virtual ~RGWHandler_ObjStore_S3Website() {} + virtual int error_handler(int err_no, string *error_content); +}; + +class RGWHandler_ObjStore_Service_S3Website : public RGWHandler_ObjStore_S3Website { +protected: + virtual RGWOp *get_obj_op(bool get_data); +public: + RGWHandler_ObjStore_Service_S3Website() {} + virtual ~RGWHandler_ObjStore_Service_S3Website() {} +}; + +class RGWHandler_ObjStore_Obj_S3Website : public RGWHandler_ObjStore_S3Website { +protected: + virtual RGWOp *get_obj_op(bool get_data); +public: + RGWHandler_ObjStore_Obj_S3Website() {} + virtual ~RGWHandler_ObjStore_Obj_S3Website() {} +}; + +/* The cross-inheritance from Obj to Bucket is deliberate! + * S3Websites do NOT support any bucket operations + */ +class RGWHandler_ObjStore_Bucket_S3Website : public RGWHandler_ObjStore_S3Website { +protected: + RGWOp *get_obj_op(bool get_data); +public: + RGWHandler_ObjStore_Bucket_S3Website() {} + virtual ~RGWHandler_ObjStore_Bucket_S3Website() {} +}; + +// TODO: do we actually need this? +class RGWGetObj_ObjStore_S3Website : public RGWGetObj_ObjStore_S3 +{ + friend class RGWHandler_REST_S3Website; +private: + bool is_errordoc_request; +public: + RGWGetObj_ObjStore_S3Website() : is_errordoc_request(false) {} + RGWGetObj_ObjStore_S3Website(bool is_errordoc_request) : is_errordoc_request(false) { this->is_errordoc_request = is_errordoc_request; } + ~RGWGetObj_ObjStore_S3Website() {} + int send_response_data_error(); + int send_response_data(bufferlist& bl, off_t ofs, off_t len); + // We override RGWGetObj_ObjStore::get_params here, to allow ignoring all + // conditional params for error pages. + int get_params() { + if (is_errordoc_request) { + range_str = NULL; + if_mod = NULL; + if_unmod = NULL; + if_match = NULL; + if_nomatch = NULL; + return 0; + } else { + return RGWGetObj_ObjStore_S3::get_params(); + } + } +}; + +#endif diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index bcbd45002c896..d9e9190ca5308 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -399,10 +399,11 @@ int RGWCreateBucket_ObjStore_SWIFT::get_params() void RGWCreateBucket_ObjStore_SWIFT::send_response() { - if (!ret) - ret = STATUS_CREATED; - else if (ret == -ERR_BUCKET_EXISTS) + if (exist_ret == -ERR_BUCKET_EXISTS) { ret = STATUS_ACCEPTED; + } else if (!ret) { + ret = STATUS_CREATED; + } set_req_state_err(s, ret); dump_errno(s); /* Propose ending HTTP header with 0 Content-Length header. */ @@ -692,6 +693,19 @@ void RGWCopyObj_ObjStore_SWIFT::send_response() } } +int RGWGetObj_ObjStore_SWIFT::get_params() +{ + const string& mm = s->info.args.get("multipart-manifest"); + + return RGWGetObj_ObjStore::get_params(); +} + +int RGWGetObj_ObjStore_SWIFT::send_response_data_error() +{ + bufferlist bl; + return send_response_data(bl, 0, 0); +} + int RGWGetObj_ObjStore_SWIFT::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { string content_type; diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h index 19028ab337b43..ebd0d33d01a5c 100644 --- a/src/rgw/rgw_rest_swift.h +++ b/src/rgw/rgw_rest_swift.h @@ -13,6 +13,8 @@ class RGWGetObj_ObjStore_SWIFT : public RGWGetObj_ObjStore { RGWGetObj_ObjStore_SWIFT() {} ~RGWGetObj_ObjStore_SWIFT() {} + int get_params(); + int send_response_data_error(); int send_response_data(bufferlist& bl, off_t ofs, off_t len); }; diff --git a/src/rgw/rgw_website.cc b/src/rgw/rgw_website.cc new file mode 100644 index 0000000000000..a69ffe16d2981 --- /dev/null +++ b/src/rgw/rgw_website.cc @@ -0,0 +1,119 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Yehuda Sadeh + * Copyright (C) 2015 Robin H. Johnson + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#include "common/debug.h" +#include "common/ceph_json.h" + +#include "acconfig.h" + +#include +#include +#include +#include "include/types.h" +#include "rgw_website.h" + +using namespace std; + + +bool RGWBWRoutingRuleCondition::check_key_condition(const string& key) { + return (key.size() >= key_prefix_equals.size() && + key.compare(0, key_prefix_equals.size(), key_prefix_equals) == 0); +} + + +void RGWBWRoutingRule::apply_rule(const string& default_protocol, const string& default_hostname, + const string& key, string *new_url, int *redirect_code) +{ + RGWRedirectInfo& redirect = redirect_info.redirect; + + string protocol = (!redirect.protocol.empty() ? redirect.protocol : default_protocol); + string hostname = (!redirect.hostname.empty() ? redirect.hostname : default_hostname); + + *new_url = protocol + "://" + hostname + "/"; + + if (!redirect_info.replace_key_prefix_with.empty()) { + *new_url += redirect_info.replace_key_prefix_with; + *new_url += key.substr(condition.key_prefix_equals.size()); + } else if (!redirect_info.replace_key_with.empty()) { + *new_url += redirect_info.replace_key_with; + } else { + *new_url += key; + } + + if(redirect.http_redirect_code > 0) + *redirect_code = redirect.http_redirect_code; +} + +bool RGWBWRoutingRules::check_key_and_error_code_condition(const string &key, int error_code, RGWBWRoutingRule **rule) +{ + for (list::iterator iter = rules.begin(); iter != rules.end(); ++iter) { + if (iter->check_key_condition(key) && iter->check_error_code_condition(error_code)) { + *rule = &(*iter); + return true; + } + } + return false; +} + +bool RGWBWRoutingRules::check_key_condition(const string& key, RGWBWRoutingRule **rule) +{ + for (list::iterator iter = rules.begin(); iter != rules.end(); ++iter) { + if (iter->check_key_condition(key)) { + *rule = &(*iter); + return true; + } + } + return false; +} + +bool RGWBWRoutingRules::check_error_code_condition(const int http_error_code, RGWBWRoutingRule **rule) +{ + for (list::iterator iter = rules.begin(); iter != rules.end(); ++iter) { + if (iter->check_error_code_condition(http_error_code)) { + *rule = &(*iter); + return true; + } + } + return false; +} + +bool RGWBucketWebsiteConf::should_redirect(const string& key, const int http_error_code, RGWBWRoutingRule *redirect) +{ + RGWBWRoutingRule *rule; + if(!redirect_all.hostname.empty()) { + RGWBWRoutingRule redirect_all_rule; + redirect_all_rule.redirect_info.redirect = redirect_all; + redirect_all.http_redirect_code = 301; + *redirect = redirect_all_rule; + return true; + } else if (!routing_rules.check_key_and_error_code_condition(key, http_error_code, &rule)) { + return false; + } + + *redirect = *rule; + + return true; +} + +void RGWBucketWebsiteConf::get_effective_key(const string& key, string *effective_key) +{ + + if (key.empty()) { + *effective_key = index_doc_suffix; + } else if (key[key.size() - 1] == '/') { + *effective_key = key + index_doc_suffix; + } else { + *effective_key = key; + } +} diff --git a/src/rgw/rgw_website.h b/src/rgw/rgw_website.h new file mode 100644 index 0000000000000..6c1a92bb47603 --- /dev/null +++ b/src/rgw/rgw_website.h @@ -0,0 +1,200 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Yehuda Sadeh + * Copyright (C) 2015 Robin H. Johnson + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#ifndef RGW_WEBSITE_H +#define RGW_WEBSITE_H + +#include "rgw_xml.h" + +struct RGWRedirectInfo +{ + string protocol; + string hostname; + uint16_t http_redirect_code; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(protocol, bl); + ::encode(hostname, bl); + ::encode(http_redirect_code, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(protocol, bl); + ::decode(hostname, bl); + ::decode(http_redirect_code, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); +}; +WRITE_CLASS_ENCODER(RGWRedirectInfo) + + +struct RGWBWRedirectInfo +{ + RGWRedirectInfo redirect; + string replace_key_prefix_with; + string replace_key_with; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(redirect, bl); + ::encode(replace_key_prefix_with, bl); + ::encode(replace_key_with, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(redirect, bl); + ::decode(replace_key_prefix_with, bl); + ::decode(replace_key_with, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + void dump_xml(Formatter *f) const; + void decode_json(JSONObj *obj); + void decode_xml(XMLObj *obj); +}; +WRITE_CLASS_ENCODER(RGWBWRedirectInfo) + +struct RGWBWRoutingRuleCondition +{ + string key_prefix_equals; + uint16_t http_error_code_returned_equals; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(key_prefix_equals, bl); + ::encode(http_error_code_returned_equals, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(key_prefix_equals, bl); + ::decode(http_error_code_returned_equals, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + void dump_xml(Formatter *f) const; + void decode_json(JSONObj *obj); + void decode_xml(XMLObj *obj); + + bool check_key_condition(const string& key); + bool check_error_code_condition(const int error_code) { + return (uint16_t)error_code == http_error_code_returned_equals; + } +}; +WRITE_CLASS_ENCODER(RGWBWRoutingRuleCondition) + +struct RGWBWRoutingRule +{ + RGWBWRoutingRuleCondition condition; + RGWBWRedirectInfo redirect_info; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(condition, bl); + ::encode(redirect_info, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(condition, bl); + ::decode(redirect_info, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + void dump_xml(Formatter *f) const; + void decode_json(JSONObj *obj); + void decode_xml(XMLObj *obj); + + bool check_key_condition(const string& key) { + return condition.check_key_condition(key); + } + bool check_error_code_condition(int error_code) { + return condition.check_error_code_condition(error_code); + } + + void apply_rule(const string& default_protocol, const string& default_hostname, const string& key, string *redirect, int *redirect_code); +}; +WRITE_CLASS_ENCODER(RGWBWRoutingRule) + +struct RGWBWRoutingRules +{ + list rules; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(rules, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(rules, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + void dump_xml(Formatter *f) const; + void decode_json(JSONObj *obj); + + bool check_key_condition(const string& key, RGWBWRoutingRule **rule); + bool check_error_code_condition(int error_code, RGWBWRoutingRule **rule); + bool check_key_and_error_code_condition(const string& key, const int error_code, RGWBWRoutingRule **rule); +}; +WRITE_CLASS_ENCODER(RGWBWRoutingRules) + +struct RGWBucketWebsiteConf +{ + RGWRedirectInfo redirect_all; + string index_doc_suffix; + string error_doc; + RGWBWRoutingRules routing_rules; + + RGWBucketWebsiteConf() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(index_doc_suffix, bl); + ::encode(error_doc, bl); + ::encode(routing_rules, bl); + ::encode(redirect_all, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(index_doc_suffix, bl); + ::decode(error_doc, bl); + ::decode(routing_rules, bl); + ::decode(redirect_all, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); + void decode_xml(XMLObj *obj); + void dump_xml(Formatter *f) const; + + bool should_redirect(const string& key, const int http_error_code, RGWBWRoutingRule *redirect); + void get_effective_key(const string& key, string *effective_key); +}; +WRITE_CLASS_ENCODER(RGWBucketWebsiteConf) + +#endif diff --git a/src/rgw/rgw_xml.cc b/src/rgw/rgw_xml.cc index 3e38f8070107f..eda0887528234 100644 --- a/src/rgw/rgw_xml.cc +++ b/src/rgw/rgw_xml.cc @@ -148,17 +148,21 @@ RGWXMLParser:: XML_ParserFree(p); free(buf); - vector::iterator iter; - for (iter = objs.begin(); iter != objs.end(); ++iter) { + list::iterator iter; + for (iter = allocated_objs.begin(); iter != allocated_objs.end(); ++iter) { XMLObj *obj = *iter; delete obj; } } + bool RGWXMLParser::xml_start(const char *el, const char **attr) { XMLObj * obj = alloc_obj(el); if (!obj) { - obj = new XMLObj(); + unallocated_objs.push_back(XMLObj()); + obj = &unallocated_objs.back(); + } else { + allocated_objs.push_back(obj); } if (!obj->xml_start(cur_obj, el, attr)) return false; @@ -238,3 +242,254 @@ bool RGWXMLParser::parse(const char *_buf, int len, int done) return success; } + +void decode_xml_obj(unsigned long& val, XMLObj *obj) +{ + string& s = obj->get_data(); + const char *start = s.c_str(); + char *p; + + errno = 0; + val = strtoul(start, &p, 10); + + /* Check for various possible errors */ + + if ((errno == ERANGE && val == ULONG_MAX) || + (errno != 0 && val == 0)) { + throw RGWXMLDecoder::err("failed to number"); + } + + if (p == start) { + throw RGWXMLDecoder::err("failed to parse number"); + } + + while (*p != '\0') { + if (!isspace(*p)) { + throw RGWXMLDecoder::err("failed to parse number"); + } + p++; + } +} + + +void decode_xml_obj(long& val, XMLObj *obj) +{ + string s = obj->get_data(); + const char *start = s.c_str(); + char *p; + + errno = 0; + val = strtol(start, &p, 10); + + /* Check for various possible errors */ + + if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || + (errno != 0 && val == 0)) { + throw RGWXMLDecoder::err("failed to parse number"); + } + + if (p == start) { + throw RGWXMLDecoder::err("failed to parse number"); + } + + while (*p != '\0') { + if (!isspace(*p)) { + throw RGWXMLDecoder::err("failed to parse number"); + } + p++; + } +} + +void decode_xml_obj(long long& val, XMLObj *obj) +{ + string s = obj->get_data(); + const char *start = s.c_str(); + char *p; + + errno = 0; + val = strtoll(start, &p, 10); + + /* Check for various possible errors */ + + if ((errno == ERANGE && (val == LLONG_MAX || val == LLONG_MIN)) || + (errno != 0 && val == 0)) { + throw RGWXMLDecoder::err("failed to parse number"); + } + + if (p == start) { + throw RGWXMLDecoder::err("failed to parse number"); + } + + while (*p != '\0') { + if (!isspace(*p)) { + throw RGWXMLDecoder::err("failed to parse number"); + } + p++; + } +} + +void decode_xml_obj(unsigned long long& val, XMLObj *obj) +{ + string s = obj->get_data(); + const char *start = s.c_str(); + char *p; + + errno = 0; + val = strtoull(start, &p, 10); + + /* Check for various possible errors */ + + if ((errno == ERANGE && val == ULLONG_MAX) || + (errno != 0 && val == 0)) { + throw RGWXMLDecoder::err("failed to number"); + } + + if (p == start) { + throw RGWXMLDecoder::err("failed to parse number"); + } + + while (*p != '\0') { + if (!isspace(*p)) { + throw RGWXMLDecoder::err("failed to parse number"); + } + p++; + } +} + +void decode_xml_obj(int& val, XMLObj *obj) +{ + long l; + decode_xml_obj(l, obj); +#if LONG_MAX > INT_MAX + if (l > INT_MAX || l < INT_MIN) { + throw RGWXMLDecoder::err("integer out of range"); + } +#endif + + val = (int)l; +} + +void decode_xml_obj(unsigned& val, XMLObj *obj) +{ + unsigned long l; + decode_xml_obj(l, obj); +#if ULONG_MAX > UINT_MAX + if (l > UINT_MAX) { + throw RGWXMLDecoder::err("unsigned integer out of range"); + } +#endif + + val = (unsigned)l; +} + +void decode_xml_obj(bool& val, XMLObj *obj) +{ + string s = obj->get_data(); + if (strcasecmp(s.c_str(), "true") == 0) { + val = true; + return; + } + if (strcasecmp(s.c_str(), "false") == 0) { + val = false; + return; + } + int i; + decode_xml_obj(i, obj); + val = (bool)i; +} + +void decode_xml_obj(bufferlist& val, XMLObj *obj) +{ + string s = obj->get_data(); + + bufferlist bl; + bl.append(s.c_str(), s.size()); + try { + val.decode_base64(bl); + } catch (buffer::error& err) { + throw RGWXMLDecoder::err("failed to decode base64"); + } +} + +void decode_xml_obj(utime_t& val, XMLObj *obj) +{ + string s = obj->get_data(); + uint64_t epoch; + uint64_t nsec; + int r = utime_t::parse_date(s, &epoch, &nsec); + if (r == 0) { + val = utime_t(epoch, nsec); + } else { + throw RGWXMLDecoder::err("failed to decode utime_t"); + } +} + +void encode_xml(const char *name, const string& val, Formatter *f) +{ + f->dump_string(name, val); +} + +void encode_xml(const char *name, const char *val, Formatter *f) +{ + f->dump_string(name, val); +} + +void encode_xml(const char *name, bool val, Formatter *f) +{ + string s; + if (val) + s = "True"; + else + s = "False"; + + f->dump_string(name, s); +} + +void encode_xml(const char *name, int val, Formatter *f) +{ + f->dump_int(name, val); +} + +void encode_xml(const char *name, long val, Formatter *f) +{ + f->dump_int(name, val); +} + +void encode_xml(const char *name, unsigned val, Formatter *f) +{ + f->dump_unsigned(name, val); +} + +void encode_xml(const char *name, unsigned long val, Formatter *f) +{ + f->dump_unsigned(name, val); +} + +void encode_xml(const char *name, unsigned long long val, Formatter *f) +{ + f->dump_unsigned(name, val); +} + +void encode_xml(const char *name, long long val, Formatter *f) +{ + f->dump_int(name, val); +} + +void encode_xml(const char *name, const utime_t& val, Formatter *f) +{ + val.gmtime(f->dump_stream(name)); +} + +void encode_xml(const char *name, const bufferlist& bl, Formatter *f) +{ + /* need to copy data from bl, as it is const bufferlist */ + bufferlist src = bl; + + bufferlist b64; + src.encode_base64(b64); + + string s(b64.c_str(), b64.length()); + + encode_xml(name, s, f); +} + diff --git a/src/rgw/rgw_xml.h b/src/rgw/rgw_xml.h index 164e97a70dc52..35f257aa97a35 100644 --- a/src/rgw/rgw_xml.h +++ b/src/rgw/rgw_xml.h @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -67,8 +68,12 @@ class RGWXMLParser : public XMLObj int buf_len; XMLObj *cur_obj; vector objs; + list allocated_objs; + list unallocated_objs; protected: - virtual XMLObj *alloc_obj(const char *el) = 0; + virtual XMLObj *alloc_obj(const char *el) { + return NULL; + } public: RGWXMLParser(); virtual ~RGWXMLParser(); @@ -85,4 +90,191 @@ class RGWXMLParser : public XMLObj bool success; }; +class RGWXMLDecoder { +public: + struct err { + string message; + + err(const string& m) : message(m) {} + }; + + class XMLParser : public RGWXMLParser { + public: + XMLParser() {} + virtual ~XMLParser() {} + } parser; + + RGWXMLDecoder(bufferlist& bl) { + if (!parser.parse(bl.c_str(), bl.length(), 1)) { + cout << "RGWXMLDecoder::err()" << std::endl; + throw RGWXMLDecoder::err("failed to parse XML input"); + } + } + + template + static bool decode_xml(const char *name, T& val, XMLObj *obj, bool mandatory = false); + + template + static bool decode_xml(const char *name, C& container, void (*cb)(C&, XMLObj *obj), XMLObj *obj, bool mandatory = false); + + template + static void decode_xml(const char *name, T& val, T& default_val, XMLObj *obj); +}; + +template +void decode_xml_obj(T& val, XMLObj *obj) +{ + val.decode_xml(obj); +} + +static inline void decode_xml_obj(string& val, XMLObj *obj) +{ + val = obj->get_data(); +} + +void decode_xml_obj(unsigned long long& val, XMLObj *obj); +void decode_xml_obj(long long& val, XMLObj *obj); +void decode_xml_obj(unsigned long& val, XMLObj *obj); +void decode_xml_obj(long& val, XMLObj *obj); +void decode_xml_obj(unsigned& val, XMLObj *obj); +void decode_xml_obj(int& val, XMLObj *obj); +void decode_xml_obj(bool& val, XMLObj *obj); +void decode_xml_obj(bufferlist& val, XMLObj *obj); +class utime_t; +void decode_xml_obj(utime_t& val, XMLObj *obj); + +template +void do_decode_xml_obj(list& l, const string& name, XMLObj *obj) +{ + l.clear(); + + XMLObjIter iter = obj->find(name); + XMLObj *o; + + while ((o = iter.get_next())) { + T val; + decode_xml_obj(val, o); + l.push_back(val); + } +} + +template +void do_decode_xml_obj(vector& l, const string& name, XMLObj *obj) +{ + l.clear(); + + XMLObjIter iter = obj->find(name); + XMLObj *o; + + while (o = iter.get_next()) { + T val; + decode_xml_obj(val, o); + l.push_back(val); + } +} + +template +bool RGWXMLDecoder::decode_xml(const char *name, T& val, XMLObj *obj, bool mandatory) +{ + XMLObjIter iter = obj->find(name); + XMLObj *o = iter.get_next(); + if (!o) { + if (mandatory) { + string s = "missing mandatory field " + string(name); + throw err(s); + } + val = T(); + return false; + } + + try { + decode_xml_obj(val, o); + } catch (err& e) { + string s = string(name) + ": "; + s.append(e.message); + throw err(s); + } + + return true; +} + +template +bool RGWXMLDecoder::decode_xml(const char *name, C& container, void (*cb)(C&, XMLObj *), XMLObj *obj, bool mandatory) +{ + container.clear(); + + XMLObjIter iter = obj->find(name); + XMLObj *o = iter.get_next(); + if (!o) { + if (mandatory) { + string s = "missing mandatory field " + string(name); + throw err(s); + } + return false; + } + + try { + decode_xml_obj(container, cb, o); + } catch (err& e) { + string s = string(name) + ": "; + s.append(e.message); + throw err(s); + } + + return true; +} + +template +void RGWXMLDecoder::decode_xml(const char *name, T& val, T& default_val, XMLObj *obj) +{ + XMLObjIter iter = obj->find(name); + XMLObj *o = iter.get_next(); + if (!o) { + val = default_val; + return; + } + + try { + decode_xml_obj(val, o); + } catch (err& e) { + val = default_val; + string s = string(name) + ": "; + s.append(e.message); + throw err(s); + } +} + +template +static void encode_xml(const char *name, const T& val, ceph::Formatter *f) +{ + f->open_object_section(name); + val.dump_xml(f); + f->close_section(); +} + +void encode_xml(const char *name, const string& val, ceph::Formatter *f); +void encode_xml(const char *name, const char *val, ceph::Formatter *f); +void encode_xml(const char *name, bool val, ceph::Formatter *f); +void encode_xml(const char *name, int val, ceph::Formatter *f); +void encode_xml(const char *name, unsigned val, ceph::Formatter *f); +void encode_xml(const char *name, long val, ceph::Formatter *f); +void encode_xml(const char *name, unsigned long val, ceph::Formatter *f); +void encode_xml(const char *name, long long val, ceph::Formatter *f); +void encode_xml(const char *name, const utime_t& val, ceph::Formatter *f); +void encode_xml(const char *name, const bufferlist& bl, ceph::Formatter *f); +void encode_xml(const char *name, long long val, ceph::Formatter *f); +void encode_xml(const char *name, long long unsigned val, ceph::Formatter *f); + +template +static void do_encode_xml(const char *name, const std::list& l, const char *entry_name, ceph::Formatter *f) +{ + f->open_array_section(name); + for (typename std::list::const_iterator iter = l.begin(); iter != l.end(); ++iter) { + encode_xml(entry_name, *iter, f); + } + f->close_section(); +} + + + #endif diff --git a/src/rgw/rgw_xml_enc.cc b/src/rgw/rgw_xml_enc.cc new file mode 100644 index 0000000000000..ff64efca7223b --- /dev/null +++ b/src/rgw/rgw_xml_enc.cc @@ -0,0 +1,131 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Yehuda Sadeh + * Copyright (C) 2015 Robin H. Johnson + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#include "rgw_common.h" +#include "rgw_xml.h" + +#include "common/Formatter.h" + +#define dout_subsys ceph_subsys_rgw + +void RGWBWRedirectInfo::dump_xml(Formatter *f) const +{ + if (!redirect.protocol.empty()) { + encode_xml("Protocol", redirect.protocol, f); + } + if (!redirect.hostname.empty()) { + encode_xml("HostName", redirect.hostname, f); + } + if (redirect.http_redirect_code > 0) { + encode_xml("HttpRedirectCode", (int)redirect.http_redirect_code, f); + } + if (!replace_key_prefix_with.empty()) { + encode_xml("ReplaceKeyPrefixWith", replace_key_prefix_with, f); + } + if (!replace_key_with.empty()) { + encode_xml("ReplaceKeyWith", replace_key_with, f); + } +} + +void RGWBWRedirectInfo::decode_xml(XMLObj *obj) { + RGWXMLDecoder::decode_xml("Protocol", redirect.protocol, obj); + RGWXMLDecoder::decode_xml("HostName", redirect.hostname, obj); + int code = 0; + RGWXMLDecoder::decode_xml("HttpRedirectCode", code, obj); + redirect.http_redirect_code = code; + RGWXMLDecoder::decode_xml("ReplaceKeyPrefixWith", replace_key_prefix_with, obj); + RGWXMLDecoder::decode_xml("ReplaceKeyWith", replace_key_with, obj); +} + +void RGWBWRoutingRuleCondition::dump_xml(Formatter *f) const +{ + if (!key_prefix_equals.empty()) { + encode_xml("KeyPrefixEquals", key_prefix_equals, f); + } + if (http_error_code_returned_equals > 0) { + encode_xml("HttpErrorCodeReturnedEquals", (int)http_error_code_returned_equals, f); + } +} + +void RGWBWRoutingRuleCondition::decode_xml(XMLObj *obj) { + RGWXMLDecoder::decode_xml("KeyPrefixEquals", key_prefix_equals, obj); + int code = 0; + RGWXMLDecoder::decode_xml("HttpErrorCodeReturnedEquals", code, obj); + http_error_code_returned_equals = code; +} + +void RGWBWRoutingRule::dump_xml(Formatter *f) const +{ + encode_xml("Condition", condition, f); + encode_xml("Redirect", redirect_info, f); +} + +void RGWBWRoutingRule::decode_xml(XMLObj *obj) { + RGWXMLDecoder::decode_xml("Condition", condition, obj); + RGWXMLDecoder::decode_xml("Redirect", redirect_info, obj); +} + +static void encode_xml(const char *name, const std::list& l, ceph::Formatter *f) +{ + do_encode_xml("RoutingRules", l, "RoutingRule", f); +} + +void RGWBucketWebsiteConf::dump_xml(Formatter *f) const +{ + if (!redirect_all.hostname.empty()) { + f->open_object_section("RedirectAllRequestsTo"); + encode_xml("HostName", redirect_all.hostname, f); + if (!redirect_all.protocol.empty()) { + encode_xml("Protocol", redirect_all.protocol, f); + } + f->close_section(); + } + if (!index_doc_suffix.empty()) { + f->open_object_section("IndexDocument"); + encode_xml("Suffix", index_doc_suffix, f); + f->close_section(); + } + if (!error_doc.empty()) { + f->open_object_section("ErrorDocument"); + encode_xml("Key", error_doc, f); + f->close_section(); + } + if (!routing_rules.rules.empty()) { + encode_xml("RoutingRules", routing_rules.rules, f); + } +} + +void decode_xml_obj(list& l, XMLObj *obj) +{ + do_decode_xml_obj(l, "RoutingRule", obj); +} + +void RGWBucketWebsiteConf::decode_xml(XMLObj *obj) { + XMLObj *o = obj->find_first("RedirectAllRequestsTo"); + if (o) { + RGWXMLDecoder::decode_xml("HostName", redirect_all.hostname, o, true); + RGWXMLDecoder::decode_xml("Protocol", redirect_all.protocol, o); + } else { + o = obj->find_first("IndexDocument"); + if (o) { + RGWXMLDecoder::decode_xml("Suffix", index_doc_suffix, o); + } + o = obj->find_first("ErrorDocument"); + if (o) { + RGWXMLDecoder::decode_xml("Key", error_doc, o); + } + RGWXMLDecoder::decode_xml("RoutingRules", routing_rules.rules, obj); + } +} + diff --git a/src/test/formatter.cc b/src/test/formatter.cc index aab7e59259504..3913cc6f15420 100644 --- a/src/test/formatter.cc +++ b/src/test/formatter.cc @@ -14,6 +14,7 @@ #include "test/unit.h" #include "common/Formatter.h" +#include "common/HTMLFormatter.h" #include #include @@ -130,7 +131,7 @@ TEST(XmlFormatter, DTD) { ostringstream oss; XMLFormatter fmt(false); - fmt.write_raw_data(XMLFormatter::XML_1_DTD); + fmt.output_header(); fmt.open_array_section("foo"); fmt.dump_stream("blah") << "hithere"; fmt.dump_float("pi", 3.14); @@ -144,7 +145,7 @@ TEST(XmlFormatter, Clear) { ostringstream oss; XMLFormatter fmt(false); - fmt.write_raw_data(XMLFormatter::XML_1_DTD); + fmt.output_header(); fmt.open_array_section("foo"); fmt.dump_stream("blah") << "hithere"; fmt.dump_float("pi", 3.14); @@ -167,7 +168,7 @@ TEST(XmlFormatter, NamespaceTest) { ostringstream oss; XMLFormatter fmt(false); - fmt.write_raw_data(XMLFormatter::XML_1_DTD); + fmt.output_header(); fmt.open_array_section_in_ns("foo", "http://s3.amazonaws.com/doc/2006-03-01/"); fmt.dump_stream("blah") << "hithere"; @@ -197,3 +198,145 @@ TEST(XmlFormatter, DumpFormatNameSpaceTest) { fmt.flush(oss2); ASSERT_EQ(oss2.str(),"bar"); } + +TEST(HtmlFormatter, Simple1) { + ostringstream oss; + HTMLFormatter fmt(false); + fmt.open_object_section("foo"); + fmt.dump_int("a", 1); + fmt.dump_int("b", 2); + fmt.dump_int("c", 3); + fmt.close_section(); + fmt.flush(oss); + ASSERT_EQ(oss.str(), "
  • a: 1
  • b: 2
  • c: 3
  • "); +} + +TEST(HtmlFormatter, Simple2) { + ostringstream oss; + HTMLFormatter fmt(false); + fmt.open_object_section("foo"); + fmt.open_object_section("bar"); + fmt.dump_int("int", 0xf00000000000ll); + fmt.dump_unsigned("unsigned", 0x8000000000000001llu); + fmt.dump_float("float", 1.234); + fmt.close_section(); + fmt.dump_string("string", "str"); + fmt.close_section(); + fmt.flush(oss); + ASSERT_EQ(oss.str(), "\ +
  • int: 263882790666240
  • \ +
  • unsigned: 9223372036854775809
  • \ +
  • float: 1.234
  • \ +
  • string: str
  • \ +
    "); +} + +TEST(HtmlFormatter, Empty) { + ostringstream oss; + HTMLFormatter fmt(false); + fmt.flush(oss); + ASSERT_EQ(oss.str(), ""); +} + +TEST(HtmlFormatter, DumpStream1) { + ostringstream oss; + HTMLFormatter fmt(false); + fmt.dump_stream("blah") << "hithere"; + fmt.flush(oss); + ASSERT_EQ(oss.str(), "
  • blah: hithere
  • "); +} + +TEST(HtmlFormatter, DumpStream2) { + ostringstream oss; + HTMLFormatter fmt(false); + + fmt.open_array_section("foo"); + fmt.dump_stream("blah") << "hithere"; + fmt.close_section(); + fmt.flush(oss); + ASSERT_EQ(oss.str(), "
  • blah: hithere
  • "); +} + +TEST(HtmlFormatter, DumpStream3) { + ostringstream oss; + HTMLFormatter fmt(false); + + fmt.open_array_section("foo"); + fmt.dump_stream("blah") << "hithere"; + fmt.dump_float("pi", 3.14); + fmt.close_section(); + fmt.flush(oss); + ASSERT_EQ(oss.str(), "
  • blah: hithere
  • pi: 3.14
  • "); +} + +TEST(HtmlFormatter, DTD) { + ostringstream oss; + HTMLFormatter fmt(false); + + fmt.write_raw_data(HTMLFormatter::XML_1_DTD); + fmt.open_array_section("foo"); + fmt.dump_stream("blah") << "hithere"; + fmt.dump_float("pi", 3.14); + fmt.close_section(); + fmt.flush(oss); + ASSERT_EQ(oss.str(), "" + "
  • blah: hithere
  • pi: 3.14
  • "); +} + +TEST(HtmlFormatter, Clear) { + ostringstream oss; + HTMLFormatter fmt(false); + + fmt.write_raw_data(HTMLFormatter::XML_1_DTD); + fmt.open_array_section("foo"); + fmt.dump_stream("blah") << "hithere"; + fmt.dump_float("pi", 3.14); + fmt.close_section(); + fmt.flush(oss); + ASSERT_EQ(oss.str(), "" + "
  • blah: hithere
  • pi: 3.14
  • "); + + ostringstream oss2; + fmt.flush(oss2); + ASSERT_EQ(oss2.str(), ""); + + ostringstream oss3; + fmt.reset(); + fmt.flush(oss3); + ASSERT_EQ(oss3.str(), ""); +} + +TEST(HtmlFormatter, NamespaceTest) { + ostringstream oss; + HTMLFormatter fmt(false); + + fmt.write_raw_data(HTMLFormatter::XML_1_DTD); + fmt.open_array_section_in_ns("foo", + "http://s3.amazonaws.com/doc/2006-03-01/"); + fmt.dump_stream("blah") << "hithere"; + fmt.dump_float("pi", 3.14); + fmt.close_section(); + fmt.flush(oss); + ASSERT_EQ(oss.str(), "" + "" + "
  • blah: hithere
  • pi: 3.14
  • "); +} + +TEST(HtmlFormatter, DumpFormatNameSpaceTest) { + ostringstream oss1; + HTMLFormatter fmt(false); + + fmt.dump_format_ns("foo", + "http://s3.amazonaws.com/doc/2006-03-01/", + "%s","bar"); + fmt.flush(oss1); + ASSERT_EQ(oss1.str(), + "
  • foo: bar
  • "); + + // Testing with a null ns..should be same as dump format + ostringstream oss2; + fmt.reset(); + fmt.dump_format_ns("foo",NULL,"%s","bar"); + fmt.flush(oss2); + ASSERT_EQ(oss2.str(),"
  • foo: bar
  • "); +} diff --git a/src/tracing/Makefile.am b/src/tracing/Makefile.am index 16d300ec1dbbb..510c0bc75732e 100644 --- a/src/tracing/Makefile.am +++ b/src/tracing/Makefile.am @@ -22,7 +22,7 @@ nodist_libosd_tp_la_SOURCES = \ pg.h \ pg.c endif -libosd_tp_la_LIBADD = -llttng-ust -ldl +libosd_tp_la_LIBADD = -ldl -llttng-ust libosd_tp_la_CPPFLAGS = -DTRACEPOINT_PROBE_DYNAMIC_LINKAGE libosd_tp_la_LDFLAGS = @@ -31,7 +31,7 @@ nodist_librados_tp_la_SOURCES = \ librados.c \ librados.h endif -librados_tp_la_LIBADD = -llttng-ust -ldl +librados_tp_la_LIBADD = -ldl -llttng-ust librados_tp_la_CPPFLAGS = -DTRACEPOINT_PROBE_DYNAMIC_LINKAGE librados_tp_la_CFLAGS = -I$(top_srcdir)/src $(AM_CFLAGS) librados_tp_la_LDFLAGS = @@ -41,7 +41,7 @@ nodist_librbd_tp_la_SOURCES = \ librbd.c \ librbd.h endif -librbd_tp_la_LIBADD = -llttng-ust -ldl +librbd_tp_la_LIBADD = -ldl -llttng-ust librbd_tp_la_CPPFLAGS = -DTRACEPOINT_PROBE_DYNAMIC_LINKAGE librbd_tp_la_CFLAGS = -I$(top_srcdir)/src $(AM_CFLAGS) librbd_tp_la_LDFLAGS = @@ -51,7 +51,7 @@ nodist_libos_tp_la_SOURCES = \ objectstore.c \ objectstore.h endif -libos_tp_la_LIBADD = -llttng-ust -ldl +libos_tp_la_LIBADD = -ldl -llttng-ust libos_tp_la_CPPFLAGS = -DTRACEPOINT_PROBE_DYNAMIC_LINKAGE libos_tp_la_CFLAGS = -I$(top_srcdir)/src $(AM_CFLAGS) libos_tp_la_LDFLAGS =