diff --git a/mooncake-integration/store/store_py.cpp b/mooncake-integration/store/store_py.cpp index 0d23317554..e942c8e67e 100644 --- a/mooncake-integration/store/store_py.cpp +++ b/mooncake-integration/store/store_py.cpp @@ -1078,25 +1078,36 @@ PYBIND11_MODULE(store, m) { return self.store_->batch_get_buffer(keys); }, py::return_value_policy::take_ownership) - .def("remove", - [](MooncakeStorePyWrapper &self, const std::string &key) { - py::gil_scoped_release release; - return self.store_->remove(key); - }) + .def( + "remove", + [](MooncakeStorePyWrapper &self, const std::string &key, + bool force) { + py::gil_scoped_release release; + return self.store_->remove(key, force); + }, + py::arg("key"), py::arg("force") = false, + "Remove an object from the store. If force=True, skip lease and " + "replication task checks.") .def( "remove_by_regex", - [](MooncakeStorePyWrapper &self, const std::string &str) { + [](MooncakeStorePyWrapper &self, const std::string &str, + bool force) { py::gil_scoped_release release; - return self.store_->removeByRegex(str); + return self.store_->removeByRegex(str, force); }, - py::arg("regex_pattern"), + py::arg("regex_pattern"), py::arg("force") = false, "Removes objects from the store whose keys match the given " - "regular expression.") - .def("remove_all", - [](MooncakeStorePyWrapper &self) { - py::gil_scoped_release release; - return self.store_->removeAll(); - }) + "regular expression. If force=True, skip lease and replication " + "task checks.") + .def( + "remove_all", + [](MooncakeStorePyWrapper &self, bool force) { + py::gil_scoped_release release; + return self.store_->removeAll(force); + }, + py::arg("force") = false, + "Remove all objects from the store. If force=True, skip lease " + "and replication task checks.") .def("is_exist", [](MooncakeStorePyWrapper &self, const std::string &key) { py::gil_scoped_release release; diff --git a/mooncake-store/include/client_service.h b/mooncake-store/include/client_service.h index f155152428..4129918c23 100644 --- a/mooncake-store/include/client_service.h +++ b/mooncake-store/include/client_service.h @@ -201,23 +201,28 @@ class Client { /** * @brief Removes an object and all its replicas * @param key Key to remove + * @param force If true, skip lease and replication task checks * @return ErrorCode indicating success/failure */ - tl::expected Remove(const ObjectKey& key); + tl::expected Remove(const ObjectKey& key, + bool force = false); /** * @brief Removes objects from the store whose keys match a regex pattern. * @param str The regular expression string to match against object keys. + * @param force If true, skip lease and replication task checks * @return An expected object containing the number of removed objects on * success, or an ErrorCode on failure. */ - tl::expected RemoveByRegex(const ObjectKey& str); + tl::expected RemoveByRegex(const ObjectKey& str, + bool force = false); /** * @brief Removes all objects and all its replicas + * @param force If true, skip lease and replication task checks * @return tl::expected number of removed objects or error */ - tl::expected RemoveAll(); + tl::expected RemoveAll(bool force = false); /** * @brief Registers a memory segment to master for allocation diff --git a/mooncake-store/include/dummy_client.h b/mooncake-store/include/dummy_client.h index c4a34d479b..3c98af583c 100644 --- a/mooncake-store/include/dummy_client.h +++ b/mooncake-store/include/dummy_client.h @@ -133,11 +133,11 @@ class DummyClient : public PyClient { [[nodiscard]] std::string get_hostname() const; - int remove(const std::string &key); + int remove(const std::string &key, bool force = false); - long removeByRegex(const std::string &str); + long removeByRegex(const std::string &str, bool force = false); - long removeAll(); + long removeAll(bool force = false); int isExist(const std::string &key); diff --git a/mooncake-store/include/master_client.h b/mooncake-store/include/master_client.h index 1cfb0773a9..63109d23f8 100644 --- a/mooncake-store/include/master_client.h +++ b/mooncake-store/include/master_client.h @@ -194,24 +194,28 @@ class MasterClient { /** * @brief Removes an object and all its replicas * @param key Key to remove + * @param force If true, skip lease and replication task checks * @return tl::expected indicating success/failure */ - [[nodiscard]] tl::expected Remove(const std::string& key); + [[nodiscard]] tl::expected Remove(const std::string& key, + bool force = false); /** * @brief Removes objects from the master whose keys match a regex pattern. * @param str The regular expression string to match against object keys. + * @param force If true, skip lease and replication task checks * @return An expected object containing the number of removed objects on * success, or an ErrorCode on failure. */ [[nodiscard]] tl::expected RemoveByRegex( - const std::string& str); + const std::string& str, bool force = false); /** * @brief Removes all objects and all its replicas + * @param force If true, skip lease and replication task checks * @return tl::expected number of removed objects or error */ - [[nodiscard]] tl::expected RemoveAll(); + [[nodiscard]] tl::expected RemoveAll(bool force = false); /** * @brief Registers a segment to master for allocation diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 609cd752a0..ba33a47d89 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -55,8 +55,8 @@ class MasterService { * be mounted temporarily, * ErrorCode::INTERNAL_ERROR on internal errors. */ - auto MountSegment(const Segment& segment, const UUID& client_id) - -> tl::expected; + auto MountSegment(const Segment& segment, + const UUID& client_id) -> tl::expected; /** * @brief Re-mount segments, invoked when the client is the first time to @@ -78,8 +78,8 @@ class MasterService { * ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS if the segment is * currently unmounting. */ - auto UnmountSegment(const UUID& segment_id, const UUID& client_id) - -> tl::expected; + auto UnmountSegment(const UUID& segment_id, + const UUID& client_id) -> tl::expected; /** * @brief Check if an object exists @@ -130,9 +130,10 @@ class MasterService { * clients are omitted from the result map. Clients that exist but have no * IPs are included with empty vectors. */ - auto BatchQueryIp(const std::vector& client_ids) -> tl::expected< - std::unordered_map, boost::hash>, - ErrorCode>; + auto BatchQueryIp(const std::vector& client_ids) + -> tl::expected, + boost::hash>, + ErrorCode>; /** * @brief Batch clear KV cache replicas for specified object keys. @@ -272,24 +273,30 @@ class MasterService { /** * @brief Remove an object and its replicas + * @param key The key to remove. + * @param force If true, skip lease and replication task checks. * @return ErrorCode::OK on success, ErrorCode::OBJECT_NOT_FOUND if not * found */ - auto Remove(const std::string& key) -> tl::expected; + auto Remove(const std::string& key, + bool force = false) -> tl::expected; /** * @brief Removes objects from the master whose keys match a regex pattern. * @param str The regular expression string to match against object keys. + * @param force If true, skip lease and replication task checks. * @return An expected object containing the number of removed objects on * success, or an ErrorCode on failure. */ - auto RemoveByRegex(const std::string& str) -> tl::expected; + auto RemoveByRegex(const std::string& str, + bool force = false) -> tl::expected; /** * @brief Remove all objects and their replicas + * @param force If true, skip lease and replication task checks. * @return return the number of objects removed */ - long RemoveAll(); + long RemoveAll(bool force = false); /** * @brief Get the count of keys @@ -344,10 +351,10 @@ class MasterService { * @param metadatas The corresponding metadata for each offloaded object, * including size, storage location, etc. */ - auto NotifyOffloadSuccess( - const UUID& client_id, const std::vector& keys, - const std::vector& metadatas) - -> tl::expected; + auto NotifyOffloadSuccess(const UUID& client_id, + const std::vector& keys, + const std::vector& + metadatas) -> tl::expected; /** * @brief Create a copy task to copy an object's replicas to target segments diff --git a/mooncake-store/include/pyclient.h b/mooncake-store/include/pyclient.h index fe7f1f3783..8ab307a8c6 100644 --- a/mooncake-store/include/pyclient.h +++ b/mooncake-store/include/pyclient.h @@ -107,11 +107,11 @@ class PyClient { [[nodiscard]] virtual std::string get_hostname() const = 0; - virtual int remove(const std::string &key) = 0; + virtual int remove(const std::string &key, bool force = false) = 0; - virtual long removeByRegex(const std::string &str) = 0; + virtual long removeByRegex(const std::string &str, bool force = false) = 0; - virtual long removeAll() = 0; + virtual long removeAll(bool force = false) = 0; virtual int isExist(const std::string &key) = 0; diff --git a/mooncake-store/include/real_client.h b/mooncake-store/include/real_client.h index 40635ab290..3fe01ce2a3 100644 --- a/mooncake-store/include/real_client.h +++ b/mooncake-store/include/real_client.h @@ -244,11 +244,11 @@ class RealClient : public PyClient { std::vector> batch_get_buffer( const std::vector &keys); - int remove(const std::string &key); + int remove(const std::string &key, bool force = false); - long removeByRegex(const std::string &str); + long removeByRegex(const std::string &str, bool force = false); - long removeAll(); + long removeAll(bool force = false); int tearDownAll(); @@ -420,12 +420,13 @@ class RealClient : public PyClient { std::shared_ptr client_buffer_allocator = nullptr); - tl::expected remove_internal(const std::string &key); + tl::expected remove_internal(const std::string &key, + bool force = false); - tl::expected removeByRegex_internal( - const std::string &str); + tl::expected removeByRegex_internal(const std::string &str, + bool force = false); - tl::expected removeAll_internal(); + tl::expected removeAll_internal(bool force = false); tl::expected tearDownAll_internal(); diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 1845323eea..2a7474c1ad 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -76,11 +76,13 @@ class WrappedMasterService { std::vector> BatchPutRevoke( const UUID& client_id, const std::vector& keys); - tl::expected Remove(const std::string& key); + tl::expected Remove(const std::string& key, + bool force = false); - tl::expected RemoveByRegex(const std::string& str); + tl::expected RemoveByRegex(const std::string& str, + bool force = false); - long RemoveAll(); + long RemoveAll(bool force = false); tl::expected MountSegment(const Segment& segment, const UUID& client_id); diff --git a/mooncake-store/src/client_service.cpp b/mooncake-store/src/client_service.cpp index 4e22193b1b..d95eefbd7c 100644 --- a/mooncake-store/src/client_service.cpp +++ b/mooncake-store/src/client_service.cpp @@ -1402,8 +1402,8 @@ std::vector> Client::BatchPut( return CollectResults(ops); } -tl::expected Client::Remove(const ObjectKey& key) { - auto result = master_client_.Remove(key); +tl::expected Client::Remove(const ObjectKey& key, bool force) { + auto result = master_client_.Remove(key, force); // if (storage_backend_) { // storage_backend_->RemoveFile(key); // } @@ -1413,8 +1413,9 @@ tl::expected Client::Remove(const ObjectKey& key) { return {}; } -tl::expected Client::RemoveByRegex(const ObjectKey& str) { - auto result = master_client_.RemoveByRegex(str); +tl::expected Client::RemoveByRegex(const ObjectKey& str, + bool force) { + auto result = master_client_.RemoveByRegex(str, force); // if (storage_backend_) { // storage_backend_->RemoveByRegex(str); // } @@ -1424,11 +1425,11 @@ tl::expected Client::RemoveByRegex(const ObjectKey& str) { return result.value(); } -tl::expected Client::RemoveAll() { +tl::expected Client::RemoveAll(bool force) { // if (storage_backend_) { // storage_backend_->RemoveAll(); // } - return master_client_.RemoveAll(); + return master_client_.RemoveAll(force); } tl::expected Client::MountSegment(const void* buffer, diff --git a/mooncake-store/src/dummy_client.cpp b/mooncake-store/src/dummy_client.cpp index b84b7b3ce1..761166b142 100644 --- a/mooncake-store/src/dummy_client.cpp +++ b/mooncake-store/src/dummy_client.cpp @@ -512,17 +512,19 @@ int DummyClient::put_parts(const std::string& key, key, values, config, client_id_)); } -int DummyClient::remove(const std::string& key) { - return to_py_ret(invoke_rpc<&RealClient::remove_internal, void>(key)); +int DummyClient::remove(const std::string& key, bool force) { + return to_py_ret( + invoke_rpc<&RealClient::remove_internal, void>(key, force)); } -long DummyClient::removeByRegex(const std::string& str) { +long DummyClient::removeByRegex(const std::string& str, bool force) { return to_py_ret( - invoke_rpc<&RealClient::removeByRegex_internal, long>(str)); + invoke_rpc<&RealClient::removeByRegex_internal, long>(str, force)); } -long DummyClient::removeAll() { - return to_py_ret(invoke_rpc<&RealClient::removeAll_internal, int64_t>()); +long DummyClient::removeAll(bool force) { + return to_py_ret( + invoke_rpc<&RealClient::removeAll_internal, int64_t>(force)); } int DummyClient::isExist(const std::string& key) { diff --git a/mooncake-store/src/master_client.cpp b/mooncake-store/src/master_client.cpp index 0af4ad15a4..0be6eced85 100644 --- a/mooncake-store/src/master_client.cpp +++ b/mooncake-store/src/master_client.cpp @@ -517,30 +517,32 @@ std::vector> MasterClient::BatchPutRevoke( return result; } -tl::expected MasterClient::Remove(const std::string& key) { +tl::expected MasterClient::Remove(const std::string& key, + bool force) { ScopedVLogTimer timer(1, "MasterClient::Remove"); - timer.LogRequest("key=", key); + timer.LogRequest("key=", key, ", force=", force); - auto result = invoke_rpc<&WrappedMasterService::Remove, void>(key); + auto result = invoke_rpc<&WrappedMasterService::Remove, void>(key, force); timer.LogResponseExpected(result); return result; } tl::expected MasterClient::RemoveByRegex( - const std::string& str) { + const std::string& str, bool force) { ScopedVLogTimer timer(1, "MasterClient::RemoveByRegex"); - timer.LogRequest("key=", str); + timer.LogRequest("key=", str, ", force=", force); - auto result = invoke_rpc<&WrappedMasterService::RemoveByRegex, long>(str); + auto result = + invoke_rpc<&WrappedMasterService::RemoveByRegex, long>(str, force); timer.LogResponseExpected(result); return result; } -tl::expected MasterClient::RemoveAll() { +tl::expected MasterClient::RemoveAll(bool force) { ScopedVLogTimer timer(1, "MasterClient::RemoveAll"); - timer.LogRequest("action=remove_all_objects"); + timer.LogRequest("action=remove_all_objects, force=", force); - auto result = invoke_rpc<&WrappedMasterService::RemoveAll, long>(); + auto result = invoke_rpc<&WrappedMasterService::RemoveAll, long>(force); timer.LogResponseExpected(result); return result; } diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 1ac731eea8..289d85fc04 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -1257,8 +1257,8 @@ tl::expected MasterService::MoveRevoke( return {}; } -auto MasterService::Remove(const std::string& key) - -> tl::expected { +auto MasterService::Remove(const std::string& key, + bool force) -> tl::expected { MetadataAccessorRW accessor(this, key); if (!accessor.Exists()) { VLOG(1) << "key=" << key << ", error=object_not_found"; @@ -1267,17 +1267,17 @@ auto MasterService::Remove(const std::string& key) auto& metadata = accessor.Get(); - if (!metadata.IsLeaseExpired()) { + if (!force && !metadata.IsLeaseExpired()) { VLOG(1) << "key=" << key << ", error=object_has_lease"; return tl::make_unexpected(ErrorCode::OBJECT_HAS_LEASE); } - if (!metadata.AllReplicas(&Replica::fn_is_completed)) { + if (!force && !metadata.AllReplicas(&Replica::fn_is_completed)) { LOG(ERROR) << "key=" << key << ", error=replica_not_ready"; return tl::make_unexpected(ErrorCode::REPLICA_IS_NOT_READY); } - if (accessor.HasReplicationTask()) { + if (!force && accessor.HasReplicationTask()) { LOG(ERROR) << "key=" << key << ", error=object_has_replication_task"; return tl::make_unexpected(ErrorCode::OBJECT_HAS_REPLICATION_TASK); } @@ -1287,8 +1287,8 @@ auto MasterService::Remove(const std::string& key) return {}; } -auto MasterService::RemoveByRegex(const std::string& regex_pattern) - -> tl::expected { +auto MasterService::RemoveByRegex(const std::string& regex_pattern, + bool force) -> tl::expected { long removed_count = 0; std::regex pattern; @@ -1305,21 +1305,23 @@ auto MasterService::RemoveByRegex(const std::string& regex_pattern) for (auto it = shard->metadata.begin(); it != shard->metadata.end();) { if (std::regex_search(it->first, pattern)) { - if (!it->second.IsLeaseExpired()) { + if (!force && !it->second.IsLeaseExpired()) { VLOG(1) << "key=" << it->first << " matched by regex, but has lease. Skipping " << "removal."; ++it; continue; } - if (!it->second.AllReplicas(&Replica::fn_is_completed)) { + if (!force && + !it->second.AllReplicas(&Replica::fn_is_completed)) { LOG(WARNING) << "key=" << it->first << " matched by regex, but not all replicas " "are complete. Skipping removal."; ++it; continue; } - if (metadata_shards_[i].replication_tasks.contains(it->first)) { + if (!force && + metadata_shards_[i].replication_tasks.contains(it->first)) { LOG(WARNING) << "key=" << it->first << ", matched by regex, but has replication " "task. Skipping removal."; @@ -1342,7 +1344,7 @@ auto MasterService::RemoveByRegex(const std::string& regex_pattern) return removed_count; } -long MasterService::RemoveAll() { +long MasterService::RemoveAll(bool force) { long removed_count = 0; uint64_t total_freed_size = 0; // Store the current time to avoid repeatedly @@ -1355,12 +1357,12 @@ long MasterService::RemoveAll() { continue; } - // Only remove completed objects with expired leases + // Only remove completed objects with expired leases (unless force=true) auto it = shard->metadata.begin(); while (it != shard->metadata.end()) { - if (it->second.IsLeaseExpired(now) && - it->second.AllReplicas(&Replica::fn_is_completed) && - !shard->replication_tasks.contains(it->first)) { + if (force || (it->second.IsLeaseExpired(now) && + it->second.AllReplicas(&Replica::fn_is_completed) && + !shard->replication_tasks.contains(it->first))) { auto mem_rep_count = it->second.CountReplicas(&Replica::fn_is_memory_replica); total_freed_size += it->second.size * mem_rep_count; diff --git a/mooncake-store/src/real_client.cpp b/mooncake-store/src/real_client.cpp index 4c367d0910..203cfe752b 100644 --- a/mooncake-store/src/real_client.cpp +++ b/mooncake-store/src/real_client.cpp @@ -637,44 +637,46 @@ int RealClient::put_parts(const std::string &key, } tl::expected RealClient::remove_internal( - const std::string &key) { + const std::string &key, bool force) { if (!client_) { LOG(ERROR) << "Client is not initialized"; return tl::unexpected(ErrorCode::INVALID_PARAMS); } - auto remove_result = client_->Remove(key); + auto remove_result = client_->Remove(key, force); if (!remove_result) { return tl::unexpected(remove_result.error()); } return {}; } -int RealClient::remove(const std::string &key) { - return to_py_ret(remove_internal(key)); +int RealClient::remove(const std::string &key, bool force) { + return to_py_ret(remove_internal(key, force)); } tl::expected RealClient::removeByRegex_internal( - const std::string &str) { + const std::string &str, bool force) { if (!client_) { LOG(ERROR) << "Client is not initialized"; return tl::unexpected(ErrorCode::INVALID_PARAMS); } - return client_->RemoveByRegex(str); + return client_->RemoveByRegex(str, force); } -long RealClient::removeByRegex(const std::string &str) { - return to_py_ret(removeByRegex_internal(str)); +long RealClient::removeByRegex(const std::string &str, bool force) { + return to_py_ret(removeByRegex_internal(str, force)); } -tl::expected RealClient::removeAll_internal() { +tl::expected RealClient::removeAll_internal(bool force) { if (!client_) { LOG(ERROR) << "Client is not initialized"; return tl::unexpected(ErrorCode::INVALID_PARAMS); } - return client_->RemoveAll(); + return client_->RemoveAll(force); } -long RealClient::removeAll() { return to_py_ret(removeAll_internal()); } +long RealClient::removeAll(bool force) { + return to_py_ret(removeAll_internal(force)); +} tl::expected RealClient::isExist_internal( const std::string &key) { diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index dfb2b6780b..f0a75af42a 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -574,28 +574,31 @@ std::vector> WrappedMasterService::BatchPutRevoke( } tl::expected WrappedMasterService::Remove( - const std::string& key) { + const std::string& key, bool force) { return execute_rpc( - "Remove", [&] { return master_service_.Remove(key); }, - [&](auto& timer) { timer.LogRequest("key=", key); }, + "Remove", [&] { return master_service_.Remove(key, force); }, + [&](auto& timer) { timer.LogRequest("key=", key, ", force=", force); }, [] { MasterMetricManager::instance().inc_remove_requests(); }, [] { MasterMetricManager::instance().inc_remove_failures(); }); } tl::expected WrappedMasterService::RemoveByRegex( - const std::string& str) { + const std::string& str, bool force) { return execute_rpc( - "RemoveByRegex", [&] { return master_service_.RemoveByRegex(str); }, - [&](auto& timer) { timer.LogRequest("regex=", str); }, + "RemoveByRegex", + [&] { return master_service_.RemoveByRegex(str, force); }, + [&](auto& timer) { + timer.LogRequest("regex=", str, ", force=", force); + }, [] { MasterMetricManager::instance().inc_remove_by_regex_requests(); }, [] { MasterMetricManager::instance().inc_remove_by_regex_failures(); }); } -long WrappedMasterService::RemoveAll() { +long WrappedMasterService::RemoveAll(bool force) { ScopedVLogTimer timer(1, "RemoveAll"); - timer.LogRequest("action=remove_all_objects"); + timer.LogRequest("action=remove_all_objects, force=", force); MasterMetricManager::instance().inc_remove_all_requests(); - long result = master_service_.RemoveAll(); + long result = master_service_.RemoveAll(force); timer.LogResponse("items_removed=", result); return result; } diff --git a/mooncake-store/tests/master_service_test.cpp b/mooncake-store/tests/master_service_test.cpp index 5122e7860e..9164bf3c89 100644 --- a/mooncake-store/tests/master_service_test.cpp +++ b/mooncake-store/tests/master_service_test.cpp @@ -4120,6 +4120,189 @@ TEST_F(MasterServiceTest, UpdateTaskNotFound) { EXPECT_EQ(update_res.error(), ErrorCode::TASK_NOT_FOUND); } +// Test force Remove - should bypass lease check +TEST_F(MasterServiceTest, ForceRemoveLeasedObject) { + // Set a long lease TTL so objects will have active leases + const uint64_t kv_lease_ttl = 10000; // 10 seconds + auto service_config = MasterServiceConfig::builder() + .set_default_kv_lease_ttl(kv_lease_ttl) + .build(); + std::unique_ptr service_(new MasterService(service_config)); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + const UUID client_id = generate_uuid(); + + // Put an object + std::string key = "leased_key"; + uint64_t slice_length = 1024; + ReplicateConfig config; + config.replica_num = 1; + auto put_start_result = + service_->PutStart(client_id, key, slice_length, config); + ASSERT_TRUE(put_start_result.has_value()); + auto put_end_result = service_->PutEnd(client_id, key, ReplicaType::MEMORY); + ASSERT_TRUE(put_end_result.has_value()); + + // Verify object exists + auto exist_result = service_->ExistKey(key); + ASSERT_TRUE(exist_result.has_value()); + ASSERT_TRUE(exist_result.value()); + + // Normal remove should fail because object has active lease + auto remove_result_no_force = service_->Remove(key, false); + EXPECT_FALSE(remove_result_no_force.has_value()); + EXPECT_EQ(ErrorCode::OBJECT_HAS_LEASE, remove_result_no_force.error()); + + // Force remove should succeed even with active lease + auto remove_result_force = service_->Remove(key, true); + EXPECT_TRUE(remove_result_force.has_value()); + + // Verify object is removed + auto get_result = service_->GetReplicaList(key); + EXPECT_FALSE(get_result.has_value()); + EXPECT_EQ(ErrorCode::OBJECT_NOT_FOUND, get_result.error()); +} + +// Test force RemoveByRegex - should bypass lease check +TEST_F(MasterServiceTest, ForceRemoveByRegexLeasedObjects) { + // Set a long lease TTL so objects will have active leases + const uint64_t kv_lease_ttl = 10000; // 10 seconds + auto service_config = MasterServiceConfig::builder() + .set_default_kv_lease_ttl(kv_lease_ttl) + .build(); + std::unique_ptr service_(new MasterService(service_config)); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + const UUID client_id = generate_uuid(); + + // Put 5 objects and grant them active leases by reading + for (int i = 0; i < 5; ++i) { + std::string key = "force_regex_key_" + std::to_string(i); + uint64_t slice_length = 1024; + ReplicateConfig config; + config.replica_num = 1; + auto put_start_result = + service_->PutStart(client_id, key, slice_length, config); + ASSERT_TRUE(put_start_result.has_value()); + auto put_end_result = + service_->PutEnd(client_id, key, ReplicaType::MEMORY); + ASSERT_TRUE(put_end_result.has_value()); + // Grant lease by reading the object + auto exist_result = service_->ExistKey(key); + ASSERT_TRUE(exist_result.has_value()); + ASSERT_TRUE(exist_result.value()); + } + + // Normal RemoveByRegex should remove 0 because all objects have active + // leases + auto remove_result_no_force = + service_->RemoveByRegex("^force_regex_key_", false); + ASSERT_TRUE(remove_result_no_force.has_value()); + EXPECT_EQ(0, remove_result_no_force.value()); + + // All objects should still exist + for (int i = 0; i < 5; ++i) { + std::string key = "force_regex_key_" + std::to_string(i); + auto exist_result = service_->ExistKey(key); + ASSERT_TRUE(exist_result.has_value()); + ASSERT_TRUE(exist_result.value()); + } + + // Force RemoveByRegex should remove all 5 objects + auto remove_result_force = + service_->RemoveByRegex("^force_regex_key_", true); + ASSERT_TRUE(remove_result_force.has_value()); + EXPECT_EQ(5, remove_result_force.value()); + + // All objects should be removed + for (int i = 0; i < 5; ++i) { + std::string key = "force_regex_key_" + std::to_string(i); + auto exist_result = service_->ExistKey(key); + ASSERT_TRUE(exist_result.has_value()); + ASSERT_FALSE(exist_result.value()); + } +} + +// Test force RemoveAll - should bypass lease check +TEST_F(MasterServiceTest, ForceRemoveAllLeasedObjects) { + // Set a long lease TTL so objects will have active leases + const uint64_t kv_lease_ttl = 10000; // 10 seconds + auto service_config = MasterServiceConfig::builder() + .set_default_kv_lease_ttl(kv_lease_ttl) + .build(); + std::unique_ptr service_(new MasterService(service_config)); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + const UUID client_id = generate_uuid(); + + // Put 10 objects and grant them active leases by reading + for (int i = 0; i < 10; ++i) { + std::string key = "force_all_key_" + std::to_string(i); + uint64_t slice_length = 1024; + ReplicateConfig config; + config.replica_num = 1; + auto put_start_result = + service_->PutStart(client_id, key, slice_length, config); + ASSERT_TRUE(put_start_result.has_value()); + auto put_end_result = + service_->PutEnd(client_id, key, ReplicaType::MEMORY); + ASSERT_TRUE(put_end_result.has_value()); + // Grant lease by reading the object + auto exist_result = service_->ExistKey(key); + ASSERT_TRUE(exist_result.has_value()); + ASSERT_TRUE(exist_result.value()); + } + + // Normal RemoveAll should remove 0 because all objects have active leases + EXPECT_EQ(0, service_->RemoveAll(false)); + + // All objects should still exist + for (int i = 0; i < 10; ++i) { + std::string key = "force_all_key_" + std::to_string(i); + auto exist_result = service_->ExistKey(key); + ASSERT_TRUE(exist_result.has_value()); + ASSERT_TRUE(exist_result.value()); + } + + // Force RemoveAll should remove all 10 objects + EXPECT_EQ(10, service_->RemoveAll(true)); + + // All objects should be removed + for (int i = 0; i < 10; ++i) { + std::string key = "force_all_key_" + std::to_string(i); + auto exist_result = service_->ExistKey(key); + ASSERT_TRUE(exist_result.has_value()); + ASSERT_FALSE(exist_result.value()); + } +} + +// Test force Remove with processing replicas (not complete) +TEST_F(MasterServiceTest, ForceRemoveProcessingObject) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + const UUID client_id = generate_uuid(); + + // Start a put but don't end it - object will be in PROCESSING state + std::string key = "processing_key"; + uint64_t slice_length = 1024; + ReplicateConfig config; + config.replica_num = 1; + auto put_start_result = + service_->PutStart(client_id, key, slice_length, config); + ASSERT_TRUE(put_start_result.has_value()); + + // Normal remove should fail because replica is not ready + auto remove_result_no_force = service_->Remove(key, false); + EXPECT_FALSE(remove_result_no_force.has_value()); + EXPECT_EQ(ErrorCode::REPLICA_IS_NOT_READY, remove_result_no_force.error()); + + // Force remove should succeed even with processing replica + auto remove_result_force = service_->Remove(key, true); + EXPECT_TRUE(remove_result_force.has_value()); + + // Verify object is removed + auto get_result = service_->GetReplicaList(key); + EXPECT_FALSE(get_result.has_value()); + EXPECT_EQ(ErrorCode::OBJECT_NOT_FOUND, get_result.error()); +} + } // namespace mooncake::test int main(int argc, char** argv) {