From 9733eed09d67431f0997b6106c346e97db04343d Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 14 Jan 2025 09:26:36 -0800 Subject: [PATCH 01/94] reuse transaction level read policy --- src/main/policy.c | 8 ++++-- src/main/policy_config.c | 61 ++++------------------------------------ 2 files changed, 12 insertions(+), 57 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 40d2d3ed94..7a37e0e743 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -451,7 +451,9 @@ as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, } //Initialize policy with global defaults - as_policy_read_copy(config_read_policy, policy); + if (config_read_policy) { + as_policy_read_copy(config_read_policy, policy); + } if (py_policy && py_policy != Py_None) { // Set policy fields @@ -472,7 +474,9 @@ as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, } // Update the policy - POLICY_UPDATE(); + if (policy_p) { + POLICY_UPDATE(); + } return err->code; } diff --git a/src/main/policy_config.c b/src/main/policy_config.c index bb7c7acf77..8c3a7381ce 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -16,6 +16,7 @@ #include #include "policy_config.h" +#include "policy.h" as_status set_optional_key(as_policy_key *target_ptr, PyObject *py_policy, const char *name); @@ -42,9 +43,13 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) { as_status set_policy_status = AEROSPIKE_OK; + as_error err; PyObject *read_policy = PyDict_GetItemString(py_policies, "read"); - set_policy_status = set_read_policy(&config->policies.read, read_policy); + set_policy_status = + pyobject_to_policy_read(NULL, &err, read_policy, &config->policies.read, + NULL, NULL, NULL, NULL); + as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -140,60 +145,6 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) return AEROSPIKE_OK; } -as_status set_read_policy(as_policy_read *read_policy, PyObject *py_policy) -{ - - as_status status = AEROSPIKE_OK; - if (!py_policy) { - return AEROSPIKE_OK; - } - - if (!PyDict_Check(py_policy)) { - return AEROSPIKE_ERR_PARAM; - } - - status = set_base_policy(&read_policy->base, py_policy); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_key(&read_policy->key, py_policy, "key"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_replica(&read_policy->replica, py_policy, "replica"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_bool_property(&read_policy->deserialize, py_policy, - "deserialize"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_ap_read_mode(&read_policy->read_mode_ap, py_policy, - "read_mode_ap"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_sc_read_mode(&read_policy->read_mode_sc, py_policy, - "read_mode_sc"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_int_property(&read_policy->read_touch_ttl_percent, - py_policy, "read_touch_ttl_percent"); - if (status != AEROSPIKE_OK) { - return status; - } - - return AEROSPIKE_OK; -} - as_status set_write_policy(as_policy_write *write_policy, PyObject *py_policy) { From 569274cefa3dbe3bc7e305d9502a509aac3a934a Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 14 Jan 2025 09:30:15 -0800 Subject: [PATCH 02/94] policy.h requires types.h --- src/include/policy.h | 2 ++ src/main/policy_config.c | 1 + 2 files changed, 3 insertions(+) diff --git a/src/include/policy.h b/src/include/policy.h index 0ba3135adf..5a5f15872a 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -29,6 +29,8 @@ #include #include +#include "types.h" + enum Aerospike_serializer_values { SERIALIZER_NONE, /* default handler for serializer type */ SERIALIZER_PYTHON, diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 8c3a7381ce..72a4ff4738 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -16,6 +16,7 @@ #include #include "policy_config.h" + #include "policy.h" as_status set_optional_key(as_policy_key *target_ptr, PyObject *py_policy, From 27cebabc7d30d02574a228d5a30bf653daa3030c Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:22:35 -0800 Subject: [PATCH 03/94] initialize? --- src/main/policy_config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 72a4ff4738..fa341d1554 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -45,6 +45,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) as_status set_policy_status = AEROSPIKE_OK; as_error err; + as_error_init(&err); PyObject *read_policy = PyDict_GetItemString(py_policies, "read"); set_policy_status = From 4aa3282bf9b1a3964881c1b651effb6bd6884815 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 09:16:32 -0700 Subject: [PATCH 04/94] wip --- src/include/policy.h | 9 ++++----- src/main/client/exists.c | 6 +++--- src/main/client/get.c | 6 +++--- src/main/client/select.c | 6 +++--- src/main/policy.c | 17 +++++++++-------- src/main/policy_config.c | 8 ++++---- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 3b7b173378..977a6a8a96 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -209,11 +209,10 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query *config_query_policy, as_exp *exp_list, as_exp **exp_list_p); -as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_read *policy, - as_policy_read **policy_p, - as_policy_read *config_read_policy, - as_exp *exp_list, as_exp **exp_list_p); +as_status as_policy_read_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_read *policy, as_policy_read **policy_p, + as_policy_read *config_read_policy, as_exp *exp_list, as_exp **exp_list_p); as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, PyObject *py_policy, diff --git a/src/main/client/exists.c b/src/main/client/exists.c index 6103530601..9f0a55f2aa 100644 --- a/src/main/client/exists.c +++ b/src/main/client/exists.c @@ -82,9 +82,9 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, key_initialised = true; // Convert python policy object to as_policy_exists - pyobject_to_policy_read(self, &err, py_policy, &read_policy, &read_policy_p, - &self->as->config.policies.read, &exp_list, - &exp_list_p); + as_policy_read_set_from_pyobject( + self, &err, py_policy, &read_policy, &read_policy_p, + &self->as->config.policies.read, &exp_list, &exp_list_p); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } diff --git a/src/main/client/get.c b/src/main/client/get.c index 4faf63b6b5..e7e4ab95e9 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -84,9 +84,9 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, key_initialised = true; // Convert python policy object to as_policy_exists - pyobject_to_policy_read(self, &err, py_policy, &read_policy, &read_policy_p, - &self->as->config.policies.read, &exp_list, - &exp_list_p); + as_policy_read_set_from_pyobject( + self, &err, py_policy, &read_policy, &read_policy_p, + &self->as->config.policies.read, &exp_list, &exp_list_p); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } diff --git a/src/main/client/select.c b/src/main/client/select.c index 92d1978fec..b76f0649bc 100644 --- a/src/main/client/select.c +++ b/src/main/client/select.c @@ -144,9 +144,9 @@ PyObject *AerospikeClient_Select_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_exists - pyobject_to_policy_read(self, &err, py_policy, &read_policy, &read_policy_p, - &self->as->config.policies.read, &exp_list, - &exp_list_p); + as_policy_read_set_from_pyobject( + self, &err, py_policy, &read_policy, &read_policy_p, + &self->as->config.policies.read, &exp_list, &exp_list_p); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } diff --git a/src/main/policy.c b/src/main/policy.c index 528955bad7..f46941e0f2 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -435,16 +435,17 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, } /** - * Converts a PyObject into an as_policy_read object. + * Sets an as_policy_read *policy* from a Python object. + * If *policy_init* is non-NULL, we first initialize *policy* to *policy_init*'s values. + * * Returns AEROSPIKE_OK on success. On error, the err argument is populated. * We assume that the error object and the policy object are already allocated * and initialized (although, we do reset the error object here). */ -as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_read *policy, - as_policy_read **policy_p, - as_policy_read *config_read_policy, - as_exp *exp_list, as_exp **exp_list_p) +as_status as_policy_read_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_read *policy, as_policy_read **policy_p, + as_policy_read *policy_init, as_exp *exp_list, as_exp **exp_list_p) { if (py_policy && py_policy != Py_None) { // Initialize Policy @@ -452,8 +453,8 @@ as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, } //Initialize policy with global defaults - if (config_read_policy) { - as_policy_read_copy(config_read_policy, policy); + if (policy_init) { + as_policy_read_copy(policy_init, policy); } if (py_policy && py_policy != Py_None) { diff --git a/src/main/policy_config.c b/src/main/policy_config.c index d31c5b99b8..aa33f07810 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -47,10 +47,10 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) as_error err; as_error_init(&err); - PyObject *read_policy = PyDict_GetItemString(py_policies, "read"); - set_policy_status = - pyobject_to_policy_read(NULL, &err, read_policy, &config->policies.read, - NULL, NULL, NULL, NULL); + PyObject *py_read_policy = PyDict_GetItemString(py_policies, "read"); + set_policy_status = as_policy_read_set_from_pyobject( + NULL, &err, py_read_policy, &config->policies.read, NULL, NULL, NULL, + NULL); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; From 68cfd9d4ce44a95cc54869adcf71a85f654c4519 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 09:42:08 -0700 Subject: [PATCH 05/94] only set txn and expression fields if we are initializing a txn-level policy --- src/main/policy.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index f46941e0f2..10feee6058 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -302,7 +302,8 @@ static inline void check_and_set_txn_field(as_error *err, static inline as_status pyobject_to_policy_base(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_base *policy, - as_exp *exp_list, as_exp **exp_list_p) + as_exp *exp_list, as_exp **exp_list_p, + bool is_config_policy) { POLICY_SET_FIELD(total_timeout, uint32_t); POLICY_SET_FIELD(socket_timeout, uint32_t); @@ -310,14 +311,16 @@ pyobject_to_policy_base(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(sleep_between_retries, uint32_t); POLICY_SET_FIELD(compress, bool); - // Setting txn field to a non-NULL value in a query or scan policy is a no-op, - // so this is safe to call for a scan/query policy's base policy - check_and_set_txn_field(err, policy, py_policy); - if (err->code != AEROSPIKE_OK) { - return err->code; - } + if (is_config_policy == false) { + // Setting txn field to a non-NULL value in a query or scan policy is a no-op, + // so this is safe to call for a scan/query policy's base policy + check_and_set_txn_field(err, policy, py_policy); + if (err->code != AEROSPIKE_OK) { + return err->code; + } - POLICY_SET_EXPRESSIONS_FIELD(); + POLICY_SET_EXPRESSIONS_FIELD(); + } return AEROSPIKE_OK; } @@ -459,8 +462,9 @@ as_status as_policy_read_set_from_pyobject( if (py_policy && py_policy != Py_None) { // Set policy fields - as_status retval = pyobject_to_policy_base( - self, err, py_policy, &policy->base, exp_list, exp_list_p); + as_status retval = + pyobject_to_policy_base(self, err, py_policy, &policy->base, + exp_list, exp_list_p, policy_init == NULL); if (retval != AEROSPIKE_OK) { return retval; } From 46a56c3d4794e1d43960591d404d5bfdf707f937 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 09:45:53 -0700 Subject: [PATCH 06/94] make more readable --- src/main/policy.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 10feee6058..3462d6570a 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -303,7 +303,7 @@ static inline as_status pyobject_to_policy_base(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_base *policy, as_exp *exp_list, as_exp **exp_list_p, - bool is_config_policy) + bool is_this_txn_policy) { POLICY_SET_FIELD(total_timeout, uint32_t); POLICY_SET_FIELD(socket_timeout, uint32_t); @@ -311,7 +311,7 @@ pyobject_to_policy_base(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(sleep_between_retries, uint32_t); POLICY_SET_FIELD(compress, bool); - if (is_config_policy == false) { + if (is_this_txn_policy) { // Setting txn field to a non-NULL value in a query or scan policy is a no-op, // so this is safe to call for a scan/query policy's base policy check_and_set_txn_field(err, policy, py_policy); @@ -346,7 +346,7 @@ as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = pyobject_to_policy_base( - self, err, py_policy, &policy->base, exp_list, exp_list_p); + self, err, py_policy, &policy->base, exp_list, exp_list_p, ); if (retval != AEROSPIKE_OK) { return retval; } @@ -455,8 +455,10 @@ as_status as_policy_read_set_from_pyobject( POLICY_INIT(as_policy_read); } + bool is_this_txn_policy = policy_init != NULL; + //Initialize policy with global defaults - if (policy_init) { + if (is_this_txn_policy) { as_policy_read_copy(policy_init, policy); } @@ -464,7 +466,7 @@ as_status as_policy_read_set_from_pyobject( // Set policy fields as_status retval = pyobject_to_policy_base(self, err, py_policy, &policy->base, - exp_list, exp_list_p, policy_init == NULL); + exp_list, exp_list_p, is_this_txn_policy); if (retval != AEROSPIKE_OK) { return retval; } From 3bdbca3bf6062ff47b0403a7ea90478036040fa0 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 09:47:19 -0700 Subject: [PATCH 07/94] wip --- src/main/policy.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 3462d6570a..ea9a32bda7 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -300,10 +300,10 @@ static inline void check_and_set_txn_field(as_error *err, } static inline as_status -pyobject_to_policy_base(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_base *policy, - as_exp *exp_list, as_exp **exp_list_p, - bool is_this_txn_policy) +as_policy_base_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, as_policy_base *policy, + as_exp *exp_list, as_exp **exp_list_p, + bool is_this_txn_policy) { POLICY_SET_FIELD(total_timeout, uint32_t); POLICY_SET_FIELD(socket_timeout, uint32_t); @@ -345,7 +345,7 @@ as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - as_status retval = pyobject_to_policy_base( + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, exp_list, exp_list_p, ); if (retval != AEROSPIKE_OK) { return retval; @@ -417,7 +417,7 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query_copy(config_query_policy, policy); if (py_policy && py_policy != Py_None) { - as_status retval = pyobject_to_policy_base( + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, exp_list, exp_list_p); if (retval != AEROSPIKE_OK) { return retval; @@ -464,9 +464,9 @@ as_status as_policy_read_set_from_pyobject( if (py_policy && py_policy != Py_None) { // Set policy fields - as_status retval = - pyobject_to_policy_base(self, err, py_policy, &policy->base, - exp_list, exp_list_p, is_this_txn_policy); + as_status retval = as_policy_base_set_from_pyobject( + self, err, py_policy, &policy->base, exp_list, exp_list_p, + is_this_txn_policy); if (retval != AEROSPIKE_OK) { return retval; } @@ -511,7 +511,7 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - as_status retval = pyobject_to_policy_base( + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, exp_list, exp_list_p); if (retval != AEROSPIKE_OK) { return retval; @@ -553,7 +553,7 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - as_status retval = pyobject_to_policy_base( + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, exp_list, exp_list_p); if (retval != AEROSPIKE_OK) { return retval; @@ -592,7 +592,7 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - as_status retval = pyobject_to_policy_base( + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, exp_list, exp_list_p); if (retval != AEROSPIKE_OK) { return retval; @@ -636,7 +636,7 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - as_status retval = pyobject_to_policy_base( + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, exp_list, exp_list_p); if (retval != AEROSPIKE_OK) { return retval; @@ -684,7 +684,7 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - as_status retval = pyobject_to_policy_base( + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, exp_list, exp_list_p); if (retval != AEROSPIKE_OK) { return retval; From 9e218bb24a42918655e85c2139800f4d80603345 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 09:49:07 -0700 Subject: [PATCH 08/94] temp fix --- src/main/policy.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index ea9a32bda7..ca0dc4bb91 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -346,7 +346,7 @@ as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, ); + self, err, py_policy, &policy->base, exp_list, exp_list_p, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -418,7 +418,7 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p); + self, err, py_policy, &policy->base, exp_list, exp_list_p, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -512,7 +512,7 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p); + self, err, py_policy, &policy->base, exp_list, exp_list_p, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -554,7 +554,7 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p); + self, err, py_policy, &policy->base, exp_list, exp_list_p, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -593,7 +593,7 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p); + self, err, py_policy, &policy->base, exp_list, exp_list_p, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -637,7 +637,7 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p); + self, err, py_policy, &policy->base, exp_list, exp_list_p, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -685,7 +685,7 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p); + self, err, py_policy, &policy->base, exp_list, exp_list_p, true); if (retval != AEROSPIKE_OK) { return retval; } From 3503c71e4eaf390dd328e23a921947bc33c08352 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 10:10:48 -0700 Subject: [PATCH 09/94] also do for write policy --- src/include/policy.h | 11 ++++++----- src/main/client/put.c | 6 +++--- src/main/client/query.c | 2 +- src/main/client/remove_bin.c | 6 +++--- src/main/policy.c | 18 ++++++++++++------ src/main/policy_config.c | 7 +++++-- src/main/query/execute_background.c | 8 ++++---- 7 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 977a6a8a96..38151ecd77 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -227,11 +227,12 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, as_policy_scan *config_scan_policy, as_exp *exp_list, as_exp **exp_list_p); -as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_write *policy, - as_policy_write **policy_p, - as_policy_write *config_write_policy, - as_exp *exp_list, as_exp **exp_list_p); +as_status +as_policy_write_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, as_policy_write *policy, + as_policy_write **policy_p, + as_policy_write *config_write_policy, + as_exp *exp_list, as_exp **exp_list_p); as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, PyObject *py_policy, diff --git a/src/main/client/put.c b/src/main/client/put.c index fa7910bb79..22ef5d64c3 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -99,9 +99,9 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_write - pyobject_to_policy_write(self, &err, py_policy, &write_policy, - &write_policy_p, &self->as->config.policies.write, - &exp_list, &exp_list_p); + as_policy_write_set_from_pyobject( + self, &err, py_policy, &write_policy, &write_policy_p, + &self->as->config.policies.write, &exp_list, &exp_list_p); if (err.code != AEROSPIKE_OK) { goto CLEANUP; diff --git a/src/main/client/query.c b/src/main/client/query.c index fcf69ea8fe..fa9118ac6e 100644 --- a/src/main/client/query.c +++ b/src/main/client/query.c @@ -329,7 +329,7 @@ static PyObject *AerospikeClient_QueryApply_Invoke( is_query_init = true; if (py_policy) { - pyobject_to_policy_write( + as_policy_write_set_from_pyobject( self, &err, py_policy, &write_policy, &write_policy_p, &self->as->config.policies.write, &exp_list, &exp_list_p); diff --git a/src/main/client/remove_bin.c b/src/main/client/remove_bin.c index 1444f57473..8e5c734db2 100644 --- a/src/main/client/remove_bin.c +++ b/src/main/client/remove_bin.c @@ -74,9 +74,9 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, key_initialized = true; // Convert python policy object to as_policy_write - pyobject_to_policy_write(self, err, py_policy, &write_policy, - &write_policy_p, &self->as->config.policies.write, - &exp_list, &exp_list_p); + as_policy_write_set_from_pyobject( + self, err, py_policy, &write_policy, &write_policy_p, + &self->as->config.policies.write, &exp_list, &exp_list_p); if (err->code != AEROSPIKE_OK) { as_error_update(err, AEROSPIKE_ERR_CLIENT, "Incorrect policy"); goto CLEANUP; diff --git a/src/main/policy.c b/src/main/policy.c index ca0dc4bb91..42b30fd6c7 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -577,11 +577,10 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, * We assume that the error object and the policy object are already allocated * and initialized (although, we do reset the error object here). */ -as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_write *policy, - as_policy_write **policy_p, - as_policy_write *config_write_policy, - as_exp *exp_list, as_exp **exp_list_p) +as_status as_policy_write_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_write *policy, as_policy_write **policy_p, + as_policy_write *config_write_policy, as_exp *exp_list, as_exp **exp_list_p) { if (py_policy && py_policy != Py_None) { // Initialize Policy @@ -590,10 +589,13 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, //Initialize policy with global defaults as_policy_write_copy(config_write_policy, policy); + bool is_this_txn_policy = config_write_policy != NULL; + if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, true); + self, err, py_policy, &policy->base, exp_list, exp_list_p, + is_this_txn_policy); if (retval != AEROSPIKE_OK) { return retval; } @@ -606,6 +608,10 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(replica, as_policy_replica); POLICY_SET_FIELD(compression_threshold, uint32_t); POLICY_SET_FIELD(on_locking_only, bool); + if (is_this_txn_policy == false) { + // Only for config level policy + POLICY_SET_FIELD(ttl, uint32_t); + } } // Update the policy diff --git a/src/main/policy_config.c b/src/main/policy_config.c index aa33f07810..4c31df04b9 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -56,8 +56,11 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) return set_policy_status; } - PyObject *write_policy = PyDict_GetItemString(py_policies, "write"); - set_policy_status = set_write_policy(&config->policies.write, write_policy); + PyObject *py_write_policy = PyDict_GetItemString(py_policies, "write"); + set_policy_status = as_policy_write_set_from_pyobject( + NULL, &err, py_write_policy, &config->policies.write, NULL, NULL, NULL, + NULL); + as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } diff --git a/src/main/query/execute_background.c b/src/main/query/execute_background.c index ec2121e242..18d274bf85 100644 --- a/src/main/query/execute_background.c +++ b/src/main/query/execute_background.c @@ -62,10 +62,10 @@ PyObject *AerospikeQuery_ExecuteBackground(AerospikeQuery *self, PyObject *args, goto CLEANUP; } - if (pyobject_to_policy_write(self->client, &err, py_policy, &write_policy, - &write_policy_p, - &self->client->as->config.policies.write, - &exp_list, &exp_list_p) != AEROSPIKE_OK) { + if (as_policy_write_set_from_pyobject( + self->client, &err, py_policy, &write_policy, &write_policy_p, + &self->client->as->config.policies.write, &exp_list, + &exp_list_p) != AEROSPIKE_OK) { goto CLEANUP; } From 7f99d5238fd03a02ca28942d90718cbcadf7fd21 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 10:11:31 -0700 Subject: [PATCH 10/94] now unused --- src/main/policy_config.c | 65 ---------------------------------------- 1 file changed, 65 deletions(-) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 4c31df04b9..e4e0215ffd 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -167,71 +167,6 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) return AEROSPIKE_OK; } -as_status set_write_policy(as_policy_write *write_policy, PyObject *py_policy) -{ - - as_status status = AEROSPIKE_OK; - - if (!py_policy) { - return AEROSPIKE_OK; - } - - if (!PyDict_Check(py_policy)) { - return AEROSPIKE_ERR_PARAM; - } - - status = set_base_policy(&write_policy->base, py_policy); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_key(&write_policy->key, py_policy, "key"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_replica(&write_policy->replica, py_policy, "replica"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_commit_level(&write_policy->commit_level, py_policy, - "commit_level"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_gen(&write_policy->gen, py_policy, "gen"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_exists(&write_policy->exists, py_policy, "exists"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_uint32_property(&write_policy->ttl, py_policy, "ttl"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_uint32_property( - (uint32_t *)&write_policy->compression_threshold, py_policy, - "compression_threshold"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_bool_property(&write_policy->durable_delete, - py_policy, "durable_delete"); - if (status != AEROSPIKE_OK) { - return status; - } - - return AEROSPIKE_OK; -} - as_status set_apply_policy(as_policy_apply *apply_policy, PyObject *py_policy) { From b0bcc0a82f40ca56f84d205e94025a0171c0eebf Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 10:19:55 -0700 Subject: [PATCH 11/94] do for apply --- src/include/policy.h | 11 ++++---- src/main/client/apply.c | 6 ++--- src/main/policy.c | 20 +++++++++------ src/main/policy_config.c | 55 ++++------------------------------------ 4 files changed, 26 insertions(+), 66 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 38151ecd77..a973f5b5c3 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -192,11 +192,12 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, as_policy_admin **policy_p, as_policy_admin *config_admin_policy); -as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_apply *policy, - as_policy_apply **policy_p, - as_policy_apply *config_apply_policy, - as_exp *exp_list, as_exp **exp_list_p); +as_status +as_policy_apply_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, as_policy_apply *policy, + as_policy_apply **policy_p, + as_policy_apply *config_apply_policy, + as_exp *exp_list, as_exp **exp_list_p); as_status pyobject_to_policy_info(as_error *err, PyObject *py_policy, as_policy_info *policy, diff --git a/src/main/client/apply.c b/src/main/client/apply.c index d5b08ae0e0..5aaf72fbd5 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -109,9 +109,9 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_apply - pyobject_to_policy_apply(self, &err, py_policy, &apply_policy, - &apply_policy_p, &self->as->config.policies.apply, - &exp_list, &exp_list_p); + as_policy_apply_set_from_pyobject( + self, &err, py_policy, &apply_policy, &apply_policy_p, + &self->as->config.policies.apply, &exp_list, &exp_list_p); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } diff --git a/src/main/policy.c b/src/main/policy.c index 42b30fd6c7..6b27ae5e5a 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -330,23 +330,27 @@ as_policy_base_set_from_pyobject(AerospikeClient *self, as_error *err, * We assume that the error object and the policy object are already allocated * and initialized (although, we do reset the error object here). */ -as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_apply *policy, - as_policy_apply **policy_p, - as_policy_apply *config_apply_policy, - as_exp *exp_list, as_exp **exp_list_p) +as_status as_policy_apply_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_apply *policy, as_policy_apply **policy_p, + as_policy_apply *config_apply_policy, as_exp *exp_list, as_exp **exp_list_p) { if (py_policy && py_policy != Py_None) { // Initialize Policy POLICY_INIT(as_policy_apply); } - //Initialize policy with global defaults - as_policy_apply_copy(config_apply_policy, policy); + + bool is_this_txn_policy = config_apply_policy != NULL; + if (is_this_txn_policy) { + //Initialize policy with global defaults + as_policy_apply_copy(config_apply_policy, policy); + } if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, true); + self, err, py_policy, &policy->base, exp_list, exp_list_p, + is_this_txn_policy); if (retval != AEROSPIKE_OK) { return retval; } diff --git a/src/main/policy_config.c b/src/main/policy_config.c index e4e0215ffd..09a0a11c87 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -65,8 +65,11 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) return set_policy_status; } - PyObject *apply_policy = PyDict_GetItemString(py_policies, "apply"); - set_policy_status = set_apply_policy(&config->policies.apply, apply_policy); + PyObject *py_apply_policy = PyDict_GetItemString(py_policies, "apply"); + set_policy_status = as_policy_apply_set_from_pyobject( + NULL, &err, py_apply_policy, &config->policies.apply, NULL, NULL, NULL, + NULL); + as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -167,54 +170,6 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) return AEROSPIKE_OK; } -as_status set_apply_policy(as_policy_apply *apply_policy, PyObject *py_policy) -{ - - as_status status = AEROSPIKE_OK; - - if (!py_policy) { - return AEROSPIKE_OK; - } - - if (!PyDict_Check(py_policy)) { - return AEROSPIKE_ERR_PARAM; - } - - status = set_base_policy(&apply_policy->base, py_policy); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_key(&apply_policy->key, py_policy, "key"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_replica(&apply_policy->replica, py_policy, "replica"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_uint32_property(&apply_policy->ttl, py_policy, "ttl"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_commit_level(&apply_policy->commit_level, py_policy, - "commit_level"); - if (status != AEROSPIKE_OK) { - return status; - } - - status = set_optional_bool_property(&apply_policy->durable_delete, - py_policy, "durable_delete"); - if (status != AEROSPIKE_OK) { - return status; - } - - return AEROSPIKE_OK; -} - as_status set_remove_policy(as_policy_remove *remove_policy, PyObject *py_policy) { From 63b79b473a16b0f3e599fd5793766a3d009ad1ab Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 10:32:45 -0700 Subject: [PATCH 12/94] fix --- src/main/policy.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 6b27ae5e5a..4a34fa86d6 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -591,10 +591,12 @@ as_status as_policy_write_set_from_pyobject( POLICY_INIT(as_policy_write); } //Initialize policy with global defaults - as_policy_write_copy(config_write_policy, policy); - bool is_this_txn_policy = config_write_policy != NULL; + if (is_this_txn_policy) { + as_policy_write_copy(config_write_policy, policy); + } + if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( From 9b1c0445af1bc4a7f0b85fb008a0b71f287c3aaf Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 10:51:11 -0700 Subject: [PATCH 13/94] fix --- src/main/policy.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 4a34fa86d6..568915dd96 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -364,8 +364,10 @@ as_status as_policy_apply_set_from_pyobject( POLICY_SET_FIELD(on_locking_only, bool); } - // Update the policy - POLICY_UPDATE(); + if (policy_p) { + // Update the policy + POLICY_UPDATE(); + } return err->code; } @@ -620,8 +622,10 @@ as_status as_policy_write_set_from_pyobject( } } - // Update the policy - POLICY_UPDATE(); + if (policy_p) { + // Update the policy + POLICY_UPDATE(); + } return err->code; } From c43d226c328476f92190cc2e3b0427b87660320e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 11:37:20 -0700 Subject: [PATCH 14/94] wip. not done --- src/main/client/batch_read.c | 1 + src/main/client/put.c | 17 ++++++++++++----- src/main/policy.c | 21 +++++++-------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/main/client/batch_read.c b/src/main/client/batch_read.c index 059b85d7df..2784a7fc4f 100644 --- a/src/main/client/batch_read.c +++ b/src/main/client/batch_read.c @@ -300,6 +300,7 @@ PyObject *AerospikeClient_BatchRead(AerospikeClient *self, PyObject *args, CLEANUP1: if (err.code != AEROSPIKE_OK) { + Py_XDECREF(br_instance); raise_exception(&err); return NULL; } diff --git a/src/main/client/put.c b/src/main/client/put.c index 22ef5d64c3..ad05eb384d 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -49,8 +49,6 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, { // Aerospike Client Arguments as_error err; - as_policy_write write_policy; - as_policy_write *write_policy_p = NULL; as_key key; as_record rec; @@ -99,9 +97,14 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_write - as_policy_write_set_from_pyobject( - self, &err, py_policy, &write_policy, &write_policy_p, - &self->as->config.policies.write, &exp_list, &exp_list_p); + as_policy_write write_policy; + as_policy_write *write_policy_p = NULL; + if (py_policy) { + as_policy_write_set_from_pyobject(self, &err, py_policy, &write_policy, + &self->as->config.policies.write, + &exp_list, &exp_list_p); + write_policy_p = &write_policy; + } if (err.code != AEROSPIKE_OK) { goto CLEANUP; @@ -172,6 +175,10 @@ PyObject *AerospikeClient_Put(AerospikeClient *self, PyObject *args, return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + if (py_serializer_option) { if (PyLong_Check(py_serializer_option)) { self->is_client_put_serializer = true; diff --git a/src/main/policy.c b/src/main/policy.c index 568915dd96..95c492321a 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -454,18 +454,18 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_status as_policy_read_set_from_pyobject( AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_read *policy, as_policy_read **policy_p, - as_policy_read *policy_init, as_exp *exp_list, as_exp **exp_list_p) + as_policy_read *config_read_policy, as_exp *exp_list, as_exp **exp_list_p) { if (py_policy && py_policy != Py_None) { // Initialize Policy POLICY_INIT(as_policy_read); } - bool is_this_txn_policy = policy_init != NULL; + bool is_this_txn_policy = config_read_policy != NULL; //Initialize policy with global defaults if (is_this_txn_policy) { - as_policy_read_copy(policy_init, policy); + as_policy_read_copy(config_read_policy, policy); } if (py_policy && py_policy != Py_None) { @@ -578,7 +578,9 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, } /** - * Converts a PyObject into an as_policy_write object. + * Sets an as_policy_write *policy* from a Python object *py_policy*. + * If *config_write_policy* is non-NULL, we first initialize *policy* to *config_write_policy*'s values. + * * Returns AEROSPIKE_OK on success. On error, the err argument is populated. * We assume that the error object and the policy object are already allocated * and initialized (although, we do reset the error object here). @@ -588,10 +590,6 @@ as_status as_policy_write_set_from_pyobject( as_policy_write *policy, as_policy_write **policy_p, as_policy_write *config_write_policy, as_exp *exp_list, as_exp **exp_list_p) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_write); - } //Initialize policy with global defaults bool is_this_txn_policy = config_write_policy != NULL; @@ -599,7 +597,7 @@ as_status as_policy_write_set_from_pyobject( as_policy_write_copy(config_write_policy, policy); } - if (py_policy && py_policy != Py_None) { + if (py_policy) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, exp_list, exp_list_p, @@ -622,11 +620,6 @@ as_status as_policy_write_set_from_pyobject( } } - if (policy_p) { - // Update the policy - POLICY_UPDATE(); - } - return err->code; } From cdf4f78a71adc4bbcd691541686278737ce2696a Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 16:13:13 -0700 Subject: [PATCH 15/94] wip. remove unnecessary args because we can already access them thru policy obj --- src/include/policy.h | 8 ++-- src/main/client/put.c | 6 +-- src/main/policy.c | 107 ++++++++++++++++-------------------------- 3 files changed, 48 insertions(+), 73 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index a973f5b5c3..05d0ac6bbc 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -210,10 +210,10 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query *config_query_policy, as_exp *exp_list, as_exp **exp_list_p); -as_status as_policy_read_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_read *policy, as_policy_read **policy_p, - as_policy_read *config_read_policy, as_exp *exp_list, as_exp **exp_list_p); +as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, + as_policy_read *policy, + as_policy_read *config_read_policy); as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, PyObject *py_policy, diff --git a/src/main/client/put.c b/src/main/client/put.c index ad05eb384d..62679214ea 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -102,7 +102,7 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, if (py_policy) { as_policy_write_set_from_pyobject(self, &err, py_policy, &write_policy, &self->as->config.policies.write, - &exp_list, &exp_list_p); + true); write_policy_p = &write_policy; } @@ -118,8 +118,8 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, CLEANUP: POOL_DESTROY(&static_pool); - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (write_policy_p && write_policy_p->base.filter_exp) { + as_exp_destroy(write_policy_p->base.filter_exp); } if (key_initialised == true) { diff --git a/src/main/policy.c b/src/main/policy.c index 95c492321a..a3824ba232 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -98,33 +98,30 @@ #define POLICY_SET_EXPRESSIONS_FIELD() \ { \ - if (exp_list) { \ - PyObject *py_field_name = PyUnicode_FromString("expressions"); \ - if (py_field_name == NULL) { \ - PyErr_Clear(); \ - return as_error_update( \ - err, AEROSPIKE_ERR_CLIENT, \ - "Unable to create Python unicode object"); \ - } \ - PyObject *py_exp_list = \ - PyDict_GetItemWithError(py_policy, py_field_name); \ - if (py_exp_list == NULL && PyErr_Occurred()) { \ - PyErr_Clear(); \ - Py_DECREF(py_field_name); \ - return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ - "Unable to fetch expressions field " \ - "from policy dictionary"); \ - } \ + PyObject *py_field_name = PyUnicode_FromString("expressions"); \ + if (py_field_name == NULL) { \ + PyErr_Clear(); \ + return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ + "Unable to create Python unicode object"); \ + } \ + PyObject *py_exp_list = \ + PyDict_GetItemWithError(py_policy, py_field_name); \ + if (py_exp_list == NULL && PyErr_Occurred()) { \ + PyErr_Clear(); \ Py_DECREF(py_field_name); \ - if (py_exp_list) { \ - if (convert_exp_list(self, py_exp_list, &exp_list, err) == \ - AEROSPIKE_OK) { \ - policy->filter_exp = exp_list; \ - *exp_list_p = exp_list; \ - } \ - else { \ - return err->code; \ - } \ + return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ + "Unable to fetch expressions field " \ + "from policy dictionary"); \ + } \ + Py_DECREF(py_field_name); \ + if (py_exp_list) { \ + as_exp exp_list; \ + if (convert_exp_list(self, py_exp_list, &exp_list, err) == \ + AEROSPIKE_OK) { \ + policy->filter_exp = exp_list; \ + } \ + else { \ + return err->code; \ } \ } \ } @@ -302,7 +299,6 @@ static inline void check_and_set_txn_field(as_error *err, static inline as_status as_policy_base_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_base *policy, - as_exp *exp_list, as_exp **exp_list_p, bool is_this_txn_policy) { POLICY_SET_FIELD(total_timeout, uint32_t); @@ -444,53 +440,32 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, } /** - * Sets an as_policy_read *policy* from a Python object. - * If *policy_init* is non-NULL, we first initialize *policy* to *policy_init*'s values. - * + * Sets an as_policy_read *policy* from a Python object *py_policy*. + * * Returns AEROSPIKE_OK on success. On error, the err argument is populated. * We assume that the error object and the policy object are already allocated * and initialized (although, we do reset the error object here). */ -as_status as_policy_read_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_read *policy, as_policy_read **policy_p, - as_policy_read *config_read_policy, as_exp *exp_list, as_exp **exp_list_p) +as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, + as_policy_read *policy, + bool is_policy_txn_level) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_read); - } - - bool is_this_txn_policy = config_read_policy != NULL; - - //Initialize policy with global defaults - if (is_this_txn_policy) { - as_policy_read_copy(config_read_policy, policy); + // Set policy fields + as_status retval = as_policy_base_set_from_pyobject( + self, err, py_policy, &policy->base, is_policy_txn_level); + if (retval != AEROSPIKE_OK) { + return retval; } - if (py_policy && py_policy != Py_None) { - // Set policy fields - as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, - is_this_txn_policy); - if (retval != AEROSPIKE_OK) { - return retval; - } - - POLICY_SET_FIELD(key, as_policy_key); - POLICY_SET_FIELD(replica, as_policy_replica); - POLICY_SET_FIELD(deserialize, bool); - POLICY_SET_FIELD(read_touch_ttl_percent, int); - - // 4.0.0 new policies - POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); - POLICY_SET_FIELD(read_mode_sc, as_policy_read_mode_sc); - } + POLICY_SET_FIELD(key, as_policy_key); + POLICY_SET_FIELD(replica, as_policy_replica); + POLICY_SET_FIELD(deserialize, bool); + POLICY_SET_FIELD(read_touch_ttl_percent, int); - // Update the policy - if (policy_p) { - POLICY_UPDATE(); - } + // 4.0.0 new policies + POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); + POLICY_SET_FIELD(read_mode_sc, as_policy_read_mode_sc); return err->code; } From df7795d509515cabc3419386bd9d2544f30b4c0e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 17:05:39 -0700 Subject: [PATCH 16/94] finish refactor work for write, apply, and read policies --- src/include/policy.h | 43 ++++++--- src/main/client/apply.c | 25 ++--- src/main/client/exists.c | 25 ++--- src/main/client/get.c | 28 +++--- src/main/client/put.c | 16 ++-- src/main/client/query.c | 20 ++-- src/main/client/remove_bin.c | 28 +++--- src/main/client/select.c | 26 ++--- src/main/policy.c | 141 +++++++++++++--------------- src/main/policy_config.c | 9 +- src/main/query/execute_background.c | 25 ++--- 11 files changed, 201 insertions(+), 185 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 05d0ac6bbc..b398e2ec24 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -192,12 +192,10 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, as_policy_admin **policy_p, as_policy_admin *config_admin_policy); -as_status -as_policy_apply_set_from_pyobject(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_apply *policy, - as_policy_apply **policy_p, - as_policy_apply *config_apply_policy, - as_exp *exp_list, as_exp **exp_list_p); +as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_apply *policy, + bool is_policy_txn_level); as_status pyobject_to_policy_info(as_error *err, PyObject *py_policy, as_policy_info *policy, @@ -213,7 +211,7 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_read *policy, - as_policy_read *config_read_policy); + bool is_policy_txn_level); as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, PyObject *py_policy, @@ -228,12 +226,31 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, as_policy_scan *config_scan_policy, as_exp *exp_list, as_exp **exp_list_p); -as_status -as_policy_write_set_from_pyobject(AerospikeClient *self, as_error *err, - PyObject *py_policy, as_policy_write *policy, - as_policy_write **policy_p, - as_policy_write *config_write_policy, - as_exp *exp_list, as_exp **exp_list_p); +// 1. Copies an as_policy_write *policy_defaults* to a txn-level *policy* +// 2. Sets an as_policy_write *policy* from a Python object *py_policy*. +// * *py_policy* must be a Python dictionary, and *policy* and *as_policy_write* must point to a valid as_policy_write instance. +as_status as_policy_write_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_write *policy, as_policy_write *policy_defaults); + +as_status as_policy_read_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_read *policy, as_policy_read *policy_defaults); + +as_status as_policy_apply_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_apply *policy, as_policy_apply *policy_defaults); + +/** + * Sets an as_policy_write *policy* from a Python object *py_policy*. + * *py_policy* must be a Python dictionary, and *policy* must point to a valid as_policy_write instance. + * + * Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. + */ +as_status as_policy_write_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_write *policy, + bool is_policy_txn_level); as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, PyObject *py_policy, diff --git a/src/main/client/apply.c b/src/main/client/apply.c index 5aaf72fbd5..8a199f0d7d 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -63,10 +63,6 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_umodule = NULL; PyObject *py_ufunction = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - as_static_pool static_pool; memset(&static_pool, 0, sizeof(static_pool)); // Initialisation flags @@ -109,11 +105,14 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_apply - as_policy_apply_set_from_pyobject( - self, &err, py_policy, &apply_policy, &apply_policy_p, - &self->as->config.policies.apply, &exp_list, &exp_list_p); - if (err.code != AEROSPIKE_OK) { - goto CLEANUP; + if (py_policy == Py_None) { + as_policy_apply_copy_and_set_from_pyobject( + self, &err, py_policy, &apply_policy, + &self->as->config.policies.apply); + if (err.code != AEROSPIKE_OK) { + goto CLEANUP; + } + apply_policy_p = &apply_policy; } if (PyUnicode_Check(py_module)) { @@ -148,8 +147,8 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, } CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (apply_policy_p && apply_policy_p->base.filter_exp) { + as_exp_destroy(apply_policy_p->base.filter_exp); } if (py_umodule) { @@ -210,6 +209,10 @@ PyObject *AerospikeClient_Apply(AerospikeClient *self, PyObject *args, return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + // Invoke Operation return AerospikeClient_Apply_Invoke(self, py_key, py_module, py_function, py_arglist, py_policy); diff --git a/src/main/client/exists.c b/src/main/client/exists.c index 9f0a55f2aa..410ac80d17 100644 --- a/src/main/client/exists.c +++ b/src/main/client/exists.c @@ -52,10 +52,6 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, as_key key; as_record *rec = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - // Initialisation flags bool key_initialised = false; @@ -82,11 +78,14 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, key_initialised = true; // Convert python policy object to as_policy_exists - as_policy_read_set_from_pyobject( - self, &err, py_policy, &read_policy, &read_policy_p, - &self->as->config.policies.read, &exp_list, &exp_list_p); - if (err.code != AEROSPIKE_OK) { - goto CLEANUP; + if (py_policy) { + as_policy_read_copy_and_set_from_pyobject( + self, &err, py_policy, &read_policy, + &self->as->config.policies.read); + if (err.code != AEROSPIKE_OK) { + goto CLEANUP; + } + read_policy_p = &read_policy; } // Invoke operation @@ -122,8 +121,8 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (read_policy_p && read_policy_p->base.filter_exp) { + as_exp_destroy(read_policy_p->base.filter_exp); } if (key_initialised == true) { @@ -171,6 +170,10 @@ PyObject *AerospikeClient_Exists(AerospikeClient *self, PyObject *args, return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + // Invoke Operation return AerospikeClient_Exists_Invoke(self, py_key, py_policy); } diff --git a/src/main/client/get.c b/src/main/client/get.c index e7e4ab95e9..e64404f262 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -53,10 +53,6 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, as_key key; as_record *rec = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - // Initialised flags bool key_initialised = false; bool record_initialised = false; @@ -83,12 +79,15 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, // Key is successfully initialised. key_initialised = true; - // Convert python policy object to as_policy_exists - as_policy_read_set_from_pyobject( - self, &err, py_policy, &read_policy, &read_policy_p, - &self->as->config.policies.read, &exp_list, &exp_list_p); - if (err.code != AEROSPIKE_OK) { - goto CLEANUP; + if (py_policy) { + // Convert python policy object to as_policy_exists + as_policy_read_copy_and_set_from_pyobject( + self, &err, py_policy, &read_policy, + &self->as->config.policies.read); + if (err.code != AEROSPIKE_OK) { + goto CLEANUP; + } + read_policy_p = &read_policy; } // Invoke operation @@ -117,9 +116,8 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); - ; + if (read_policy_p && read_policy_p->base.filter_exp) { + as_exp_destroy(read_policy_p->base.filter_exp); } if (key_initialised == true) { @@ -169,6 +167,10 @@ PyObject *AerospikeClient_Get(AerospikeClient *self, PyObject *args, return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + // Invoke Operation return AerospikeClient_Get_Invoke(self, py_key, py_policy); } diff --git a/src/main/client/put.c b/src/main/client/put.c index 62679214ea..5b668ec563 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -52,10 +52,6 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, as_key key; as_record rec; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - // Initialisation flags bool key_initialised = false; bool record_initialised = false; @@ -100,14 +96,14 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, as_policy_write write_policy; as_policy_write *write_policy_p = NULL; if (py_policy) { - as_policy_write_set_from_pyobject(self, &err, py_policy, &write_policy, - &self->as->config.policies.write, - true); + as_policy_write_copy_and_set_from_pyobject( + self, &err, py_policy, &write_policy, + &self->as->config.policies.write); write_policy_p = &write_policy; - } - if (err.code != AEROSPIKE_OK) { - goto CLEANUP; + if (err.code != AEROSPIKE_OK) { + goto CLEANUP; + } } // Invoke operation diff --git a/src/main/client/query.c b/src/main/client/query.c index fa9118ac6e..96633011b7 100644 --- a/src/main/client/query.c +++ b/src/main/client/query.c @@ -278,10 +278,6 @@ static PyObject *AerospikeClient_QueryApply_Invoke( PyObject *py_ustr2 = NULL; PyObject *py_ustr3 = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - as_static_pool static_pool; memset(&static_pool, 0, sizeof(static_pool)); @@ -329,10 +325,10 @@ static PyObject *AerospikeClient_QueryApply_Invoke( is_query_init = true; if (py_policy) { - as_policy_write_set_from_pyobject( - self, &err, py_policy, &write_policy, &write_policy_p, - &self->as->config.policies.write, &exp_list, &exp_list_p); - + as_policy_write_copy_and_set_from_pyobject( + self, &err, py_policy, &write_policy, + &self->as->config.policies.write); + write_policy_p = &write_policy; if (err.code != AEROSPIKE_OK) { goto CLEANUP; } @@ -457,8 +453,8 @@ static PyObject *AerospikeClient_QueryApply_Invoke( } CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (write_policy_p) { + as_exp_destroy(write_policy_p->base.filter_exp); } if (py_ustr1) { @@ -525,6 +521,10 @@ PyObject *AerospikeClient_QueryApply(AerospikeClient *self, PyObject *args, return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + // Invoke Operation return AerospikeClient_QueryApply_Invoke( self, namespace, py_set, py_predicate, py_module, py_function, py_args, diff --git a/src/main/client/remove_bin.c b/src/main/client/remove_bin.c index 8e5c734db2..56957f064b 100644 --- a/src/main/client/remove_bin.c +++ b/src/main/client/remove_bin.c @@ -57,10 +57,6 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, int count = 0; PyObject *py_ustr = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - // Get the bin list size; Py_ssize_t size = PyList_Size(py_binList); // Initialize record @@ -74,12 +70,16 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, key_initialized = true; // Convert python policy object to as_policy_write - as_policy_write_set_from_pyobject( - self, err, py_policy, &write_policy, &write_policy_p, - &self->as->config.policies.write, &exp_list, &exp_list_p); - if (err->code != AEROSPIKE_OK) { - as_error_update(err, AEROSPIKE_ERR_CLIENT, "Incorrect policy"); - goto CLEANUP; + if (py_policy != NULL) { + as_policy_write_copy_and_set_from_pyobject( + self, err, py_policy, &write_policy, + &self->as->config.policies.write); + write_policy_p = &write_policy; + + if (err->code != AEROSPIKE_OK) { + as_error_update(err, AEROSPIKE_ERR_CLIENT, "Incorrect policy"); + goto CLEANUP; + } } // Invoke operation @@ -151,8 +151,8 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, as_record_destroy(&rec); - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (write_policy_p->base.filter_exp) { + as_exp_destroy(write_policy_p->base.filter_exp); } if (key_initialized) { @@ -202,6 +202,10 @@ PyObject *AerospikeClient_RemoveBin(AerospikeClient *self, PyObject *args, return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + if (!self || !self->as) { as_error_update(&err, AEROSPIKE_ERR_PARAM, "Invalid aerospike object"); goto CLEANUP; diff --git a/src/main/client/select.c b/src/main/client/select.c index b76f0649bc..57119b97a6 100644 --- a/src/main/client/select.c +++ b/src/main/client/select.c @@ -56,10 +56,6 @@ PyObject *AerospikeClient_Select_Invoke(AerospikeClient *self, PyObject *py_key, bool select_succeeded = false; char **bins = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - // Initialisation flags bool key_initialised = false; @@ -144,11 +140,14 @@ PyObject *AerospikeClient_Select_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_exists - as_policy_read_set_from_pyobject( - self, &err, py_policy, &read_policy, &read_policy_p, - &self->as->config.policies.read, &exp_list, &exp_list_p); - if (err.code != AEROSPIKE_OK) { - goto CLEANUP; + if (py_policy) { + as_policy_read_copy_and_set_from_pyobject( + self, &err, py_policy, &read_policy, + &self->as->config.policies.read); + if (err.code != AEROSPIKE_OK) { + goto CLEANUP; + } + read_policy_p = &read_policy; } // Invoke operation @@ -163,9 +162,8 @@ PyObject *AerospikeClient_Select_Invoke(AerospikeClient *self, PyObject *py_key, } CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); - ; + if (read_policy_p && read_policy_p->base.filter_exp) { + as_exp_destroy(read_policy_p->base.filter_exp); } if (py_ustr) { @@ -219,6 +217,10 @@ PyObject *AerospikeClient_Select(AerospikeClient *self, PyObject *args, return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + // Invoke Operation return AerospikeClient_Select_Invoke(self, py_key, py_bins, py_policy); } diff --git a/src/main/policy.c b/src/main/policy.c index a3824ba232..c1bd6765d9 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -115,7 +115,7 @@ } \ Py_DECREF(py_field_name); \ if (py_exp_list) { \ - as_exp exp_list; \ + as_exp *exp_list = NULL; \ if (convert_exp_list(self, py_exp_list, &exp_list, err) == \ AEROSPIKE_OK) { \ policy->filter_exp = exp_list; \ @@ -326,44 +326,25 @@ as_policy_base_set_from_pyobject(AerospikeClient *self, as_error *err, * We assume that the error object and the policy object are already allocated * and initialized (although, we do reset the error object here). */ -as_status as_policy_apply_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_apply *policy, as_policy_apply **policy_p, - as_policy_apply *config_apply_policy, as_exp *exp_list, as_exp **exp_list_p) +as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_apply *policy, + bool is_this_txn_policy) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_apply); - } - - bool is_this_txn_policy = config_apply_policy != NULL; - if (is_this_txn_policy) { - //Initialize policy with global defaults - as_policy_apply_copy(config_apply_policy, policy); - } - - if (py_policy && py_policy != Py_None) { - // Set policy fields - as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, - is_this_txn_policy); - if (retval != AEROSPIKE_OK) { - return retval; - } - - POLICY_SET_FIELD(key, as_policy_key); - POLICY_SET_FIELD(replica, as_policy_replica); - //POLICY_SET_FIELD(gen, as_policy_gen); removed - POLICY_SET_FIELD(commit_level, as_policy_commit_level); - POLICY_SET_FIELD(durable_delete, bool); - POLICY_SET_FIELD(ttl, uint32_t); - POLICY_SET_FIELD(on_locking_only, bool); + // Set policy fields + as_status retval = as_policy_base_set_from_pyobject( + self, err, py_policy, &policy->base, is_this_txn_policy); + if (retval != AEROSPIKE_OK) { + return retval; } - if (policy_p) { - // Update the policy - POLICY_UPDATE(); - } + POLICY_SET_FIELD(key, as_policy_key); + POLICY_SET_FIELD(replica, as_policy_replica); + //POLICY_SET_FIELD(gen, as_policy_gen); removed + POLICY_SET_FIELD(commit_level, as_policy_commit_level); + POLICY_SET_FIELD(durable_delete, bool); + POLICY_SET_FIELD(ttl, uint32_t); + POLICY_SET_FIELD(on_locking_only, bool); return err->code; } @@ -420,7 +401,7 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, true); + self, err, py_policy, &policy->base, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -439,6 +420,23 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, return err->code; } +as_status as_policy_apply_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_apply *policy, as_policy_apply *policy_defaults) +{ + as_policy_apply_copy(policy_defaults, policy); + return as_policy_apply_set_from_pyobject(self, err, py_policy, policy, + true); +} + +as_status as_policy_read_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_read *policy, as_policy_read *policy_defaults) +{ + as_policy_read_copy(policy_defaults, policy); + return as_policy_read_set_from_pyobject(self, err, py_policy, policy, true); +} + /** * Sets an as_policy_read *policy* from a Python object *py_policy*. * @@ -493,7 +491,7 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, true); + self, err, py_policy, &policy->base, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -535,7 +533,7 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, true); + self, err, py_policy, &policy->base, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -552,47 +550,38 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, return err->code; } -/** - * Sets an as_policy_write *policy* from a Python object *py_policy*. - * If *config_write_policy* is non-NULL, we first initialize *policy* to *config_write_policy*'s values. - * - * Returns AEROSPIKE_OK on success. On error, the err argument is populated. - * We assume that the error object and the policy object are already allocated - * and initialized (although, we do reset the error object here). - */ -as_status as_policy_write_set_from_pyobject( +as_status as_policy_write_copy_and_set_from_pyobject( AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_write *policy, as_policy_write **policy_p, - as_policy_write *config_write_policy, as_exp *exp_list, as_exp **exp_list_p) + as_policy_write *policy, as_policy_write *policy_defaults) { - //Initialize policy with global defaults - bool is_this_txn_policy = config_write_policy != NULL; + as_policy_write_copy(policy_defaults, policy); + return as_policy_write_set_from_pyobject(self, err, py_policy, policy, + true); +} - if (is_this_txn_policy) { - as_policy_write_copy(config_write_policy, policy); +as_status as_policy_write_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_write *policy, + bool is_policy_txn_level) +{ + as_status retval = as_policy_base_set_from_pyobject( + self, err, py_policy, &policy->base, is_policy_txn_level); + if (retval != AEROSPIKE_OK) { + return retval; } - if (py_policy) { - // Set policy fields - as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, - is_this_txn_policy); - if (retval != AEROSPIKE_OK) { - return retval; - } + POLICY_SET_FIELD(key, as_policy_key); + POLICY_SET_FIELD(gen, as_policy_gen); + POLICY_SET_FIELD(exists, as_policy_exists); + POLICY_SET_FIELD(commit_level, as_policy_commit_level); + POLICY_SET_FIELD(durable_delete, bool); + POLICY_SET_FIELD(replica, as_policy_replica); + POLICY_SET_FIELD(compression_threshold, uint32_t); + POLICY_SET_FIELD(on_locking_only, bool); - POLICY_SET_FIELD(key, as_policy_key); - POLICY_SET_FIELD(gen, as_policy_gen); - POLICY_SET_FIELD(exists, as_policy_exists); - POLICY_SET_FIELD(commit_level, as_policy_commit_level); - POLICY_SET_FIELD(durable_delete, bool); - POLICY_SET_FIELD(replica, as_policy_replica); - POLICY_SET_FIELD(compression_threshold, uint32_t); - POLICY_SET_FIELD(on_locking_only, bool); - if (is_this_txn_policy == false) { - // Only for config level policy - POLICY_SET_FIELD(ttl, uint32_t); - } + if (is_policy_txn_level == false) { + // Only for config level policy + POLICY_SET_FIELD(ttl, uint32_t); } return err->code; @@ -621,7 +610,7 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, true); + self, err, py_policy, &policy->base, true); if (retval != AEROSPIKE_OK) { return retval; } @@ -669,7 +658,7 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, exp_list, exp_list_p, true); + self, err, py_policy, &policy->base, true); if (retval != AEROSPIKE_OK) { return retval; } diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 09a0a11c87..474f9d4583 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -49,8 +49,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) PyObject *py_read_policy = PyDict_GetItemString(py_policies, "read"); set_policy_status = as_policy_read_set_from_pyobject( - NULL, &err, py_read_policy, &config->policies.read, NULL, NULL, NULL, - NULL); + NULL, &err, py_read_policy, &config->policies.read, false); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -58,8 +57,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) PyObject *py_write_policy = PyDict_GetItemString(py_policies, "write"); set_policy_status = as_policy_write_set_from_pyobject( - NULL, &err, py_write_policy, &config->policies.write, NULL, NULL, NULL, - NULL); + NULL, &err, py_write_policy, &config->policies.write, false); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -67,8 +65,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) PyObject *py_apply_policy = PyDict_GetItemString(py_policies, "apply"); set_policy_status = as_policy_apply_set_from_pyobject( - NULL, &err, py_apply_policy, &config->policies.apply, NULL, NULL, NULL, - NULL); + NULL, &err, py_apply_policy, &config->policies.apply, false); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; diff --git a/src/main/query/execute_background.c b/src/main/query/execute_background.c index 18d274bf85..d9a8aaab6e 100644 --- a/src/main/query/execute_background.c +++ b/src/main/query/execute_background.c @@ -40,15 +40,15 @@ PyObject *AerospikeQuery_ExecuteBackground(AerospikeQuery *self, PyObject *args, static char *kwlist[] = {"policy", NULL}; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - if (PyArg_ParseTupleAndKeywords(args, kwds, "|O:execute_background", kwlist, &py_policy) == false) { return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + as_error err; as_error_init(&err); @@ -62,11 +62,14 @@ PyObject *AerospikeQuery_ExecuteBackground(AerospikeQuery *self, PyObject *args, goto CLEANUP; } - if (as_policy_write_set_from_pyobject( - self->client, &err, py_policy, &write_policy, &write_policy_p, - &self->client->as->config.policies.write, &exp_list, - &exp_list_p) != AEROSPIKE_OK) { - goto CLEANUP; + if (write_policy_p) { + as_policy_write_copy_and_set_from_pyobject( + self->client, &err, py_policy, &write_policy, + &self->client->as->config.policies.write); + if (err.code != AEROSPIKE_OK) { + goto CLEANUP; + } + write_policy_p = &write_policy; } Py_BEGIN_ALLOW_THREADS @@ -76,8 +79,8 @@ PyObject *AerospikeQuery_ExecuteBackground(AerospikeQuery *self, PyObject *args, CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (write_policy_p && write_policy_p->base.filter_exp) { + as_exp_destroy(write_policy_p->base.filter_exp); ; } From 32fe4965cb707ef16f648e12f93321fdc2a90d91 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 May 2025 17:15:50 -0700 Subject: [PATCH 17/94] make consistent with c client api. fix --- src/include/policy.h | 28 +++++++++++++++++----------- src/main/client/put.c | 8 ++++---- src/main/policy.c | 25 ++++++++++++++----------- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index b398e2ec24..82e6108c3e 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -229,17 +229,23 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, // 1. Copies an as_policy_write *policy_defaults* to a txn-level *policy* // 2. Sets an as_policy_write *policy* from a Python object *py_policy*. // * *py_policy* must be a Python dictionary, and *policy* and *as_policy_write* must point to a valid as_policy_write instance. -as_status as_policy_write_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_write *policy, as_policy_write *policy_defaults); - -as_status as_policy_read_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_read *policy, as_policy_read *policy_defaults); - -as_status as_policy_apply_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_apply *policy, as_policy_apply *policy_defaults); +as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_write *dst, + as_policy_write *src); + +as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_read *dst, + as_policy_read *src); + +as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_apply *dst, + as_policy_apply *src); /** * Sets an as_policy_write *policy* from a Python object *py_policy*. diff --git a/src/main/client/put.c b/src/main/client/put.c index 5b668ec563..c91932ff53 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -66,6 +66,9 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, // Initialize error as_error_init(&err); + as_policy_write write_policy; + as_policy_write *write_policy_p = NULL; + if (!self || !self->as) { as_error_update(&err, AEROSPIKE_ERR_PARAM, "Invalid aerospike object"); goto CLEANUP; @@ -93,17 +96,14 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_write - as_policy_write write_policy; - as_policy_write *write_policy_p = NULL; if (py_policy) { as_policy_write_copy_and_set_from_pyobject( self, &err, py_policy, &write_policy, &self->as->config.policies.write); - write_policy_p = &write_policy; - if (err.code != AEROSPIKE_OK) { goto CLEANUP; } + write_policy_p = &write_policy; } // Invoke operation diff --git a/src/main/policy.c b/src/main/policy.c index c1bd6765d9..7be8d84b14 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -429,12 +429,14 @@ as_status as_policy_apply_copy_and_set_from_pyobject( true); } -as_status as_policy_read_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_read *policy, as_policy_read *policy_defaults) +as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_read *dst, + as_policy_read *src) { - as_policy_read_copy(policy_defaults, policy); - return as_policy_read_set_from_pyobject(self, err, py_policy, policy, true); + as_policy_read_copy(src, dst); + return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true); } /** @@ -550,13 +552,14 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, return err->code; } -as_status as_policy_write_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_write *policy, as_policy_write *policy_defaults) +as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_write *dst, + as_policy_write *src) { - as_policy_write_copy(policy_defaults, policy); - return as_policy_write_set_from_pyobject(self, err, py_policy, policy, - true); + as_policy_write_copy(src, dst); + return as_policy_write_set_from_pyobject(self, err, py_policy, dst, true); } as_status as_policy_write_set_from_pyobject(AerospikeClient *self, From 5e8618ad5d5f8590cbd33f94ffe91522b7b6e7d1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 08:23:44 -0700 Subject: [PATCH 18/94] reorg --- src/include/policy.h | 85 ++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 82e6108c3e..69d86bd688 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -192,11 +192,6 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, as_policy_admin **policy_p, as_policy_admin *config_admin_policy); -as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, - as_error *err, PyObject *py_policy, - as_policy_apply *policy, - bool is_policy_txn_level); - as_status pyobject_to_policy_info(as_error *err, PyObject *py_policy, as_policy_info *policy, as_policy_info **policy_p, @@ -208,11 +203,6 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query *config_query_policy, as_exp *exp_list, as_exp **exp_list_p); -as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, - PyObject *py_policy, - as_policy_read *policy, - bool is_policy_txn_level); - as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_remove *policy, @@ -226,38 +216,6 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, as_policy_scan *config_scan_policy, as_exp *exp_list, as_exp **exp_list_p); -// 1. Copies an as_policy_write *policy_defaults* to a txn-level *policy* -// 2. Sets an as_policy_write *policy* from a Python object *py_policy*. -// * *py_policy* must be a Python dictionary, and *policy* and *as_policy_write* must point to a valid as_policy_write instance. -as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_write *dst, - as_policy_write *src); - -as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_read *dst, - as_policy_read *src); - -as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_apply *dst, - as_policy_apply *src); - -/** - * Sets an as_policy_write *policy* from a Python object *py_policy*. - * *py_policy* must be a Python dictionary, and *policy* must point to a valid as_policy_write instance. - * - * Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. - */ -as_status as_policy_write_set_from_pyobject(AerospikeClient *self, - as_error *err, PyObject *py_policy, - as_policy_write *policy, - bool is_policy_txn_level); - as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_operate *policy, @@ -313,6 +271,49 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_exp *exp_list, as_exp **exp_list_p); +// 1. Copies an as_policy_* *src* to a txn-level *dst* policy +// 2. Sets an as_policy_* *dst* from a Python object *py_policy*. +// *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_write instances. +// Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. + +as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_write *dst, + as_policy_write *src); + +as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_read *dst, + as_policy_read *src); + +as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_apply *dst, + as_policy_apply *src); + +as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_apply *policy, + bool is_policy_txn_level); + +// Sets an as_policy_* *policy* from a Python object *py_policy*. +// *py_policy* must be a Python dictionary, and *policy* must point to a valid as_policy_* instance. +// Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. + +as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, + as_policy_read *policy, + bool is_policy_txn_level); + +as_status as_policy_write_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_write *policy, + bool is_policy_txn_level); + +// TODO: make consistent // metrics_policy must be declared already // py_metrics_policy must be non-NULL as_status From 14d6c0862487b1902c80e3467ccbc1acb0042b02 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 08:24:57 -0700 Subject: [PATCH 19/94] fix --- src/main/client/apply.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/client/apply.c b/src/main/client/apply.c index 8a199f0d7d..5d64f82396 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -105,7 +105,7 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_apply - if (py_policy == Py_None) { + if (py_policy == NULL) { as_policy_apply_copy_and_set_from_pyobject( self, &err, py_policy, &apply_policy, &self->as->config.policies.apply); From 1a63620220b34876f7b80005c69d8d7b996ee7e1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 08:43:29 -0700 Subject: [PATCH 20/94] first round revisions --- src/include/policy.h | 3 +++ src/main/client/apply.c | 2 +- src/main/client/exists.c | 3 +-- src/main/client/get.c | 2 +- src/main/client/put.c | 2 +- src/main/client/query.c | 2 +- src/main/client/remove_bin.c | 2 +- src/main/client/select.c | 2 +- src/main/policy.c | 20 +++++++------------- src/main/query/execute_background.c | 2 +- 10 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 69d86bd688..d3419c868b 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -301,6 +301,9 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, // Sets an as_policy_* *policy* from a Python object *py_policy*. // *py_policy* must be a Python dictionary, and *policy* must point to a valid as_policy_* instance. +// Client instance *self* can be NULL if is_policy_txn_level is false. We only need *self* for parsing Python client +// expressions. +// // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, diff --git a/src/main/client/apply.c b/src/main/client/apply.c index 5d64f82396..e99c98e0b3 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -147,7 +147,7 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, } CLEANUP: - if (apply_policy_p && apply_policy_p->base.filter_exp) { + if (apply_policy_p) { as_exp_destroy(apply_policy_p->base.filter_exp); } diff --git a/src/main/client/exists.c b/src/main/client/exists.c index 410ac80d17..6ba9e316c0 100644 --- a/src/main/client/exists.c +++ b/src/main/client/exists.c @@ -77,7 +77,6 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, // key is initialised successfully key_initialised = true; - // Convert python policy object to as_policy_exists if (py_policy) { as_policy_read_copy_and_set_from_pyobject( self, &err, py_policy, &read_policy, @@ -121,7 +120,7 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, CLEANUP: - if (read_policy_p && read_policy_p->base.filter_exp) { + if (read_policy_p) { as_exp_destroy(read_policy_p->base.filter_exp); } diff --git a/src/main/client/get.c b/src/main/client/get.c index e64404f262..3d6ef84f9c 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -116,7 +116,7 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, CLEANUP: - if (read_policy_p && read_policy_p->base.filter_exp) { + if (read_policy_p) { as_exp_destroy(read_policy_p->base.filter_exp); } diff --git a/src/main/client/put.c b/src/main/client/put.c index c91932ff53..db3da975ec 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -114,7 +114,7 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, CLEANUP: POOL_DESTROY(&static_pool); - if (write_policy_p && write_policy_p->base.filter_exp) { + if (write_policy_p) { as_exp_destroy(write_policy_p->base.filter_exp); } diff --git a/src/main/client/query.c b/src/main/client/query.c index 96633011b7..44d0bca4ec 100644 --- a/src/main/client/query.c +++ b/src/main/client/query.c @@ -328,10 +328,10 @@ static PyObject *AerospikeClient_QueryApply_Invoke( as_policy_write_copy_and_set_from_pyobject( self, &err, py_policy, &write_policy, &self->as->config.policies.write); - write_policy_p = &write_policy; if (err.code != AEROSPIKE_OK) { goto CLEANUP; } + write_policy_p = &write_policy; } char *module_p = NULL; diff --git a/src/main/client/remove_bin.c b/src/main/client/remove_bin.c index 56957f064b..78630065d9 100644 --- a/src/main/client/remove_bin.c +++ b/src/main/client/remove_bin.c @@ -74,12 +74,12 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, as_policy_write_copy_and_set_from_pyobject( self, err, py_policy, &write_policy, &self->as->config.policies.write); - write_policy_p = &write_policy; if (err->code != AEROSPIKE_OK) { as_error_update(err, AEROSPIKE_ERR_CLIENT, "Incorrect policy"); goto CLEANUP; } + write_policy_p = &write_policy; } // Invoke operation diff --git a/src/main/client/select.c b/src/main/client/select.c index 57119b97a6..f78c63e1d7 100644 --- a/src/main/client/select.c +++ b/src/main/client/select.c @@ -162,7 +162,7 @@ PyObject *AerospikeClient_Select_Invoke(AerospikeClient *self, PyObject *py_key, } CLEANUP: - if (read_policy_p && read_policy_p->base.filter_exp) { + if (read_policy_p) { as_exp_destroy(read_policy_p->base.filter_exp); } diff --git a/src/main/policy.c b/src/main/policy.c index 7be8d84b14..966795b337 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -420,13 +420,14 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, return err->code; } -as_status as_policy_apply_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_apply *policy, as_policy_apply *policy_defaults) +as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_apply *dst, + as_policy_apply *src) { - as_policy_apply_copy(policy_defaults, policy); - return as_policy_apply_set_from_pyobject(self, err, py_policy, policy, - true); + as_policy_apply_copy(src, dst); + return as_policy_apply_set_from_pyobject(self, err, py_policy, dst, true); } as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, @@ -439,13 +440,6 @@ as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true); } -/** - * Sets an as_policy_read *policy* from a Python object *py_policy*. - * - * Returns AEROSPIKE_OK on success. On error, the err argument is populated. - * We assume that the error object and the policy object are already allocated - * and initialized (although, we do reset the error object here). - */ as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_read *policy, diff --git a/src/main/query/execute_background.c b/src/main/query/execute_background.c index d9a8aaab6e..94aac4264e 100644 --- a/src/main/query/execute_background.c +++ b/src/main/query/execute_background.c @@ -79,7 +79,7 @@ PyObject *AerospikeQuery_ExecuteBackground(AerospikeQuery *self, PyObject *args, CLEANUP: - if (write_policy_p && write_policy_p->base.filter_exp) { + if (write_policy_p) { as_exp_destroy(write_policy_p->base.filter_exp); ; } From 9eeaee69d7d747a7a3e8bd43e66e52c3a35156e0 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 08:54:28 -0700 Subject: [PATCH 21/94] fix --- src/main/client/apply.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/client/apply.c b/src/main/client/apply.c index e99c98e0b3..00f5610232 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -105,7 +105,7 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_apply - if (py_policy == NULL) { + if (py_policy) { as_policy_apply_copy_and_set_from_pyobject( self, &err, py_policy, &apply_policy, &self->as->config.policies.apply); From a5fcb6fe4be9b1e5deb5cdbffec931a98b212afa Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 09:49:25 -0700 Subject: [PATCH 22/94] wip --- src/include/policy.h | 12 +++-- src/main/policy.c | 17 ++++--- src/main/policy_config.c | 72 +++++++++++++++++++-------- test/new_tests/test_dynamic_config.py | 2 +- 4 files changed, 70 insertions(+), 33 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index d3419c868b..181692fe45 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -300,16 +300,18 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, bool is_policy_txn_level); // Sets an as_policy_* *policy* from a Python object *py_policy*. -// *py_policy* must be a Python dictionary, and *policy* must point to a valid as_policy_* instance. +// *py_policy* must be a non-NULL Python object. If it is not a Python dictionary, an error will be raised. +// (We check if it's a dictionary here because *py_policy*'s type is not validated when parsing API arguments) +// *policy* must point to a valid as_policy_* instance. // Client instance *self* can be NULL if is_policy_txn_level is false. We only need *self* for parsing Python client // expressions. // // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. -as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, - PyObject *py_policy, - as_policy_read *policy, - bool is_policy_txn_level); +as_status as_policy_read_set_from_py_dict(AerospikeClient *self, as_error *err, + PyObject *py_policy, + as_policy_read *policy, + bool is_policy_txn_level); as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, diff --git a/src/main/policy.c b/src/main/policy.c index 966795b337..1b1bb6902b 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -301,6 +301,11 @@ as_policy_base_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_base *policy, bool is_this_txn_policy) { + if (!PyDict_Check(py_policy)) { + return as_error_update(err, AEROSPIKE_ERR_PARAM, + "policy must be a dict"); + } + POLICY_SET_FIELD(total_timeout, uint32_t); POLICY_SET_FIELD(socket_timeout, uint32_t); POLICY_SET_FIELD(max_retries, uint32_t); @@ -437,17 +442,17 @@ as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, as_policy_read *src) { as_policy_read_copy(src, dst); - return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true); + return as_policy_read_set_from_py_dict(self, err, py_policy, dst, true); } -as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, - PyObject *py_policy, - as_policy_read *policy, - bool is_policy_txn_level) +as_status as_policy_read_set_from_py_dict(AerospikeClient *self, as_error *err, + PyObject *py_policy_dict, + as_policy_read *policy, + bool is_policy_txn_level) { // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy, &policy->base, is_policy_txn_level); + self, err, py_policy_dict, &policy->base, is_policy_txn_level); if (retval != AEROSPIKE_OK) { return retval; } diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 474f9d4583..df5c210e45 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -39,23 +39,52 @@ as_status set_optional_int_property(int *property_ptr, PyObject *py_policy, /* * py_policies must exist, and be a dictionary + * TODO: pass in err */ -as_status set_subpolicies(as_config *config, PyObject *py_policies) +as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) { as_status set_policy_status = AEROSPIKE_OK; as_error err; as_error_init(&err); - PyObject *py_read_policy = PyDict_GetItemString(py_policies, "read"); - set_policy_status = as_policy_read_set_from_pyobject( - NULL, &err, py_read_policy, &config->policies.read, false); - as_error_reset(&err); - if (set_policy_status != AEROSPIKE_OK) { - return set_policy_status; + const char *dict_keys[] = {"read", "write", + "apply", "remove", + "query", "scan", + "operate", "batch", + "info", "admin", + "batch_apply", "batch_remove", + "batch_write", "batch_parent_write", + "txn_verify", "txn_roll"}; + PyObject *py_policy_dicts[sizeof(dict_keys)] = {0}; + + for (unsigned long i = 0; i < sizeof(py_policies_dict); i++) { + PyObject *py_policy = + PyDict_GetItemString(py_policies_dict, dict_keys[i]); + if (py_policy == Py_None) { + py_policy = NULL; + } + + if (py_policy) { + if (PyDict_Check(py_policy)) { + py_policy_dicts[i] = py_policy; + } + else { + return AEROSPIKE_ERR_PARAM; + } + } + } + + if (py_policy) { + set_policy_status = as_policy_read_set_from_py_dict( + NULL, &err, py_read_policy, &config->policies.read, false); + as_error_reset(&err); + if (set_policy_status != AEROSPIKE_OK) { + return set_policy_status; + } } - PyObject *py_write_policy = PyDict_GetItemString(py_policies, "write"); + PyObject *py_write_policy = PyDict_GetItemString(py_policies_dict, "write"); set_policy_status = as_policy_write_set_from_pyobject( NULL, &err, py_write_policy, &config->policies.write, false); as_error_reset(&err); @@ -63,7 +92,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) return set_policy_status; } - PyObject *py_apply_policy = PyDict_GetItemString(py_policies, "apply"); + PyObject *py_apply_policy = PyDict_GetItemString(py_policies_dict, "apply"); set_policy_status = as_policy_apply_set_from_pyobject( NULL, &err, py_apply_policy, &config->policies.apply, false); as_error_reset(&err); @@ -71,52 +100,53 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) return set_policy_status; } - PyObject *remove_policy = PyDict_GetItemString(py_policies, "remove"); + PyObject *remove_policy = PyDict_GetItemString(py_policies_dict, "remove"); set_policy_status = set_remove_policy(&config->policies.remove, remove_policy); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } - PyObject *query_policy = PyDict_GetItemString(py_policies, "query"); + PyObject *query_policy = PyDict_GetItemString(py_policies_dict, "query"); set_policy_status = set_query_policy(&config->policies.query, query_policy); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } - PyObject *scan_policy = PyDict_GetItemString(py_policies, "scan"); + PyObject *scan_policy = PyDict_GetItemString(py_policies_dict, "scan"); set_policy_status = set_scan_policy(&config->policies.scan, scan_policy); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } - PyObject *operate_policy = PyDict_GetItemString(py_policies, "operate"); + PyObject *operate_policy = + PyDict_GetItemString(py_policies_dict, "operate"); set_policy_status = set_operate_policy(&config->policies.operate, operate_policy); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } - PyObject *batch_policy = PyDict_GetItemString(py_policies, "batch"); + PyObject *batch_policy = PyDict_GetItemString(py_policies_dict, "batch"); set_policy_status = set_batch_policy(&config->policies.batch, batch_policy); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } - PyObject *info_policy = PyDict_GetItemString(py_policies, "info"); + PyObject *info_policy = PyDict_GetItemString(py_policies_dict, "info"); set_policy_status = set_info_policy(&config->policies.info, info_policy); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } - PyObject *admin_policy = PyDict_GetItemString(py_policies, "admin"); + PyObject *admin_policy = PyDict_GetItemString(py_policies_dict, "admin"); set_policy_status = set_admin_policy(&config->policies.admin, admin_policy); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_apply_policy = - PyDict_GetItemString(py_policies, "batch_apply"); + PyDict_GetItemString(py_policies_dict, "batch_apply"); set_policy_status = set_batch_apply_policy(&config->policies.batch_apply, batch_apply_policy); if (set_policy_status != AEROSPIKE_OK) { @@ -124,7 +154,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) } PyObject *batch_remove_policy = - PyDict_GetItemString(py_policies, "batch_remove"); + PyDict_GetItemString(py_policies_dict, "batch_remove"); set_policy_status = set_batch_remove_policy(&config->policies.batch_remove, batch_remove_policy); if (set_policy_status != AEROSPIKE_OK) { @@ -132,7 +162,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) } PyObject *batch_write_policy = - PyDict_GetItemString(py_policies, "batch_write"); + PyDict_GetItemString(py_policies_dict, "batch_write"); set_policy_status = set_batch_write_policy(&config->policies.batch_write, batch_write_policy); if (set_policy_status != AEROSPIKE_OK) { @@ -140,7 +170,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) } PyObject *batch_parent_write_policy = - PyDict_GetItemString(py_policies, "batch_parent_write"); + PyDict_GetItemString(py_policies_dict, "batch_parent_write"); set_policy_status = set_batch_policy(&config->policies.batch_parent_write, batch_parent_write_policy); if (set_policy_status != AEROSPIKE_OK) { @@ -154,7 +184,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies) for (unsigned long i = 0; i < sizeof(batch_policy_names) / sizeof(batch_policy_names[0]); i++) { PyObject *py_batch_policy = - PyDict_GetItemString(py_policies, batch_policy_names[i]); + PyDict_GetItemString(py_policies_dict, batch_policy_names[i]); set_policy_status = set_batch_policy(batch_policies[i], py_batch_policy); if (set_policy_status != AEROSPIKE_OK) { diff --git a/test/new_tests/test_dynamic_config.py b/test/new_tests/test_dynamic_config.py index cc3abdd156..5cba422648 100644 --- a/test/new_tests/test_dynamic_config.py +++ b/test/new_tests/test_dynamic_config.py @@ -51,7 +51,7 @@ def test_dyn_config_file_has_highest_precedence(self, functional_test_setup, use config = TestBaseClass.get_connection_config() # Uncomment if we want to check that the config file we pass in is valid # The C client prints logs showing that it detects changes to the dynamic config file - # aerospike.set_log_level(aerospike.LOG_LEVEL_TRACE) + aerospike.set_log_level(aerospike.LOG_LEVEL_TRACE) DYN_CONFIG_PATH = "./dyn_config.yml" if use_env_var: AEROSPIKE_CLIENT_CONFIG_URL = "AEROSPIKE_CLIENT_CONFIG_URL" From 6c5d09305b2ddd2edbab1b3140ff072ac1eb8469 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 12:39:42 -0700 Subject: [PATCH 23/94] just validate pyobject in helper function, since we arent doing it elsewhere --- src/include/policy.h | 10 ++--- src/main/policy.c | 86 +++++++++++++++++----------------------- src/main/policy_config.c | 34 ++++++++-------- 3 files changed, 57 insertions(+), 73 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 181692fe45..8587cd4505 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -300,7 +300,7 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, bool is_policy_txn_level); // Sets an as_policy_* *policy* from a Python object *py_policy*. -// *py_policy* must be a non-NULL Python object. If it is not a Python dictionary, an error will be raised. +// *py_policy*. If py_policy is NULL, do nothing. Else if it is not a Python dictionary, an error will be raised. // (We check if it's a dictionary here because *py_policy*'s type is not validated when parsing API arguments) // *policy* must point to a valid as_policy_* instance. // Client instance *self* can be NULL if is_policy_txn_level is false. We only need *self* for parsing Python client @@ -308,10 +308,10 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, // // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. -as_status as_policy_read_set_from_py_dict(AerospikeClient *self, as_error *err, - PyObject *py_policy, - as_policy_read *policy, - bool is_policy_txn_level); +as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, + as_policy_read *policy, + bool is_policy_txn_level); as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, diff --git a/src/main/policy.c b/src/main/policy.c index 1b1bb6902b..3c0ca05cc1 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -39,18 +39,16 @@ #define MAP_WRITE_FLAGS_KEY "map_write_flags" #define BIT_WRITE_FLAGS_KEY "bit_write_flags" -#define POLICY_INIT(__policy) \ - as_error_reset(err); \ - if (!py_policy || py_policy == Py_None) { \ - return err->code; \ +#define POLICY_UPDATE() *policy_p = policy; + +#define VALIDATE_POLICY() \ + if (py_policy == NULL || py_policy == Py_None) { \ + return AEROSPIKE_OK; \ } \ - if (!PyDict_Check(py_policy)) { \ + else if (!PyDict_Check(py_policy)) { \ return as_error_update(err, AEROSPIKE_ERR_PARAM, \ "policy must be a dict"); \ - } \ - __policy##_init(policy); - -#define POLICY_UPDATE() *policy_p = policy; + } // TODO: Python exceptions should be propagated up instead of being cleared // but the policy helper functions don't handle this case and they only populate @@ -242,10 +240,8 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, as_policy_admin *config_admin_policy) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_admin); - } + VALIDATE_POLICY(); + //Initialize policy with global defaults as_policy_admin_copy(config_admin_policy, policy); @@ -301,10 +297,7 @@ as_policy_base_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_base *policy, bool is_this_txn_policy) { - if (!PyDict_Check(py_policy)) { - return as_error_update(err, AEROSPIKE_ERR_PARAM, - "policy must be a dict"); - } + VALIDATE_POLICY() POLICY_SET_FIELD(total_timeout, uint32_t); POLICY_SET_FIELD(socket_timeout, uint32_t); @@ -336,6 +329,8 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, as_policy_apply *policy, bool is_this_txn_policy) { + VALIDATE_POLICY(); + // Set policy fields as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_this_txn_policy); @@ -365,10 +360,8 @@ as_status pyobject_to_policy_info(as_error *err, PyObject *py_policy, as_policy_info **policy_p, as_policy_info *config_info_policy) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_info); - } + VALIDATE_POLICY(); + //Initialize policy with global defaults as_policy_info_copy(config_info_policy, policy); @@ -397,10 +390,8 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query *config_query_policy, as_exp *exp_list, as_exp **exp_list_p) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_query); - } + VALIDATE_POLICY(); + //Initialize policy with global defaults as_policy_query_copy(config_query_policy, policy); @@ -442,17 +433,19 @@ as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, as_policy_read *src) { as_policy_read_copy(src, dst); - return as_policy_read_set_from_py_dict(self, err, py_policy, dst, true); + return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true); } -as_status as_policy_read_set_from_py_dict(AerospikeClient *self, as_error *err, - PyObject *py_policy_dict, - as_policy_read *policy, - bool is_policy_txn_level) +as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, + as_policy_read *policy, + bool is_policy_txn_level) { + VALIDATE_POLICY(); + // Set policy fields as_status retval = as_policy_base_set_from_pyobject( - self, err, py_policy_dict, &policy->base, is_policy_txn_level); + self, err, py_policy, &policy->base, is_policy_txn_level); if (retval != AEROSPIKE_OK) { return retval; } @@ -482,10 +475,7 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, as_policy_remove *config_remove_policy, as_exp *exp_list, as_exp **exp_list_p) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_remove); - } + VALIDATE_POLICY(); //Initialize policy with global defaults as_policy_remove_copy(config_remove_policy, policy); @@ -526,7 +516,7 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, { if (py_policy && py_policy != Py_None) { // Initialize Policy - POLICY_INIT(as_policy_scan); + ; } //Initialize policy with global defaults as_policy_scan_copy(config_scan_policy, policy); @@ -602,10 +592,8 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, as_policy_operate *config_operate_policy, as_exp *exp_list, as_exp **exp_list_p) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_operate); - } + VALIDATE_POLICY(); + //Initialize policy with global defaults as_policy_operate_copy(config_operate_policy, policy); @@ -650,10 +638,8 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, as_policy_batch *config_batch_policy, as_exp *exp_list, as_exp **exp_list_p) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - POLICY_INIT(as_policy_batch); - } + VALIDATE_POLICY() + //Initialize policy with global defaults as_policy_batch_copy(config_batch_policy, policy); @@ -693,7 +679,7 @@ as_status pyobject_to_batch_write_policy(AerospikeClient *self, as_error *err, as_policy_batch_write **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - POLICY_INIT(as_policy_batch_write); + VALIDATE_POLICY() // Set policy fields POLICY_SET_FIELD(key, as_policy_key); @@ -719,7 +705,7 @@ as_status pyobject_to_batch_read_policy(AerospikeClient *self, as_error *err, as_policy_batch_read **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - POLICY_INIT(as_policy_batch_read); + VALIDATE_POLICY() // Set policy fields POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); @@ -742,7 +728,7 @@ as_status pyobject_to_batch_apply_policy(AerospikeClient *self, as_error *err, as_policy_batch_apply **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - POLICY_INIT(as_policy_batch_apply); + VALIDATE_POLICY() // Set policy fields POLICY_SET_FIELD(key, as_policy_key); @@ -767,7 +753,7 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_policy_batch_remove **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - POLICY_INIT(as_policy_batch_remove); + VALIDATE_POLICY() // Set policy fields POLICY_SET_FIELD(key, as_policy_key); @@ -789,7 +775,7 @@ as_status pyobject_to_bit_policy(as_error *err, PyObject *py_policy, as_bit_policy *policy) { as_bit_policy_init(policy); - POLICY_INIT(as_bit_policy); + VALIDATE_POLICY() PyObject *py_bit_flags = PyDict_GetItemString(py_policy, BIT_WRITE_FLAGS_KEY); @@ -813,7 +799,7 @@ as_status pyobject_to_map_policy(as_error *err, PyObject *py_policy, as_map_policy *policy) { // Initialize Policy - POLICY_INIT(as_map_policy); + VALIDATE_POLICY() // Defaults long map_order = AS_MAP_UNORDERED; diff --git a/src/main/policy_config.c b/src/main/policy_config.c index df5c210e45..7c68324933 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -48,6 +48,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) as_error err; as_error_init(&err); + // Validate policy dictionaries before using them to set the C client's config policies const char *dict_keys[] = {"read", "write", "apply", "remove", "query", "scan", @@ -59,29 +60,26 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) PyObject *py_policy_dicts[sizeof(dict_keys)] = {0}; for (unsigned long i = 0; i < sizeof(py_policies_dict); i++) { - PyObject *py_policy = - PyDict_GetItemString(py_policies_dict, dict_keys[i]); - if (py_policy == Py_None) { - py_policy = NULL; + PyObject *py_key = PyUnicode_FromString(dict_keys[i]); + if (py_key == NULL) { + return AEROSPIKE_ERR_CLIENT; } - if (py_policy) { - if (PyDict_Check(py_policy)) { - py_policy_dicts[i] = py_policy; - } - else { - return AEROSPIKE_ERR_PARAM; - } + PyObject *py_policy = + PyDict_GetItemWithError(py_policies_dict, dict_keys[i]); + Py_DECREF(py_key); + if (py_policy == NULL && PyErr_Occurred()) { + return AEROSPIKE_ERR_CLIENT; } + + py_policy_dicts[i] = py_policy; } - if (py_policy) { - set_policy_status = as_policy_read_set_from_py_dict( - NULL, &err, py_read_policy, &config->policies.read, false); - as_error_reset(&err); - if (set_policy_status != AEROSPIKE_OK) { - return set_policy_status; - } + set_policy_status = as_policy_read_set_from_pyobject( + NULL, &err, py_policy_dicts[0], &config->policies.read, false); + as_error_reset(&err); + if (set_policy_status != AEROSPIKE_OK) { + return set_policy_status; } PyObject *py_write_policy = PyDict_GetItemString(py_policies_dict, "write"); From 835815a67a6b398c15c9054005b7a551e72c80e1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 12:47:54 -0700 Subject: [PATCH 24/94] add a useful comment. --- src/main/policy.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/policy.c b/src/main/policy.c index 3c0ca05cc1..3e8f56887b 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -41,6 +41,9 @@ #define POLICY_UPDATE() *policy_p = policy; +// For transaction level policies, before calling this, we convert policy = None to NULL and this never gets called. +// But for config-level policies, we do not check whether py_policy is NULL or None. +// So we want as_policy_*_set_from_pyobject to be a no-op. #define VALIDATE_POLICY() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ From 97aa4a0932111b48a1edd0b1d2078b2de007d796 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 12:52:36 -0700 Subject: [PATCH 25/94] revert --- src/main/policy.c | 2 +- src/main/policy_config.c | 34 ++++------------------------------ 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 3e8f56887b..30e0d23583 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -43,7 +43,7 @@ // For transaction level policies, before calling this, we convert policy = None to NULL and this never gets called. // But for config-level policies, we do not check whether py_policy is NULL or None. -// So we want as_policy_*_set_from_pyobject to be a no-op. +// Either way, we want as_policy_*_set_from_pyobject to be a no-op. #define VALIDATE_POLICY() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 7c68324933..e4109d6154 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -48,43 +48,17 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) as_error err; as_error_init(&err); - // Validate policy dictionaries before using them to set the C client's config policies - const char *dict_keys[] = {"read", "write", - "apply", "remove", - "query", "scan", - "operate", "batch", - "info", "admin", - "batch_apply", "batch_remove", - "batch_write", "batch_parent_write", - "txn_verify", "txn_roll"}; - PyObject *py_policy_dicts[sizeof(dict_keys)] = {0}; - - for (unsigned long i = 0; i < sizeof(py_policies_dict); i++) { - PyObject *py_key = PyUnicode_FromString(dict_keys[i]); - if (py_key == NULL) { - return AEROSPIKE_ERR_CLIENT; - } - - PyObject *py_policy = - PyDict_GetItemWithError(py_policies_dict, dict_keys[i]); - Py_DECREF(py_key); - if (py_policy == NULL && PyErr_Occurred()) { - return AEROSPIKE_ERR_CLIENT; - } - - py_policy_dicts[i] = py_policy; - } - + PyObject *read_policy = PyDict_GetItemString(py_policies_dict, "read"); set_policy_status = as_policy_read_set_from_pyobject( - NULL, &err, py_policy_dicts[0], &config->policies.read, false); + NULL, &err, read_policy, &config->policies.read, false); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } - PyObject *py_write_policy = PyDict_GetItemString(py_policies_dict, "write"); + PyObject *write_policy = PyDict_GetItemString(py_policies_dict, "write"); set_policy_status = as_policy_write_set_from_pyobject( - NULL, &err, py_write_policy, &config->policies.write, false); + NULL, &err, write_policy, &config->policies.write, false); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; From f213a653928722b38f072d66a03307bc1932697a Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 13:39:46 -0700 Subject: [PATCH 26/94] revert --- test/new_tests/test_dynamic_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/new_tests/test_dynamic_config.py b/test/new_tests/test_dynamic_config.py index 5cba422648..cc3abdd156 100644 --- a/test/new_tests/test_dynamic_config.py +++ b/test/new_tests/test_dynamic_config.py @@ -51,7 +51,7 @@ def test_dyn_config_file_has_highest_precedence(self, functional_test_setup, use config = TestBaseClass.get_connection_config() # Uncomment if we want to check that the config file we pass in is valid # The C client prints logs showing that it detects changes to the dynamic config file - aerospike.set_log_level(aerospike.LOG_LEVEL_TRACE) + # aerospike.set_log_level(aerospike.LOG_LEVEL_TRACE) DYN_CONFIG_PATH = "./dyn_config.yml" if use_env_var: AEROSPIKE_CLIENT_CONFIG_URL = "AEROSPIKE_CLIENT_CONFIG_URL" From 284eec3bcd03c0727fcf3a9cb0be5713a4f390da Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 13:58:46 -0700 Subject: [PATCH 27/94] prevent undefined behavior --- src/main/policy.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 30e0d23583..59bfee48ea 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -637,14 +637,13 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, */ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch *policy, - as_policy_batch **policy_p, - as_policy_batch *config_batch_policy, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_batch **policy_p, as_exp *exp_list, + as_exp **exp_list_p) { VALIDATE_POLICY() //Initialize policy with global defaults - as_policy_batch_copy(config_batch_policy, policy); + as_policy_batch_init(policy); if (py_policy && py_policy != Py_None) { // Set policy fields @@ -683,6 +682,7 @@ as_status pyobject_to_batch_write_policy(AerospikeClient *self, as_error *err, as_exp *exp_list, as_exp **exp_list_p) { VALIDATE_POLICY() + as_policy_batch_write_init(policy); // Set policy fields POLICY_SET_FIELD(key, as_policy_key); @@ -709,6 +709,7 @@ as_status pyobject_to_batch_read_policy(AerospikeClient *self, as_error *err, as_exp *exp_list, as_exp **exp_list_p) { VALIDATE_POLICY() + as_policy_batch_read_init(policy); // Set policy fields POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); @@ -729,9 +730,11 @@ as_status pyobject_to_batch_apply_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_apply *policy, as_policy_batch_apply **policy_p, + as_policy_batch_apply *src, as_exp *exp_list, as_exp **exp_list_p) { VALIDATE_POLICY() + as_policy_batch_apply_init(policy); // Set policy fields POLICY_SET_FIELD(key, as_policy_key); @@ -757,6 +760,7 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_exp *exp_list, as_exp **exp_list_p) { VALIDATE_POLICY() + as_policy_batch_remove_init(policy); // Set policy fields POLICY_SET_FIELD(key, as_policy_key); @@ -803,6 +807,7 @@ as_status pyobject_to_map_policy(as_error *err, PyObject *py_policy, { // Initialize Policy VALIDATE_POLICY() + as_map_policy_init(policy); // Defaults long map_order = AS_MAP_UNORDERED; From 8bfb080728ef05f1fdf75a1c45ebf5c0121ea118 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 15:22:53 -0700 Subject: [PATCH 28/94] fix --- src/main/policy.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/policy.c b/src/main/policy.c index 59bfee48ea..23021f4028 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -730,7 +730,6 @@ as_status pyobject_to_batch_apply_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_apply *policy, as_policy_batch_apply **policy_p, - as_policy_batch_apply *src, as_exp *exp_list, as_exp **exp_list_p) { VALIDATE_POLICY() From bbb82dffbfaf2439d45de26862a1cc3168009063 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 15:36:53 -0700 Subject: [PATCH 29/94] fix --- src/main/policy.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/policy.c b/src/main/policy.c index 23021f4028..75cfd9fece 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -637,13 +637,14 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, */ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch *policy, + as_policy_batch *config_batch_policy, as_policy_batch **policy_p, as_exp *exp_list, as_exp **exp_list_p) { VALIDATE_POLICY() //Initialize policy with global defaults - as_policy_batch_init(policy); + as_policy_batch_copy(config_batch_policy, policy); if (py_policy && py_policy != Py_None) { // Set policy fields From f04bd2decaea59e5454312d388861221261e3af4 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 15:39:11 -0700 Subject: [PATCH 30/94] wrong order --- src/main/policy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 75cfd9fece..c6a2197651 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -637,9 +637,9 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, */ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch *policy, + as_policy_batch **policy_p, as_policy_batch *config_batch_policy, - as_policy_batch **policy_p, as_exp *exp_list, - as_exp **exp_list_p) + as_exp *exp_list, as_exp **exp_list_p) { VALIDATE_POLICY() From 57f9f4119d2eb8d34ce445958729f9415a2f79d0 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 21 May 2025 15:46:54 -0700 Subject: [PATCH 31/94] missing validation --- src/main/policy.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index c6a2197651..c1e2584e13 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -517,10 +517,8 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, as_policy_scan *config_scan_policy, as_exp *exp_list, as_exp **exp_list_p) { - if (py_policy && py_policy != Py_None) { - // Initialize Policy - ; - } + VALIDATE_POLICY(); + //Initialize policy with global defaults as_policy_scan_copy(config_scan_policy, policy); @@ -559,6 +557,8 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_policy_write *policy, bool is_policy_txn_level) { + VALIDATE_POLICY(); + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_policy_txn_level); if (retval != AEROSPIKE_OK) { From 77fe7077459ca103c82c3b328457eb7bdb3e8d70 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 22 May 2025 13:51:03 -0700 Subject: [PATCH 32/94] fix --- src/main/client/remove_bin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/client/remove_bin.c b/src/main/client/remove_bin.c index 78630065d9..5fd2e0067d 100644 --- a/src/main/client/remove_bin.c +++ b/src/main/client/remove_bin.c @@ -151,7 +151,7 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, as_record_destroy(&rec); - if (write_policy_p->base.filter_exp) { + if (write_policy_p) { as_exp_destroy(write_policy_p->base.filter_exp); } From 7d026419cd7d60a031e90bf51a4895adf0e1d6bd Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 22 May 2025 14:24:53 -0700 Subject: [PATCH 33/94] fix --- src/main/query/execute_background.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/query/execute_background.c b/src/main/query/execute_background.c index 94aac4264e..5827871c99 100644 --- a/src/main/query/execute_background.c +++ b/src/main/query/execute_background.c @@ -62,7 +62,7 @@ PyObject *AerospikeQuery_ExecuteBackground(AerospikeQuery *self, PyObject *args, goto CLEANUP; } - if (write_policy_p) { + if (py_policy) { as_policy_write_copy_and_set_from_pyobject( self->client, &err, py_policy, &write_policy, &self->client->as->config.policies.write); From 1cec276b201e31c1374da9211b4925e045b194bd Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 22 May 2025 14:38:41 -0700 Subject: [PATCH 34/94] key being overwritten --- src/main/client/type.c | 107 +++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/src/main/client/type.c b/src/main/client/type.c index 8225978425..e1f2769261 100644 --- a/src/main/client/type.c +++ b/src/main/client/type.c @@ -769,7 +769,61 @@ static int AerospikeClient_Type_Init(AerospikeClient *self, PyObject *args, //Set default value of use_batch_direct PyObject *py_policies = PyDict_GetItemString(py_config, "policies"); + if (py_policies && PyDict_Check(py_policies)) { + /* + * Generation policy is removed from constructor. + */ + + /* + * Set individual policy groups, and base policies for each + * Set the individual policy groups new in 3.0 + * */ + + if (set_subpolicies(&config, py_policies) != AEROSPIKE_OK) { + error_code = INIT_POLICY_PARAM_ERR; + goto CONSTRUCTOR_ERROR; + } + + // See comment at end of set_subpolicies() for why we process metrics policy here + PyObject *py_metrics_policy_option_name = + PyUnicode_FromString("metrics"); + if (py_metrics_policy_option_name == NULL) { + goto RAISE_EXCEPTION_WITHOUT_AS_ERROR; + } + PyObject *py_obj_metrics_policy = + PyDict_GetItemWithError(py_policies, py_metrics_policy_option_name); + Py_DECREF(py_metrics_policy_option_name); + + if (py_obj_metrics_policy == NULL) { + if (PyErr_Occurred()) { + goto RAISE_EXCEPTION_WITHOUT_AS_ERROR; + } + // User didn't provide default metrics policy. + // It is optional so just move on + } + else if (is_pyobj_correct_as_helpers_type(py_obj_metrics_policy, + "metrics", "MetricsPolicy", + false) == false) { + // set_as_metrics_policy_using_pyobject also checks the type of the pyobject + // But we want to set a different error message here + as_error_update( + &constructor_err, AEROSPIKE_ERR_PARAM, + "metrics must be an " + "aerospike_helpers.metrics.MetricsPolicy class instance. But " + "a %s was received instead", + py_obj_metrics_policy->ob_type->tp_name); + goto RAISE_EXCEPTION_WITH_AS_ERROR; + } + else { + as_status status = set_as_metrics_policy_using_pyobject( + &constructor_err, py_obj_metrics_policy, + &(config.policies.metrics)); + if (status != AEROSPIKE_OK) { + goto RAISE_EXCEPTION_WITH_AS_ERROR; + } + } + //global defaults setting PyObject *py_key_policy = PyDict_GetItemString(py_policies, "key"); if (py_key_policy && PyLong_Check(py_key_policy)) { @@ -890,59 +944,6 @@ static int AerospikeClient_Type_Init(AerospikeClient *self, PyObject *args, config.thread_pool_size = PyLong_AsLong(py_thread_pool_size); } - /* - * Generation policy is removed from constructor. - */ - - /* - * Set individual policy groups, and base policies for each - * Set the individual policy groups new in 3.0 - * */ - - if (set_subpolicies(&config, py_policies) != AEROSPIKE_OK) { - error_code = INIT_POLICY_PARAM_ERR; - goto CONSTRUCTOR_ERROR; - } - - // See comment at end of set_subpolicies() for why we process metrics policy here - PyObject *py_metrics_policy_option_name = - PyUnicode_FromString("metrics"); - if (py_metrics_policy_option_name == NULL) { - goto RAISE_EXCEPTION_WITHOUT_AS_ERROR; - } - PyObject *py_obj_metrics_policy = - PyDict_GetItemWithError(py_policies, py_metrics_policy_option_name); - Py_DECREF(py_metrics_policy_option_name); - - if (py_obj_metrics_policy == NULL) { - if (PyErr_Occurred()) { - goto RAISE_EXCEPTION_WITHOUT_AS_ERROR; - } - // User didn't provide default metrics policy. - // It is optional so just move on - } - else if (is_pyobj_correct_as_helpers_type(py_obj_metrics_policy, - "metrics", "MetricsPolicy", - false) == false) { - // set_as_metrics_policy_using_pyobject also checks the type of the pyobject - // But we want to set a different error message here - as_error_update( - &constructor_err, AEROSPIKE_ERR_PARAM, - "metrics must be an " - "aerospike_helpers.metrics.MetricsPolicy class instance. But " - "a %s was received instead", - py_obj_metrics_policy->ob_type->tp_name); - goto RAISE_EXCEPTION_WITH_AS_ERROR; - } - else { - as_status status = set_as_metrics_policy_using_pyobject( - &constructor_err, py_obj_metrics_policy, - &(config.policies.metrics)); - if (status != AEROSPIKE_OK) { - goto RAISE_EXCEPTION_WITH_AS_ERROR; - } - } - PyObject *py_login_timeout = PyDict_GetItemString(py_policies, "login_timeout_ms"); if (py_login_timeout && PyLong_Check(py_login_timeout)) { From c7cbfc10ef753cc5a2ba1ec9457cbda9cfc8468d Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 23 May 2025 16:00:43 +0000 Subject: [PATCH 35/94] Revert "key being overwritten" This reverts commit 1cec276b201e31c1374da9211b4925e045b194bd. --- src/main/client/type.c | 107 ++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 54 deletions(-) diff --git a/src/main/client/type.c b/src/main/client/type.c index e1f2769261..8225978425 100644 --- a/src/main/client/type.c +++ b/src/main/client/type.c @@ -769,61 +769,7 @@ static int AerospikeClient_Type_Init(AerospikeClient *self, PyObject *args, //Set default value of use_batch_direct PyObject *py_policies = PyDict_GetItemString(py_config, "policies"); - if (py_policies && PyDict_Check(py_policies)) { - /* - * Generation policy is removed from constructor. - */ - - /* - * Set individual policy groups, and base policies for each - * Set the individual policy groups new in 3.0 - * */ - - if (set_subpolicies(&config, py_policies) != AEROSPIKE_OK) { - error_code = INIT_POLICY_PARAM_ERR; - goto CONSTRUCTOR_ERROR; - } - - // See comment at end of set_subpolicies() for why we process metrics policy here - PyObject *py_metrics_policy_option_name = - PyUnicode_FromString("metrics"); - if (py_metrics_policy_option_name == NULL) { - goto RAISE_EXCEPTION_WITHOUT_AS_ERROR; - } - PyObject *py_obj_metrics_policy = - PyDict_GetItemWithError(py_policies, py_metrics_policy_option_name); - Py_DECREF(py_metrics_policy_option_name); - - if (py_obj_metrics_policy == NULL) { - if (PyErr_Occurred()) { - goto RAISE_EXCEPTION_WITHOUT_AS_ERROR; - } - // User didn't provide default metrics policy. - // It is optional so just move on - } - else if (is_pyobj_correct_as_helpers_type(py_obj_metrics_policy, - "metrics", "MetricsPolicy", - false) == false) { - // set_as_metrics_policy_using_pyobject also checks the type of the pyobject - // But we want to set a different error message here - as_error_update( - &constructor_err, AEROSPIKE_ERR_PARAM, - "metrics must be an " - "aerospike_helpers.metrics.MetricsPolicy class instance. But " - "a %s was received instead", - py_obj_metrics_policy->ob_type->tp_name); - goto RAISE_EXCEPTION_WITH_AS_ERROR; - } - else { - as_status status = set_as_metrics_policy_using_pyobject( - &constructor_err, py_obj_metrics_policy, - &(config.policies.metrics)); - if (status != AEROSPIKE_OK) { - goto RAISE_EXCEPTION_WITH_AS_ERROR; - } - } - //global defaults setting PyObject *py_key_policy = PyDict_GetItemString(py_policies, "key"); if (py_key_policy && PyLong_Check(py_key_policy)) { @@ -944,6 +890,59 @@ static int AerospikeClient_Type_Init(AerospikeClient *self, PyObject *args, config.thread_pool_size = PyLong_AsLong(py_thread_pool_size); } + /* + * Generation policy is removed from constructor. + */ + + /* + * Set individual policy groups, and base policies for each + * Set the individual policy groups new in 3.0 + * */ + + if (set_subpolicies(&config, py_policies) != AEROSPIKE_OK) { + error_code = INIT_POLICY_PARAM_ERR; + goto CONSTRUCTOR_ERROR; + } + + // See comment at end of set_subpolicies() for why we process metrics policy here + PyObject *py_metrics_policy_option_name = + PyUnicode_FromString("metrics"); + if (py_metrics_policy_option_name == NULL) { + goto RAISE_EXCEPTION_WITHOUT_AS_ERROR; + } + PyObject *py_obj_metrics_policy = + PyDict_GetItemWithError(py_policies, py_metrics_policy_option_name); + Py_DECREF(py_metrics_policy_option_name); + + if (py_obj_metrics_policy == NULL) { + if (PyErr_Occurred()) { + goto RAISE_EXCEPTION_WITHOUT_AS_ERROR; + } + // User didn't provide default metrics policy. + // It is optional so just move on + } + else if (is_pyobj_correct_as_helpers_type(py_obj_metrics_policy, + "metrics", "MetricsPolicy", + false) == false) { + // set_as_metrics_policy_using_pyobject also checks the type of the pyobject + // But we want to set a different error message here + as_error_update( + &constructor_err, AEROSPIKE_ERR_PARAM, + "metrics must be an " + "aerospike_helpers.metrics.MetricsPolicy class instance. But " + "a %s was received instead", + py_obj_metrics_policy->ob_type->tp_name); + goto RAISE_EXCEPTION_WITH_AS_ERROR; + } + else { + as_status status = set_as_metrics_policy_using_pyobject( + &constructor_err, py_obj_metrics_policy, + &(config.policies.metrics)); + if (status != AEROSPIKE_OK) { + goto RAISE_EXCEPTION_WITH_AS_ERROR; + } + } + PyObject *py_login_timeout = PyDict_GetItemString(py_policies, "login_timeout_ms"); if (py_login_timeout && PyLong_Check(py_login_timeout)) { From eee5b0c028e9d74cb3f877d97bdfa87999f79069 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 23 May 2025 09:15:10 -0700 Subject: [PATCH 36/94] this should not be handled here --- src/main/client/get.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/main/client/get.c b/src/main/client/get.c index 3d6ef84f9c..1a37bee284 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -101,17 +101,6 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, AEROSPIKE_OK) { goto CLEANUP; } - if (!read_policy_p || - (read_policy_p && read_policy_p->key == AS_POLICY_KEY_DIGEST)) { - // This is a special case. - // C-client returns NULL key, so to the user - // response will be (, , None, ) - // Using the same input key, just making primary key part to be None - // Only in case of POLICY_KEY_DIGEST or no policy specified - PyObject *p_key = PyTuple_GetItem(py_rec, 0); - Py_INCREF(Py_None); - PyTuple_SetItem(p_key, 2, Py_None); - } } CLEANUP: From 6cbd98ac3b78b8a9d11a8d9dab3c650b867aafee Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 15 Jul 2025 07:45:34 -0700 Subject: [PATCH 37/94] clear up --- src/include/policy.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 9e4da3c18f..a7b2668fe6 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -271,8 +271,9 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_exp *exp_list, as_exp **exp_list_p); -// 1. Copies an as_policy_* *src* to a txn-level *dst* policy -// 2. Sets an as_policy_* *dst* from a Python object *py_policy*. +// These methods are used for setting transaction level policies +// 1. Copies an config-level as_policy_* *src* to a txn-level *dst* policy +// 2. Sets an as_policy_* *dst* from a Python object *py_policy*. to override the defaults from *src* // *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_write instances. // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. From a458fcf714dfcd4104dac43a05c0562ae40f937c Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 15 Jul 2025 07:50:31 -0700 Subject: [PATCH 38/94] clear up --- src/include/policy.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index a7b2668fe6..3eba4cde05 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -300,13 +300,19 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, as_policy_apply *policy, bool is_policy_txn_level); -// Sets an as_policy_* *policy* from a Python object *py_policy*. -// *py_policy*. If py_policy is NULL, do nothing. Else if it is not a Python dictionary, an error will be raised. -// (We check if it's a dictionary here because *py_policy*'s type is not validated when parsing API arguments) +// Overrides an as_policy_* *policy* fields using entries from a Python object *py_policy*. +// // *policy* must point to a valid as_policy_* instance. +// +// If py_policy is NULL, do nothing. Else if it is not a Python dictionary, an error will be raised. +// (We check if it's a dictionary here because *py_policy*'s type is not validated when parsing API arguments) +// // Client instance *self* can be NULL if is_policy_txn_level is false. We only need *self* for parsing Python client // expressions. // +// *is_this_txn_policy* is required because there are fields that should only be set in *policy* for txn-level +// policies. +// // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, From c701fa5a46b7d25c76e25fdc5f1ba138e15264f1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 15 Jul 2025 13:19:37 -0700 Subject: [PATCH 39/94] comment --- src/main/client/apply.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/client/apply.c b/src/main/client/apply.c index 00f5610232..de01bd4f86 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -105,6 +105,11 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, } // Convert python policy object to as_policy_apply + // This is a double check if py_policy is NULL where as_policy_apply_set_from_pyobject + // also checks the same thing before setting the apply policy. + // But this is because for the client config dictionary, as_policy_apply_set_from_pyobject is the only + // place where we check if py_policy is NULL before setting apply policy. + // I didn't want to set the pointer to the apply policy in this helper function as well. if (py_policy) { as_policy_apply_copy_and_set_from_pyobject( self, &err, py_policy, &apply_policy, From 4709edb948f9c544a3075ff2f23056d703677b88 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 15 Jul 2025 13:23:39 -0700 Subject: [PATCH 40/94] revise --- src/main/client/apply.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/client/apply.c b/src/main/client/apply.c index de01bd4f86..8f0a8f3344 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -109,7 +109,9 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, // also checks the same thing before setting the apply policy. // But this is because for the client config dictionary, as_policy_apply_set_from_pyobject is the only // place where we check if py_policy is NULL before setting apply policy. - // I didn't want to set the pointer to the apply policy in this helper function as well. + // + // Also, I didn't want to set the pointer to the apply policy in this helper function as well to make + // the latter more simple if (py_policy) { as_policy_apply_copy_and_set_from_pyobject( self, &err, py_policy, &apply_policy, From 6ff8b3c65d73ca926ab004722de2b2548e1a65e6 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 15 Jul 2025 13:25:03 -0700 Subject: [PATCH 41/94] rm confusing comment --- src/main/client/get.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/client/get.c b/src/main/client/get.c index 1a37bee284..595cab1d2a 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -80,7 +80,6 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, key_initialised = true; if (py_policy) { - // Convert python policy object to as_policy_exists as_policy_read_copy_and_set_from_pyobject( self, &err, py_policy, &read_policy, &self->as->config.policies.read); From c3fb9752c095127960572a8b0b2fc23c9da6ddee Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 15 Jul 2025 13:35:28 -0700 Subject: [PATCH 42/94] more todo --- src/include/policy.h | 53 +++++++++++++++++++++++--------------------- src/main/policy.c | 25 ++++++++++----------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 3eba4cde05..6d1ca034b1 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -271,6 +271,34 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_exp *exp_list, as_exp **exp_list_p); +// These methods are used for setting config-level policies and are re-used for the helper methods +// that set transaction-level policies. +// +// Overrides an as_policy_* *policy* fields using entries from a Python object *py_policy*. +// +// *policy* must point to a valid as_policy_* instance. +// +// If py_policy is NULL, do nothing. Else if it is not a Python dictionary, an error will be raised. +// (We check if it's a dictionary here because *py_policy*'s type is not validated when parsing API arguments) +// +// Client instance *self* can be NULL if is_policy_txn_level is false. We only need *self* for parsing Python client +// expressions. +// +// *is_this_txn_policy* is required because there are fields that should only be set in *policy* for txn-level +// policies. +// +// Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. + +as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, + PyObject *py_policy, + as_policy_read *policy, + bool is_policy_txn_level); + +as_status as_policy_write_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_write *policy, + bool is_policy_txn_level); + // These methods are used for setting transaction level policies // 1. Copies an config-level as_policy_* *src* to a txn-level *dst* policy // 2. Sets an as_policy_* *dst* from a Python object *py_policy*. to override the defaults from *src* @@ -300,31 +328,6 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, as_policy_apply *policy, bool is_policy_txn_level); -// Overrides an as_policy_* *policy* fields using entries from a Python object *py_policy*. -// -// *policy* must point to a valid as_policy_* instance. -// -// If py_policy is NULL, do nothing. Else if it is not a Python dictionary, an error will be raised. -// (We check if it's a dictionary here because *py_policy*'s type is not validated when parsing API arguments) -// -// Client instance *self* can be NULL if is_policy_txn_level is false. We only need *self* for parsing Python client -// expressions. -// -// *is_this_txn_policy* is required because there are fields that should only be set in *policy* for txn-level -// policies. -// -// Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. - -as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, - PyObject *py_policy, - as_policy_read *policy, - bool is_policy_txn_level); - -as_status as_policy_write_set_from_pyobject(AerospikeClient *self, - as_error *err, PyObject *py_policy, - as_policy_write *policy, - bool is_policy_txn_level); - // TODO: make consistent // metrics_policy must be declared already // py_metrics_policy must be non-NULL diff --git a/src/main/policy.c b/src/main/policy.c index e5ceacaa7a..b47f323f6e 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -46,11 +46,11 @@ // Either way, we want as_policy_*_set_from_pyobject to be a no-op. #define VALIDATE_POLICY() \ if (py_policy == NULL || py_policy == Py_None) { \ - return AEROSPIKE_OK; \ + return NULL; \ } \ else if (!PyDict_Check(py_policy)) { \ - return as_error_update(err, AEROSPIKE_ERR_PARAM, \ - "policy must be a dict"); \ + as_error_update(err, AEROSPIKE_ERR_PARAM, "policy must be a dict"); \ + return NULL; \ } // TODO: Python exceptions should be propagated up instead of being cleared @@ -542,20 +542,19 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, return err->code; } -as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_write *dst, - as_policy_write *src) +as_policy_write *as_policy_write_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_write *dst, as_policy_write *src) { as_policy_write_copy(src, dst); return as_policy_write_set_from_pyobject(self, err, py_policy, dst, true); } -as_status as_policy_write_set_from_pyobject(AerospikeClient *self, - as_error *err, PyObject *py_policy, - as_policy_write *policy, - bool is_policy_txn_level) +as_policy_write *as_policy_write_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_write *policy, + bool is_policy_txn_level) { VALIDATE_POLICY(); @@ -579,7 +578,7 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, POLICY_SET_FIELD(ttl, uint32_t); } - return err->code; + return policy; } /** From 1c1187bfc07de21fd2825379cb7eb358811f519f Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:57:39 -0700 Subject: [PATCH 43/94] wip --- src/include/policy.h | 4 ++-- src/main/policy.c | 36 +++++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 6d1ca034b1..e2f9eafae3 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -294,7 +294,7 @@ as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, as_policy_read *policy, bool is_policy_txn_level); -as_status as_policy_write_set_from_pyobject(AerospikeClient *self, +as_policy_write *as_policy_write_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_write *policy, bool is_policy_txn_level); @@ -305,7 +305,7 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, // *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_write instances. // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. -as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, +as_policy_write * as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_write *dst, diff --git a/src/main/policy.c b/src/main/policy.c index b47f323f6e..4d66934585 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -46,11 +46,11 @@ // Either way, we want as_policy_*_set_from_pyobject to be a no-op. #define VALIDATE_POLICY() \ if (py_policy == NULL || py_policy == Py_None) { \ - return NULL; \ + return AEROSPIKE_OK; \ } \ else if (!PyDict_Check(py_policy)) { \ - as_error_update(err, AEROSPIKE_ERR_PARAM, "policy must be a dict"); \ - return NULL; \ + return as_error_update(err, AEROSPIKE_ERR_PARAM, \ + "policy must be a dict"); \ } // TODO: Python exceptions should be propagated up instead of being cleared @@ -542,26 +542,36 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, return err->code; } -as_policy_write *as_policy_write_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_write *dst, as_policy_write *src) +as_policy_write *as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_write *dst, + as_policy_write *src) { as_policy_write_copy(src, dst); return as_policy_write_set_from_pyobject(self, err, py_policy, dst, true); } -as_policy_write *as_policy_write_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_write *policy, - bool is_policy_txn_level) +as_policy_write* as_policy_write_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_write *policy, + bool is_policy_txn_level) { - VALIDATE_POLICY(); + if (py_policy == NULL || py_policy == Py_None) { + return NULL; + } + else if (!PyDict_Check(py_policy)) { + as_error_update(err, AEROSPIKE_ERR_PARAM, + "policy must be a dict"); + return NULL; + } + + // VALIDATE_POLICY(); as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_policy_txn_level); if (retval != AEROSPIKE_OK) { - return retval; + return NULL; } POLICY_SET_FIELD(key, as_policy_key); From 441e3ccfb926befbc7ac82de4b6f2302ea31bb65 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 11:19:05 -0700 Subject: [PATCH 44/94] fix --- src/main/policy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 94245e78fa..71af93301e 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -117,8 +117,8 @@ Py_DECREF(py_field_name); \ if (py_exp_list) { \ as_exp *exp_list = NULL; \ - if (as_exp_new_from_pyobject(self, py_exp_list, &exp_list, err) == \ - AEROSPIKE_OK) { \ + if (as_exp_new_from_pyobject(self, py_exp_list, &exp_list, err, \ + false) == AEROSPIKE_OK) { \ policy->filter_exp = exp_list; \ } \ else { \ From 95e5e3fc6eb5b8f17bd5e02b107ae56021545654 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 11:31:01 -0700 Subject: [PATCH 45/94] set helper method shouldnt need to return the object it is setting attributes for --- src/include/policy.h | 4 ++-- src/main/policy.c | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index e2f9eafae3..6d1ca034b1 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -294,7 +294,7 @@ as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, as_policy_read *policy, bool is_policy_txn_level); -as_policy_write *as_policy_write_set_from_pyobject(AerospikeClient *self, +as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_write *policy, bool is_policy_txn_level); @@ -305,7 +305,7 @@ as_policy_write *as_policy_write_set_from_pyobject(AerospikeClient *self, // *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_write instances. // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. -as_policy_write * as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, +as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_write *dst, diff --git a/src/main/policy.c b/src/main/policy.c index 71af93301e..45e76facfa 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -545,19 +545,20 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, return err->code; } -as_policy_write *as_policy_write_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_write *dst, as_policy_write *src) +as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_write *dst, + as_policy_write *src) { as_policy_write_copy(src, dst); return as_policy_write_set_from_pyobject(self, err, py_policy, dst, true); } -as_policy_write *as_policy_write_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_write *policy, - bool is_policy_txn_level) +as_status as_policy_write_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_write *policy, + bool is_policy_txn_level) { if (py_policy == NULL || py_policy == Py_None) { return NULL; From 83e5efad15ea8bffa967b1cb9b925b56ffe4e93b Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 11:35:00 -0700 Subject: [PATCH 46/94] fix --- src/main/policy.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 45e76facfa..684d333019 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -561,11 +561,11 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, bool is_policy_txn_level) { if (py_policy == NULL || py_policy == Py_None) { - return NULL; + goto exit; } else if (!PyDict_Check(py_policy)) { as_error_update(err, AEROSPIKE_ERR_PARAM, "policy must be a dict"); - return NULL; + goto exit; } // VALIDATE_POLICY(); @@ -573,7 +573,7 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_policy_txn_level); if (retval != AEROSPIKE_OK) { - return NULL; + goto exit; } POLICY_SET_FIELD(key, as_policy_key); @@ -590,7 +590,8 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, POLICY_SET_FIELD(ttl, uint32_t); } - return policy; +exit: + return err->code; } /** From 4647b94653b94d9042cd3ec9446fde7be5724193 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 12:52:51 -0700 Subject: [PATCH 47/94] Try using inline function to check if py_policy is correct type instead of macro --- src/main/policy.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 684d333019..309518876e 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -41,9 +41,7 @@ #define POLICY_UPDATE() *policy_p = policy; -// For transaction level policies, before calling this, we convert policy = None to NULL and this never gets called. -// But for config-level policies, we do not check whether py_policy is NULL or None. -// Either way, we want as_policy_*_set_from_pyobject to be a no-op. +// TODO: replace and remove #define VALIDATE_POLICY() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ @@ -53,6 +51,22 @@ "policy must be a dict"); \ } +// For transaction level policies, before calling this, we convert policy = None to NULL and this never gets called. +// But for config-level policies, we do not check whether py_policy is NULL or None. +// Either way, we want as_policy_*_set_from_pyobject to be a no-op. +static inline bool is_pyobject_valid_policy_type(as_error *err, + PyObject *py_object) +{ + if (py_object == NULL || py_object == Py_None) { + return true; + } + else if (!PyDict_Check(py_object)) { + as_error_update(err, AEROSPIKE_ERR_PARAM, "policy must be a dict"); + return false; + } + return true; +} + // TODO: Python exceptions should be propagated up instead of being cleared // but the policy helper functions don't handle this case and they only populate // an as_error object and return a status code. @@ -560,15 +574,9 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_policy_write *policy, bool is_policy_txn_level) { - if (py_policy == NULL || py_policy == Py_None) { + if (is_pyobject_valid_policy_type(err, py_policy) == false) { goto exit; } - else if (!PyDict_Check(py_policy)) { - as_error_update(err, AEROSPIKE_ERR_PARAM, "policy must be a dict"); - goto exit; - } - - // VALIDATE_POLICY(); as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_policy_txn_level); From 529e5fc067ae96d96d5a11f6eb06cd90ff71bea2 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 17:35:28 -0700 Subject: [PATCH 48/94] Improve naming to make less confusing --- src/main/policy.c | 52 +++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 309518876e..9bb166d7e5 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -42,7 +42,7 @@ #define POLICY_UPDATE() *policy_p = policy; // TODO: replace and remove -#define VALIDATE_POLICY() \ +#define RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ } \ @@ -54,17 +54,13 @@ // For transaction level policies, before calling this, we convert policy = None to NULL and this never gets called. // But for config-level policies, we do not check whether py_policy is NULL or None. // Either way, we want as_policy_*_set_from_pyobject to be a no-op. -static inline bool is_pyobject_valid_policy_type(as_error *err, - PyObject *py_object) +static inline as_status is_pyobject_valid_policy_type(as_error *err, + PyObject *py_object) { - if (py_object == NULL || py_object == Py_None) { - return true; - } - else if (!PyDict_Check(py_object)) { - as_error_update(err, AEROSPIKE_ERR_PARAM, "policy must be a dict"); - return false; + if (py_object == NULL || py_object == Py_None || PyDict_Check(py_object)) { + return AEROSPIKE_OK; } - return true; + return as_error_update(err, AEROSPIKE_ERR_PARAM, "policy must be a dict"); } // TODO: Python exceptions should be propagated up instead of being cleared @@ -257,7 +253,7 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, as_policy_admin *config_admin_policy) { - VALIDATE_POLICY(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); //Initialize policy with global defaults as_policy_admin_copy(config_admin_policy, policy); @@ -314,7 +310,7 @@ as_policy_base_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_base *policy, bool is_this_txn_policy) { - VALIDATE_POLICY() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() POLICY_SET_FIELD(total_timeout, uint32_t); POLICY_SET_FIELD(socket_timeout, uint32_t); @@ -348,7 +344,7 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, as_policy_apply *policy, bool is_this_txn_policy) { - VALIDATE_POLICY(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); // Set policy fields as_status retval = as_policy_base_set_from_pyobject( @@ -379,7 +375,7 @@ as_status pyobject_to_policy_info(as_error *err, PyObject *py_policy, as_policy_info **policy_p, as_policy_info *config_info_policy) { - VALIDATE_POLICY(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); //Initialize policy with global defaults as_policy_info_copy(config_info_policy, policy); @@ -410,7 +406,7 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query *config_query_policy, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); //Initialize policy with global defaults as_policy_query_copy(config_query_policy, policy); @@ -461,7 +457,7 @@ as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, as_policy_read *policy, bool is_policy_txn_level) { - VALIDATE_POLICY(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); // Set policy fields as_status retval = as_policy_base_set_from_pyobject( @@ -495,7 +491,7 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, as_policy_remove *config_remove_policy, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); //Initialize policy with global defaults as_policy_remove_copy(config_remove_policy, policy); @@ -534,7 +530,7 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, as_policy_scan *config_scan_policy, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); //Initialize policy with global defaults as_policy_scan_copy(config_scan_policy, policy); @@ -574,9 +570,7 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_policy_write *policy, bool is_policy_txn_level) { - if (is_pyobject_valid_policy_type(err, py_policy) == false) { - goto exit; - } + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_policy_txn_level); @@ -615,7 +609,7 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, as_policy_operate *config_operate_policy, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); //Initialize policy with global defaults as_policy_operate_copy(config_operate_policy, policy); @@ -661,7 +655,7 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, as_policy_batch *config_batch_policy, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() //Initialize policy with global defaults as_policy_batch_copy(config_batch_policy, policy); @@ -702,7 +696,7 @@ as_status pyobject_to_batch_write_policy(AerospikeClient *self, as_error *err, as_policy_batch_write **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() as_policy_batch_write_init(policy); // Set policy fields @@ -729,7 +723,7 @@ as_status pyobject_to_batch_read_policy(AerospikeClient *self, as_error *err, as_policy_batch_read **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() as_policy_batch_read_init(policy); // Set policy fields @@ -753,7 +747,7 @@ as_status pyobject_to_batch_apply_policy(AerospikeClient *self, as_error *err, as_policy_batch_apply **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() as_policy_batch_apply_init(policy); // Set policy fields @@ -779,7 +773,7 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_policy_batch_remove **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - VALIDATE_POLICY() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() as_policy_batch_remove_init(policy); // Set policy fields @@ -802,7 +796,7 @@ as_status pyobject_to_bit_policy(as_error *err, PyObject *py_policy, as_bit_policy *policy) { as_bit_policy_init(policy); - VALIDATE_POLICY() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() PyObject *py_bit_flags = PyDict_GetItemString(py_policy, BIT_WRITE_FLAGS_KEY); @@ -826,7 +820,7 @@ as_status pyobject_to_map_policy(as_error *err, PyObject *py_policy, as_map_policy *policy) { // Initialize Policy - VALIDATE_POLICY() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() as_map_policy_init(policy); // Defaults From ce4f87364c64164d50911ee9444db644eef21bcb Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 17:37:07 -0700 Subject: [PATCH 49/94] explain.. --- src/main/policy.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/policy.c b/src/main/policy.c index 9bb166d7e5..69fd044353 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -42,6 +42,9 @@ #define POLICY_UPDATE() *policy_p = policy; // TODO: replace and remove +// Don't make an inline function because +// we want to return early if the policy is NULL or Py_None. +// If we make this an inline function, we have to check again outside the inline function #define RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ From e4987915b6123fa166f4e935cef45bc240e4f05e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 17:42:18 -0700 Subject: [PATCH 50/94] Rm duplicate policy type check since policy set functions already check policy type --- src/main/policy.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 69fd044353..7797be5544 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -308,13 +308,12 @@ static inline void check_and_set_txn_field(as_error *err, policy_base->txn = py_txn->txn; } +// py_policy must be a Python dictionary static inline as_status as_policy_base_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_base *policy, bool is_this_txn_policy) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() - POLICY_SET_FIELD(total_timeout, uint32_t); POLICY_SET_FIELD(socket_timeout, uint32_t); POLICY_SET_FIELD(timeout_delay, uint32_t); From c9f48b454be2062f981542e41bb6f600780fea82 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 17:52:06 -0700 Subject: [PATCH 51/94] move to proper section --- src/include/policy.h | 10 +++++----- src/main/policy.c | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 6d1ca034b1..b63118d612 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -299,6 +299,11 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_policy_write *policy, bool is_policy_txn_level); +as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, + as_error *err, PyObject *py_policy, + as_policy_apply *policy, + bool is_policy_txn_level); + // These methods are used for setting transaction level policies // 1. Copies an config-level as_policy_* *src* to a txn-level *dst* policy // 2. Sets an as_policy_* *dst* from a Python object *py_policy*. to override the defaults from *src* @@ -323,11 +328,6 @@ as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, as_policy_apply *dst, as_policy_apply *src); -as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, - as_error *err, PyObject *py_policy, - as_policy_apply *policy, - bool is_policy_txn_level); - // TODO: make consistent // metrics_policy must be declared already // py_metrics_policy must be non-NULL diff --git a/src/main/policy.c b/src/main/policy.c index 7797be5544..89fa1a99fa 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -45,6 +45,7 @@ // Don't make an inline function because // we want to return early if the policy is NULL or Py_None. // If we make this an inline function, we have to check again outside the inline function +// TODO: wrap around do while? #define RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ From ef25215d93a9ad1a97f6af151c756021ee565e73 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 18:12:55 -0700 Subject: [PATCH 52/94] experiment with converter function to convert python type to as_policy_* --- src/main/client/exists.c | 25 ++++++++++++++++++++----- src/main/policy.c | 1 + 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/main/client/exists.c b/src/main/client/exists.c index 6ba9e316c0..fbce66c9ba 100644 --- a/src/main/client/exists.c +++ b/src/main/client/exists.c @@ -140,6 +140,25 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, return py_result; } +// TODO: retval type +static int py_policy_converter(PyObject *py_policy, void *converted_ref) +{ + if (py_policy == Py_None) { + py_policy = NULL; + } + + if (!py_policy || PyDict_Check(py_policy)) { + return 1; + } + + // Invalid py_policy type + as_error err; + // TODO: get the exact err description + as_error_update(&err, AEROSPIKE_ERR_PARAM, "Invalid policy type"); + raise_exception(&err); + return 0; +} + /** ******************************************************************************************************* * Checks if a record exists in the Aerospike DB. @@ -164,15 +183,11 @@ PyObject *AerospikeClient_Exists(AerospikeClient *self, PyObject *args, static char *kwlist[] = {"key", "policy", NULL}; // Python Function Argument Parsing - if (PyArg_ParseTupleAndKeywords(args, kwds, "O|O:exists", kwlist, &py_key, + if (PyArg_ParseTupleAndKeywords(args, kwds, "O|O&:exists", kwlist, &py_key, &py_policy) == false) { return NULL; } - if (py_policy == Py_None) { - py_policy = NULL; - } - // Invoke Operation return AerospikeClient_Exists_Invoke(self, py_key, py_policy); } diff --git a/src/main/policy.c b/src/main/policy.c index 89fa1a99fa..ee85b77ed5 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -451,6 +451,7 @@ as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, as_policy_read *dst, as_policy_read *src) { + // TODO: unnecessary copy if py_policy is NULL/Py_None as_policy_read_copy(src, dst); return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true); } From c4abe37a8fd3bd92a152d7e1b72c481622f4c19c Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 18:36:48 -0700 Subject: [PATCH 53/94] See if these two test cases pass now. --- test/new_tests/test_append.py | 2 +- test/new_tests/test_touch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/new_tests/test_append.py b/test/new_tests/test_append.py index bf0a4cbbef..ea36378723 100644 --- a/test/new_tests/test_append.py +++ b/test/new_tests/test_append.py @@ -84,7 +84,7 @@ def test_pos_append_with_policy_key_send(self): assert key == ( "test", "demo", - None, + 1, bytearray(b"\xb7\xf4\xb88\x89\xe2\xdag\xdeh>\x1d\xf6\x91\x9a\x1e\xac\xc4F\xc8"), ) diff --git a/test/new_tests/test_touch.py b/test/new_tests/test_touch.py index b4f4a54140..2a5ea2a612 100644 --- a/test/new_tests/test_touch.py +++ b/test/new_tests/test_touch.py @@ -74,7 +74,7 @@ def test_touch_with_policy_key_send(self): (key, _, bins) = self.as_connection.get(key) assert bins == {"age": 1, "name": "name1"} - assert key == ("test", "demo", None, self.test_demo_1_digest) + assert key == ("test", "demo", 1, self.test_demo_1_digest) def test_touch_with_policy_key_digest(self): """ From 821d32a771b445a3160c55bb45a3bda529ac34a5 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 2 Oct 2025 18:36:53 -0700 Subject: [PATCH 54/94] Revert "experiment with converter function to convert python type to as_policy_*" This reverts commit ef25215d93a9ad1a97f6af151c756021ee565e73. --- src/main/client/exists.c | 25 +++++-------------------- src/main/policy.c | 1 - 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/main/client/exists.c b/src/main/client/exists.c index fbce66c9ba..6ba9e316c0 100644 --- a/src/main/client/exists.c +++ b/src/main/client/exists.c @@ -140,25 +140,6 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, return py_result; } -// TODO: retval type -static int py_policy_converter(PyObject *py_policy, void *converted_ref) -{ - if (py_policy == Py_None) { - py_policy = NULL; - } - - if (!py_policy || PyDict_Check(py_policy)) { - return 1; - } - - // Invalid py_policy type - as_error err; - // TODO: get the exact err description - as_error_update(&err, AEROSPIKE_ERR_PARAM, "Invalid policy type"); - raise_exception(&err); - return 0; -} - /** ******************************************************************************************************* * Checks if a record exists in the Aerospike DB. @@ -183,11 +164,15 @@ PyObject *AerospikeClient_Exists(AerospikeClient *self, PyObject *args, static char *kwlist[] = {"key", "policy", NULL}; // Python Function Argument Parsing - if (PyArg_ParseTupleAndKeywords(args, kwds, "O|O&:exists", kwlist, &py_key, + if (PyArg_ParseTupleAndKeywords(args, kwds, "O|O:exists", kwlist, &py_key, &py_policy) == false) { return NULL; } + if (py_policy == Py_None) { + py_policy = NULL; + } + // Invoke Operation return AerospikeClient_Exists_Invoke(self, py_key, py_policy); } diff --git a/src/main/policy.c b/src/main/policy.c index ee85b77ed5..89fa1a99fa 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -451,7 +451,6 @@ as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, as_policy_read *dst, as_policy_read *src) { - // TODO: unnecessary copy if py_policy is NULL/Py_None as_policy_read_copy(src, dst); return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true); } From 196a4c72642643f19e0293ddea3312cf08cf61b5 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:13:20 -0700 Subject: [PATCH 55/94] Fix merge --- src/include/policy.h | 37 ++++++++---------- src/main/client/put.c | 2 +- src/main/client/query.c | 2 +- src/main/client/remove_bin.c | 2 +- src/main/policy.c | 22 +++++------ src/main/policy_config.c | 60 +++++++++++++++++------------ src/main/query/execute_background.c | 3 +- 7 files changed, 67 insertions(+), 61 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 9d37748da0..1f2b6e9847 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -327,17 +327,20 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_read *policy, - bool is_policy_txn_level); + bool is_policy_txn_level, + bool validate_keys); as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_write *policy, - bool is_policy_txn_level); + bool is_policy_txn_level, + bool validate_keys); as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_apply *policy, - bool is_policy_txn_level); + bool is_policy_txn_level, + bool validate_keys); // These methods are used for setting transaction level policies // 1. Copies an config-level as_policy_* *src* to a txn-level *dst* policy @@ -345,23 +348,17 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, // *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_write instances. // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. -as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_write *dst, - as_policy_write *src); - -as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_read *dst, - as_policy_read *src); - -as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_apply *dst, - as_policy_apply *src); +as_status as_policy_write_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_write *dst, as_policy_write *src, bool validate_keys); + +as_status as_policy_read_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_read *dst, as_policy_read *src, bool validate_keys); + +as_status as_policy_apply_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_apply *dst, as_policy_apply *src, bool validate_keys); // TODO: make consistent // metrics_policy must be declared already diff --git a/src/main/client/put.c b/src/main/client/put.c index db3da975ec..d946b87674 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -99,7 +99,7 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, if (py_policy) { as_policy_write_copy_and_set_from_pyobject( self, &err, py_policy, &write_policy, - &self->as->config.policies.write); + &self->as->config.policies.write, self->validate_keys); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } diff --git a/src/main/client/query.c b/src/main/client/query.c index afaa1817ab..6e13b1608f 100644 --- a/src/main/client/query.c +++ b/src/main/client/query.c @@ -325,7 +325,7 @@ static PyObject *AerospikeClient_QueryApply_Invoke( if (py_policy) { as_policy_write_copy_and_set_from_pyobject( self, &err, py_policy, &write_policy, - &self->as->config.policies.write); + &self->as->config.policies.write, self->validate_keys); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } diff --git a/src/main/client/remove_bin.c b/src/main/client/remove_bin.c index 5fd2e0067d..ee3e993ba7 100644 --- a/src/main/client/remove_bin.c +++ b/src/main/client/remove_bin.c @@ -73,7 +73,7 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, if (py_policy != NULL) { as_policy_write_copy_and_set_from_pyobject( self, err, py_policy, &write_policy, - &self->as->config.policies.write); + &self->as->config.policies.write, self->validate_keys); if (err->code != AEROSPIKE_OK) { as_error_update(err, AEROSPIKE_ERR_CLIENT, "Incorrect policy"); diff --git a/src/main/policy.c b/src/main/policy.c index ab67cf87d6..31e3461fa9 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -475,30 +475,28 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, return err->code; } -as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_apply *dst, - as_policy_apply *src) +as_status as_policy_apply_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_apply *dst, as_policy_apply *src, bool validate_keys) { as_policy_apply_copy(src, dst); return as_policy_apply_set_from_pyobject(self, err, py_policy, dst, true); } -as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, - as_error *err, - PyObject *py_policy, - as_policy_read *dst, - as_policy_read *src) +as_status as_policy_read_copy_and_set_from_pyobject( + AerospikeClient *self, as_error *err, PyObject *py_policy, + as_policy_read *dst, as_policy_read *src, bool validate_keys) { as_policy_read_copy(src, dst); - return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true); + return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true, + validate_keys); } as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_read *policy, - bool is_policy_txn_level) + bool is_policy_txn_level, + bool validate_keys) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 3c86e2494c..9a3772f654 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -42,7 +42,8 @@ as_status set_optional_int_property(int *property_ptr, PyObject *py_policy, * py_policies must exist, and be a dictionary * TODO: pass in err */ -as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) +as_status set_subpolicies(as_error *err, as_config *config, + PyObject *py_policies_dict, int validate_keys) { as_status set_policy_status = AEROSPIKE_OK; @@ -51,7 +52,7 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) PyObject *read_policy = PyDict_GetItemString(py_policies_dict, "read"); set_policy_status = as_policy_read_set_from_pyobject( - NULL, &err, read_policy, &config->policies.read, false); + NULL, &err, read_policy, &config->policies.read, false, validate_keys); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -59,7 +60,8 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) PyObject *write_policy = PyDict_GetItemString(py_policies_dict, "write"); set_policy_status = as_policy_write_set_from_pyobject( - NULL, &err, write_policy, &config->policies.write, false); + NULL, &err, write_policy, &config->policies.write, false, + validate_keys); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -67,85 +69,93 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) PyObject *py_apply_policy = PyDict_GetItemString(py_policies_dict, "apply"); set_policy_status = as_policy_apply_set_from_pyobject( - NULL, &err, py_apply_policy, &config->policies.apply, false); + NULL, &err, py_apply_policy, &config->policies.apply, false, + validate_keys); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *remove_policy = PyDict_GetItemString(py_policies_dict, "remove"); - set_policy_status = - set_remove_policy(&config->policies.remove, remove_policy); + set_policy_status = set_remove_policy(&err, &config->policies.remove, + remove_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *query_policy = PyDict_GetItemString(py_policies_dict, "query"); - set_policy_status = set_query_policy(&config->policies.query, query_policy); + set_policy_status = set_query_policy(&err, &config->policies.query, + query_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *scan_policy = PyDict_GetItemString(py_policies_dict, "scan"); - set_policy_status = set_scan_policy(&config->policies.scan, scan_policy); + set_policy_status = set_scan_policy(&err, &config->policies.scan, + scan_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *operate_policy = PyDict_GetItemString(py_policies_dict, "operate"); - set_policy_status = - set_operate_policy(&config->policies.operate, operate_policy); + set_policy_status = set_operate_policy(&err, &config->policies.operate, + operate_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_policy = PyDict_GetItemString(py_policies_dict, "batch"); - set_policy_status = set_batch_policy(&config->policies.batch, batch_policy); + set_policy_status = set_batch_policy(&err, &config->policies.batch, + batch_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *info_policy = PyDict_GetItemString(py_policies_dict, "info"); - set_policy_status = set_info_policy(&config->policies.info, info_policy); + set_policy_status = set_info_policy(&err, &config->policies.info, + info_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *admin_policy = PyDict_GetItemString(py_policies_dict, "admin"); - set_policy_status = set_admin_policy(&config->policies.admin, admin_policy); + set_policy_status = set_admin_policy(&err, &config->policies.admin, + admin_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_apply_policy = PyDict_GetItemString(py_policies_dict, "batch_apply"); - set_policy_status = set_batch_apply_policy(&config->policies.batch_apply, - batch_apply_policy); + set_policy_status = set_batch_apply_policy( + &err, &config->policies.batch_apply, batch_apply_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_remove_policy = PyDict_GetItemString(py_policies_dict, "batch_remove"); - set_policy_status = set_batch_remove_policy(&config->policies.batch_remove, - batch_remove_policy); + set_policy_status = + set_batch_remove_policy(&err, &config->policies.batch_remove, + batch_remove_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_write_policy = PyDict_GetItemString(py_policies_dict, "batch_write"); - set_policy_status = set_batch_write_policy(&config->policies.batch_write, - batch_write_policy); + set_policy_status = set_batch_write_policy( + &err, &config->policies.batch_write, batch_write_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_parent_write_policy = PyDict_GetItemString(py_policies_dict, "batch_parent_write"); - set_policy_status = set_batch_policy(&config->policies.batch_parent_write, - batch_parent_write_policy); + set_policy_status = + set_batch_policy(&err, &config->policies.batch_parent_write, + batch_parent_write_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -158,8 +168,8 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) i < sizeof(batch_policy_names) / sizeof(batch_policy_names[0]); i++) { PyObject *py_batch_policy = PyDict_GetItemString(py_policies_dict, batch_policy_names[i]); - set_policy_status = - set_batch_policy(batch_policies[i], py_batch_policy); + set_policy_status = set_batch_policy(&err, batch_policies[i], + py_batch_policy, validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -170,8 +180,8 @@ as_status set_subpolicies(as_config *config, PyObject *py_policies_dict) return AEROSPIKE_OK; } -as_status set_remove_policy(as_policy_remove *remove_policy, - PyObject *py_policy) +as_status set_remove_policy(as_error *err, as_policy_remove *remove_policy, + PyObject *py_policy, int validate_keys) { as_status status = AEROSPIKE_OK; diff --git a/src/main/query/execute_background.c b/src/main/query/execute_background.c index 5827871c99..096550c41d 100644 --- a/src/main/query/execute_background.c +++ b/src/main/query/execute_background.c @@ -65,7 +65,8 @@ PyObject *AerospikeQuery_ExecuteBackground(AerospikeQuery *self, PyObject *args, if (py_policy) { as_policy_write_copy_and_set_from_pyobject( self->client, &err, py_policy, &write_policy, - &self->client->as->config.policies.write); + &self->client->as->config.policies.write, + self->client->validate_keys); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } From f77e1807a5acbe38e55c3eb6b2c6d2bfbd0bb451 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:30:36 -0700 Subject: [PATCH 56/94] fix --- src/include/policy.h | 40 +++++++++++++++++++++------------------- src/main/policy.c | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index 1f2b6e9847..9883ccc563 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -316,8 +316,7 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, // If py_policy is NULL, do nothing. Else if it is not a Python dictionary, an error will be raised. // (We check if it's a dictionary here because *py_policy*'s type is not validated when parsing API arguments) // -// Client instance *self* can be NULL if is_policy_txn_level is false. We only need *self* for parsing Python client -// expressions. +// We only need the AerospikeClient *self* for parsing Python client expressions and to tell whether to validate keys. // // *is_this_txn_policy* is required because there are fields that should only be set in *policy* for txn-level // policies. @@ -327,20 +326,17 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_read *policy, - bool is_policy_txn_level, - bool validate_keys); + bool is_policy_txn_level); as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_write *policy, - bool is_policy_txn_level, - bool validate_keys); + bool is_policy_txn_level); as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_apply *policy, - bool is_policy_txn_level, - bool validate_keys); + bool is_policy_txn_level); // These methods are used for setting transaction level policies // 1. Copies an config-level as_policy_* *src* to a txn-level *dst* policy @@ -348,17 +344,23 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, // *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_write instances. // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. -as_status as_policy_write_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_write *dst, as_policy_write *src, bool validate_keys); - -as_status as_policy_read_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_read *dst, as_policy_read *src, bool validate_keys); - -as_status as_policy_apply_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_apply *dst, as_policy_apply *src, bool validate_keys); +as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_write *dst, + as_policy_write *src); + +as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_read *dst, + as_policy_read *src); + +as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_apply *dst, + as_policy_apply *src); // TODO: make consistent // metrics_policy must be declared already diff --git a/src/main/policy.c b/src/main/policy.c index 31e3461fa9..ce71cacc1c 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -362,6 +362,19 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + if (self->validate_keys) { + PyObject *py_policy_valid_keys = NULL; + as_status retval = does_py_dict_contain_valid_keys( + err, py_policy, py_apply_policy_valid_keys, true); + if (retval == -1) { + return as_error_update(err, AEROSPIKE_ERR, + ERR_MSG_FAILED_TO_VALIDATE_POLICY_KEYS); + } + else if (retval == 0) { + return err->code; + } + } + // Set policy fields as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_this_txn_policy); @@ -500,6 +513,19 @@ as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + if (self->validate_keys) { + PyObject *py_policy_valid_keys = NULL; + as_status retval = does_py_dict_contain_valid_keys( + err, py_policy, py_read_policy_valid_keys, true); + if (retval == -1) { + return as_error_update(err, AEROSPIKE_ERR, + ERR_MSG_FAILED_TO_VALIDATE_POLICY_KEYS); + } + else if (retval == 0) { + return err->code; + } + } + // Set policy fields as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_policy_txn_level); @@ -646,6 +672,19 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + if (self->validate_keys) { + PyObject *py_policy_valid_keys = NULL; + as_status retval = does_py_dict_contain_valid_keys( + err, py_policy, py_write_policy_valid_keys, true); + if (retval == -1) { + return as_error_update(err, AEROSPIKE_ERR, + ERR_MSG_FAILED_TO_VALIDATE_POLICY_KEYS); + } + else if (retval == 0) { + return err->code; + } + } + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, is_policy_txn_level); if (retval != AEROSPIKE_OK) { From 17e798aba6039a260ce36f3e973a8e8298704016 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:34:29 -0700 Subject: [PATCH 57/94] fix --- src/main/client/put.c | 2 +- src/main/client/query.c | 2 +- src/main/client/remove_bin.c | 2 +- src/main/query/execute_background.c | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/client/put.c b/src/main/client/put.c index d946b87674..db3da975ec 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -99,7 +99,7 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, if (py_policy) { as_policy_write_copy_and_set_from_pyobject( self, &err, py_policy, &write_policy, - &self->as->config.policies.write, self->validate_keys); + &self->as->config.policies.write); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } diff --git a/src/main/client/query.c b/src/main/client/query.c index 6e13b1608f..afaa1817ab 100644 --- a/src/main/client/query.c +++ b/src/main/client/query.c @@ -325,7 +325,7 @@ static PyObject *AerospikeClient_QueryApply_Invoke( if (py_policy) { as_policy_write_copy_and_set_from_pyobject( self, &err, py_policy, &write_policy, - &self->as->config.policies.write, self->validate_keys); + &self->as->config.policies.write); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } diff --git a/src/main/client/remove_bin.c b/src/main/client/remove_bin.c index ee3e993ba7..5fd2e0067d 100644 --- a/src/main/client/remove_bin.c +++ b/src/main/client/remove_bin.c @@ -73,7 +73,7 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, if (py_policy != NULL) { as_policy_write_copy_and_set_from_pyobject( self, err, py_policy, &write_policy, - &self->as->config.policies.write, self->validate_keys); + &self->as->config.policies.write); if (err->code != AEROSPIKE_OK) { as_error_update(err, AEROSPIKE_ERR_CLIENT, "Incorrect policy"); diff --git a/src/main/query/execute_background.c b/src/main/query/execute_background.c index 096550c41d..5827871c99 100644 --- a/src/main/query/execute_background.c +++ b/src/main/query/execute_background.c @@ -65,8 +65,7 @@ PyObject *AerospikeQuery_ExecuteBackground(AerospikeQuery *self, PyObject *args, if (py_policy) { as_policy_write_copy_and_set_from_pyobject( self->client, &err, py_policy, &write_policy, - &self->client->as->config.policies.write, - self->client->validate_keys); + &self->client->as->config.policies.write); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } From a2551e6ca3ed9bc6b59dec665388ed81b4732da3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:43:16 -0700 Subject: [PATCH 58/94] fix --- src/main/policy.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index ce71cacc1c..94164988ad 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -514,7 +514,6 @@ as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); if (self->validate_keys) { - PyObject *py_policy_valid_keys = NULL; as_status retval = does_py_dict_contain_valid_keys( err, py_policy, py_read_policy_valid_keys, true); if (retval == -1) { @@ -673,7 +672,6 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() if (self->validate_keys) { - PyObject *py_policy_valid_keys = NULL; as_status retval = does_py_dict_contain_valid_keys( err, py_policy, py_write_policy_valid_keys, true); if (retval == -1) { From 6a7d53cdc109610301e350bddbb3bf6297e0c0cb Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:45:25 -0700 Subject: [PATCH 59/94] fix --- src/main/policy.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 94164988ad..cc7f426a16 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -363,7 +363,6 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); if (self->validate_keys) { - PyObject *py_policy_valid_keys = NULL; as_status retval = does_py_dict_contain_valid_keys( err, py_policy, py_apply_policy_valid_keys, true); if (retval == -1) { @@ -488,28 +487,30 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, return err->code; } -as_status as_policy_apply_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_apply *dst, as_policy_apply *src, bool validate_keys) +as_status as_policy_apply_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_apply *dst, + as_policy_apply *src) { as_policy_apply_copy(src, dst); return as_policy_apply_set_from_pyobject(self, err, py_policy, dst, true); } -as_status as_policy_read_copy_and_set_from_pyobject( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_read *dst, as_policy_read *src, bool validate_keys) +as_status as_policy_read_copy_and_set_from_pyobject(AerospikeClient *self, + as_error *err, + PyObject *py_policy, + as_policy_read *dst, + as_policy_read *src) { as_policy_read_copy(src, dst); - return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true, - validate_keys); + return as_policy_read_set_from_pyobject(self, err, py_policy, dst, true); } as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_read *policy, - bool is_policy_txn_level, - bool validate_keys) + bool is_policy_txn_level) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); From 68790a9c77f0aa9620f2333390b41e331695e479 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:57:04 -0700 Subject: [PATCH 60/94] fix --- src/include/policy_config.h | 4 ++-- src/main/client/type.c | 4 ++-- src/main/policy_config.c | 45 ++++++++++++++++++------------------- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/include/policy_config.h b/src/include/policy_config.h index 1e8c59c927..79f5b4bf23 100644 --- a/src/include/policy_config.h +++ b/src/include/policy_config.h @@ -41,8 +41,8 @@ as_status set_optional_exists(as_policy_exists *target_ptr, PyObject *py_policy, // This only sets the err object if an invalid dictionary key is passed // On error, return an error code -as_status set_subpolicies(as_error *err, as_config *config, - PyObject *py_policies, int validate_keys); +as_status set_subpolicies(AerospikeClient *self, as_error *err, + as_config *config, PyObject *py_policies); as_status set_read_policy(as_error *err, as_policy_read *read_policy, PyObject *py_policy, int validate_keys); as_status set_write_policy(as_error *err, as_policy_write *write_policy, diff --git a/src/main/client/type.c b/src/main/client/type.c index 0b2864a4e7..6182e27a8a 100644 --- a/src/main/client/type.c +++ b/src/main/client/type.c @@ -1032,8 +1032,8 @@ static int AerospikeClient_Type_Init(AerospikeClient *self, PyObject *args, * Set the individual policy groups new in 3.0 * */ - if (set_subpolicies(&constructor_err, &config, py_policies, - validate_keys) != AEROSPIKE_OK) { + if (set_subpolicies(self, &constructor_err, &config, py_policies) != + AEROSPIKE_OK) { if (constructor_err.code != AEROSPIKE_OK) { // This would only be set if an invalid key was passed to a policy. // Don't override the error caused by validating the dictionary keys diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 9a3772f654..4486cc4578 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -40,10 +40,9 @@ as_status set_optional_int_property(int *property_ptr, PyObject *py_policy, /* * py_policies must exist, and be a dictionary - * TODO: pass in err */ -as_status set_subpolicies(as_error *err, as_config *config, - PyObject *py_policies_dict, int validate_keys) +as_status set_subpolicies(AerospikeClient *self, as_error *err, + as_config *config, PyObject *py_policies_dict) { as_status set_policy_status = AEROSPIKE_OK; @@ -52,7 +51,7 @@ as_status set_subpolicies(as_error *err, as_config *config, PyObject *read_policy = PyDict_GetItemString(py_policies_dict, "read"); set_policy_status = as_policy_read_set_from_pyobject( - NULL, &err, read_policy, &config->policies.read, false, validate_keys); + self, &err, read_policy, &config->policies.read, false); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -60,8 +59,7 @@ as_status set_subpolicies(as_error *err, as_config *config, PyObject *write_policy = PyDict_GetItemString(py_policies_dict, "write"); set_policy_status = as_policy_write_set_from_pyobject( - NULL, &err, write_policy, &config->policies.write, false, - validate_keys); + self, &err, write_policy, &config->policies.write, false); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -69,8 +67,7 @@ as_status set_subpolicies(as_error *err, as_config *config, PyObject *py_apply_policy = PyDict_GetItemString(py_policies_dict, "apply"); set_policy_status = as_policy_apply_set_from_pyobject( - NULL, &err, py_apply_policy, &config->policies.apply, false, - validate_keys); + self, &err, py_apply_policy, &config->policies.apply, false); as_error_reset(&err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -78,21 +75,21 @@ as_status set_subpolicies(as_error *err, as_config *config, PyObject *remove_policy = PyDict_GetItemString(py_policies_dict, "remove"); set_policy_status = set_remove_policy(&err, &config->policies.remove, - remove_policy, validate_keys); + remove_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *query_policy = PyDict_GetItemString(py_policies_dict, "query"); set_policy_status = set_query_policy(&err, &config->policies.query, - query_policy, validate_keys); + query_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *scan_policy = PyDict_GetItemString(py_policies_dict, "scan"); set_policy_status = set_scan_policy(&err, &config->policies.scan, - scan_policy, validate_keys); + scan_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -100,36 +97,37 @@ as_status set_subpolicies(as_error *err, as_config *config, PyObject *operate_policy = PyDict_GetItemString(py_policies_dict, "operate"); set_policy_status = set_operate_policy(&err, &config->policies.operate, - operate_policy, validate_keys); + operate_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_policy = PyDict_GetItemString(py_policies_dict, "batch"); set_policy_status = set_batch_policy(&err, &config->policies.batch, - batch_policy, validate_keys); + batch_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *info_policy = PyDict_GetItemString(py_policies_dict, "info"); set_policy_status = set_info_policy(&err, &config->policies.info, - info_policy, validate_keys); + info_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *admin_policy = PyDict_GetItemString(py_policies_dict, "admin"); set_policy_status = set_admin_policy(&err, &config->policies.admin, - admin_policy, validate_keys); + admin_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_apply_policy = PyDict_GetItemString(py_policies_dict, "batch_apply"); - set_policy_status = set_batch_apply_policy( - &err, &config->policies.batch_apply, batch_apply_policy, validate_keys); + set_policy_status = + set_batch_apply_policy(&err, &config->policies.batch_apply, + batch_apply_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -138,15 +136,16 @@ as_status set_subpolicies(as_error *err, as_config *config, PyDict_GetItemString(py_policies_dict, "batch_remove"); set_policy_status = set_batch_remove_policy(&err, &config->policies.batch_remove, - batch_remove_policy, validate_keys); + batch_remove_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_write_policy = PyDict_GetItemString(py_policies_dict, "batch_write"); - set_policy_status = set_batch_write_policy( - &err, &config->policies.batch_write, batch_write_policy, validate_keys); + set_policy_status = + set_batch_write_policy(&err, &config->policies.batch_write, + batch_write_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -155,7 +154,7 @@ as_status set_subpolicies(as_error *err, as_config *config, PyDict_GetItemString(py_policies_dict, "batch_parent_write"); set_policy_status = set_batch_policy(&err, &config->policies.batch_parent_write, - batch_parent_write_policy, validate_keys); + batch_parent_write_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -168,8 +167,8 @@ as_status set_subpolicies(as_error *err, as_config *config, i < sizeof(batch_policy_names) / sizeof(batch_policy_names[0]); i++) { PyObject *py_batch_policy = PyDict_GetItemString(py_policies_dict, batch_policy_names[i]); - set_policy_status = set_batch_policy(&err, batch_policies[i], - py_batch_policy, validate_keys); + set_policy_status = set_batch_policy( + &err, batch_policies[i], py_batch_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } From f20f7d1b8dfb036472cc4c47fa4fdc8f934e6a97 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:00:00 -0700 Subject: [PATCH 61/94] Fix --- src/include/policy_config.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/policy_config.h b/src/include/policy_config.h index 79f5b4bf23..2f3127bfa7 100644 --- a/src/include/policy_config.h +++ b/src/include/policy_config.h @@ -19,6 +19,7 @@ #include #include "macros.h" +#include "types.h" as_status set_optional_uint32_property(uint32_t *target_ptr, PyObject *policy_dict, const char *name); From 04630c66c4547ad00edcdf379ace00fc6c62852e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:00:44 -0700 Subject: [PATCH 62/94] fix --- src/main/policy_config.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 4486cc4578..9b4c6050ca 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -46,49 +46,46 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, { as_status set_policy_status = AEROSPIKE_OK; - as_error err; - as_error_init(&err); - PyObject *read_policy = PyDict_GetItemString(py_policies_dict, "read"); set_policy_status = as_policy_read_set_from_pyobject( self, &err, read_policy, &config->policies.read, false); - as_error_reset(&err); + as_error_reset(err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *write_policy = PyDict_GetItemString(py_policies_dict, "write"); set_policy_status = as_policy_write_set_from_pyobject( - self, &err, write_policy, &config->policies.write, false); - as_error_reset(&err); + self, err, write_policy, &config->policies.write, false); + as_error_reset(err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *py_apply_policy = PyDict_GetItemString(py_policies_dict, "apply"); set_policy_status = as_policy_apply_set_from_pyobject( - self, &err, py_apply_policy, &config->policies.apply, false); - as_error_reset(&err); + self, err, py_apply_policy, &config->policies.apply, false); + as_error_reset(err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *remove_policy = PyDict_GetItemString(py_policies_dict, "remove"); - set_policy_status = set_remove_policy(&err, &config->policies.remove, + set_policy_status = set_remove_policy(err, &config->policies.remove, remove_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *query_policy = PyDict_GetItemString(py_policies_dict, "query"); - set_policy_status = set_query_policy(&err, &config->policies.query, + set_policy_status = set_query_policy(err, &config->policies.query, query_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *scan_policy = PyDict_GetItemString(py_policies_dict, "scan"); - set_policy_status = set_scan_policy(&err, &config->policies.scan, + set_policy_status = set_scan_policy(err, &config->policies.scan, scan_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -96,28 +93,28 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, PyObject *operate_policy = PyDict_GetItemString(py_policies_dict, "operate"); - set_policy_status = set_operate_policy(&err, &config->policies.operate, + set_policy_status = set_operate_policy(err, &config->policies.operate, operate_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *batch_policy = PyDict_GetItemString(py_policies_dict, "batch"); - set_policy_status = set_batch_policy(&err, &config->policies.batch, + set_policy_status = set_batch_policy(err, &config->policies.batch, batch_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *info_policy = PyDict_GetItemString(py_policies_dict, "info"); - set_policy_status = set_info_policy(&err, &config->policies.info, + set_policy_status = set_info_policy(err, &config->policies.info, info_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } PyObject *admin_policy = PyDict_GetItemString(py_policies_dict, "admin"); - set_policy_status = set_admin_policy(&err, &config->policies.admin, + set_policy_status = set_admin_policy(err, &config->policies.admin, admin_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -126,7 +123,7 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, PyObject *batch_apply_policy = PyDict_GetItemString(py_policies_dict, "batch_apply"); set_policy_status = - set_batch_apply_policy(&err, &config->policies.batch_apply, + set_batch_apply_policy(err, &config->policies.batch_apply, batch_apply_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -135,7 +132,7 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, PyObject *batch_remove_policy = PyDict_GetItemString(py_policies_dict, "batch_remove"); set_policy_status = - set_batch_remove_policy(&err, &config->policies.batch_remove, + set_batch_remove_policy(err, &config->policies.batch_remove, batch_remove_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -144,7 +141,7 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, PyObject *batch_write_policy = PyDict_GetItemString(py_policies_dict, "batch_write"); set_policy_status = - set_batch_write_policy(&err, &config->policies.batch_write, + set_batch_write_policy(err, &config->policies.batch_write, batch_write_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -153,7 +150,7 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, PyObject *batch_parent_write_policy = PyDict_GetItemString(py_policies_dict, "batch_parent_write"); set_policy_status = - set_batch_policy(&err, &config->policies.batch_parent_write, + set_batch_policy(err, &config->policies.batch_parent_write, batch_parent_write_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; @@ -168,7 +165,7 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, PyObject *py_batch_policy = PyDict_GetItemString(py_policies_dict, batch_policy_names[i]); set_policy_status = set_batch_policy( - &err, batch_policies[i], py_batch_policy, self->validate_keys); + err, batch_policies[i], py_batch_policy, self->validate_keys); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } From 5377d7941b8fae645c9afac1448c541cd5d24c46 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:06:05 -0700 Subject: [PATCH 63/94] fix --- src/main/policy_config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 9b4c6050ca..880e251f26 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -48,7 +48,7 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, as_status set_policy_status = AEROSPIKE_OK; PyObject *read_policy = PyDict_GetItemString(py_policies_dict, "read"); set_policy_status = as_policy_read_set_from_pyobject( - self, &err, read_policy, &config->policies.read, false); + self, err, read_policy, &config->policies.read, false); as_error_reset(err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; From 0d5a730836b9d981adeaeb15b9fa5b37d8d6f8f2 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 4 Nov 2025 09:19:33 -0800 Subject: [PATCH 64/94] Remove duplicate include --- src/main/policy_config.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 8a58b42d9c..b99a7d5d10 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -17,7 +17,6 @@ #include "policy_config.h" #include "types.h" -#include "policy.h" #include "policy.h" From fc50d4cc1d30c629acc2dcead308fafa5cec6635 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 4 Nov 2025 09:25:28 -0800 Subject: [PATCH 65/94] Fix --- src/main/policy.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index af1f1a9b13..ead9866239 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -365,7 +365,8 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, if (self->validate_keys) { as_status retval = does_py_dict_contain_valid_keys( - err, py_policy, py_apply_policy_valid_keys, true); + err, py_policy, py_apply_policy_valid_keys, + POLICY_DICTIONARY_ADJECTIVE_FOR_ERROR_MESSAGE); if (retval == -1) { return as_error_update(err, AEROSPIKE_ERR, ERR_MSG_FAILED_TO_VALIDATE_POLICY_KEYS); @@ -518,7 +519,8 @@ as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, if (self->validate_keys) { as_status retval = does_py_dict_contain_valid_keys( - err, py_policy, py_read_policy_valid_keys, true); + err, py_policy, py_read_policy_valid_keys, + POLICY_DICTIONARY_ADJECTIVE_FOR_ERROR_MESSAGE); if (retval == -1) { return as_error_update(err, AEROSPIKE_ERR, ERR_MSG_FAILED_TO_VALIDATE_POLICY_KEYS); @@ -678,7 +680,8 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, if (self->validate_keys) { as_status retval = does_py_dict_contain_valid_keys( - err, py_policy, py_write_policy_valid_keys, true); + err, py_policy, py_write_policy_valid_keys, + POLICY_DICTIONARY_ADJECTIVE_FOR_ERROR_MESSAGE); if (retval == -1) { return as_error_update(err, AEROSPIKE_ERR, ERR_MSG_FAILED_TO_VALIDATE_POLICY_KEYS); From 5ec11bfbf1bc7f2ecc2cc8dd54b2c0678934c43c Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:12:16 -0800 Subject: [PATCH 66/94] Revert to old behavior to stop tests from regressing --- src/main/client/get.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/client/get.c b/src/main/client/get.c index 595cab1d2a..19fb153541 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -100,6 +100,17 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, AEROSPIKE_OK) { goto CLEANUP; } + if (!read_policy_p || + (read_policy_p && read_policy_p->key == AS_POLICY_KEY_DIGEST)) { + // This is a special case. + // C-client returns NULL key, so to the user + // response will be (, , None, ) + // Using the same input key, just making primary key part to be None + // Only in case of POLICY_KEY_DIGEST or no policy specified + PyObject *p_key = PyTuple_GetItem(py_rec, 0); + Py_INCREF(Py_None); + PyTuple_SetItem(p_key, 2, Py_None); + } } CLEANUP: From aee0fbb9425233c24d2cdf0c6c64e03969d2802c Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 4 Nov 2025 18:45:12 +0000 Subject: [PATCH 67/94] Forgot to revert --- test/new_tests/test_append.py | 2 +- test/new_tests/test_touch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/new_tests/test_append.py b/test/new_tests/test_append.py index ea36378723..bf0a4cbbef 100644 --- a/test/new_tests/test_append.py +++ b/test/new_tests/test_append.py @@ -84,7 +84,7 @@ def test_pos_append_with_policy_key_send(self): assert key == ( "test", "demo", - 1, + None, bytearray(b"\xb7\xf4\xb88\x89\xe2\xdag\xdeh>\x1d\xf6\x91\x9a\x1e\xac\xc4F\xc8"), ) diff --git a/test/new_tests/test_touch.py b/test/new_tests/test_touch.py index 2139b95aef..b000351c50 100644 --- a/test/new_tests/test_touch.py +++ b/test/new_tests/test_touch.py @@ -74,7 +74,7 @@ def test_touch_with_policy_key_send(self): (key, _, bins) = self.as_connection.get(key) assert bins == {"age": 1, "name": "name1"} - assert key == ("test", "demo", 1, self.test_demo_1_digest) + assert key == ("test", "demo", None, self.test_demo_1_digest) def test_touch_with_policy_key_digest(self): """ From d48bb110e907060aeb43e055de7a114253efd8de Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:51:00 -0800 Subject: [PATCH 68/94] Also another possible exc --- test/new_tests/test_validate_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/new_tests/test_validate_keys.py b/test/new_tests/test_validate_keys.py index 9c9959f04a..f7c2de4055 100644 --- a/test/new_tests/test_validate_keys.py +++ b/test/new_tests/test_validate_keys.py @@ -62,7 +62,7 @@ def setup(self, as_connection): [ (aerospike.Client.operate, {"key": KEY, "list": OPS_LIST, "policy": INVALID_POLICY}, nullcontext()), # User doesn't exist - (aerospike.Client.admin_query_user_info, {"user": "asdf", "policy": INVALID_POLICY}, pytest.raises((e.InvalidUser, e.SecurityNotSupported))), + (aerospike.Client.admin_query_user_info, {"user": "asdf", "policy": INVALID_POLICY}, pytest.raises((e.InvalidUser, e.SecurityNotSupported, e.SecurityNotEnabled))), (aerospike.Client.info_all, {"command": "status", "policy": INVALID_POLICY}, nullcontext()), # UDF doesn't exist on server (aerospike.Client.apply, {"key": KEY, "module": "module", "function": "function", "args": [], "policy": INVALID_POLICY}, pytest.raises(e.UDFError)), From 47dfb3ef2748caae4ac75611e71b4b48d7ed57c4 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 4 Nov 2025 12:11:49 -0800 Subject: [PATCH 69/94] Resetting error after validating keys doesn't make sense because we want to use the error msg from the validate helper code --- src/main/policy_config.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index b99a7d5d10..1d26e1c32c 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -44,12 +44,12 @@ as_status set_optional_int_property(int *property_ptr, PyObject *py_policy, as_status set_subpolicies(AerospikeClient *self, as_error *err, as_config *config, PyObject *py_policies_dict) { + as_error_reset(err); as_status set_policy_status = AEROSPIKE_OK; PyObject *read_policy = PyDict_GetItemString(py_policies_dict, "read"); set_policy_status = as_policy_read_set_from_pyobject( self, err, read_policy, &config->policies.read, false); - as_error_reset(err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -57,7 +57,6 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, PyObject *write_policy = PyDict_GetItemString(py_policies_dict, "write"); set_policy_status = as_policy_write_set_from_pyobject( self, err, write_policy, &config->policies.write, false); - as_error_reset(err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } @@ -65,7 +64,6 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, PyObject *py_apply_policy = PyDict_GetItemString(py_policies_dict, "apply"); set_policy_status = as_policy_apply_set_from_pyobject( self, err, py_apply_policy, &config->policies.apply, false); - as_error_reset(err); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } From 42bbd78e1783685ed26dd39962826e9b06653a08 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:08:18 -0800 Subject: [PATCH 70/94] Clean up redundant part of test. This part has nothing to do with the second half --- test/new_tests/test_get_put.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/test/new_tests/test_get_put.py b/test/new_tests/test_get_put.py index 628078a5bd..7c0f2e3f95 100644 --- a/test/new_tests/test_get_put.py +++ b/test/new_tests/test_get_put.py @@ -125,23 +125,6 @@ def test_pos_get_initkey_with_client_policy_send(self, put_data): """ Invoke get() for a record having string data. """ - - key = ("test", "demo", 1) - - rec = {"name": "john", "age": 1} - - put_data(self.as_connection, key, rec) - - key, _, bins = self.as_connection.get(key) - - assert bins == {"name": "john", "age": 1} - assert key == ( - "test", - "demo", - None, - bytearray(b"\xb7\xf4\xb88\x89\xe2\xdag\xdeh>\x1d\xf6\x91\x9a\x1e\xac\xc4F\xc8"), - ) - config = self.connection_config.copy() config["policies"] = {"key": aerospike.POLICY_KEY_SEND} From c6c8949bac05487465c74aa4d36d909a28dd6f08 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 6 Nov 2025 13:42:33 -0800 Subject: [PATCH 71/94] Final naming --- aerospike-client-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aerospike-client-c b/aerospike-client-c index 3f15b94be5..3bf664e7cf 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 3f15b94be5aef435e5ae6681953638b680b91a6e +Subproject commit 3bf664e7cfc56908a2479b11cf2cf4f985c178e3 From e7b6ccc18cd32c9b417a43e6ad7621c0d5a62816 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 6 Nov 2025 13:42:47 -0800 Subject: [PATCH 72/94] Revert "Final naming" This reverts commit c6c8949bac05487465c74aa4d36d909a28dd6f08. --- aerospike-client-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aerospike-client-c b/aerospike-client-c index 3bf664e7cf..3f15b94be5 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 3bf664e7cfc56908a2479b11cf2cf4f985c178e3 +Subproject commit 3f15b94be5aef435e5ae6681953638b680b91a6e From 1aee1e9e029bba1ec27ad7e60659662ad07851e3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 08:36:26 -0800 Subject: [PATCH 73/94] Fix regression where using client config read policy with POLICY_KEY_SEND does not include key in the record returned to the user --- src/main/client/get.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/client/get.c b/src/main/client/get.c index 19fb153541..f6b87e79a2 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -100,7 +100,8 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, AEROSPIKE_OK) { goto CLEANUP; } - if (!read_policy_p || + if ((!read_policy_p && + self->as->config.policies.read.key == AS_POLICY_KEY_DIGEST) || (read_policy_p && read_policy_p->key == AS_POLICY_KEY_DIGEST)) { // This is a special case. // C-client returns NULL key, so to the user From c93d92472021407167b74dbc7ec793a88f61746e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 08:45:54 -0800 Subject: [PATCH 74/94] Add back. not sure how this went missing --- src/main/policy.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/policy.c b/src/main/policy.c index ead9866239..6bc19ee033 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -470,6 +470,20 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query_copy(config_query_policy, policy); if (py_policy && py_policy != Py_None) { + if (self->validate_keys) { + as_status retval = does_py_dict_contain_valid_keys( + err, py_policy, py_query_policy_valid_keys, + POLICY_DICTIONARY_ADJECTIVE_FOR_ERROR_MESSAGE); + if (retval == -1) { + // This shouldn't happen, but if it did... + return as_error_update(err, AEROSPIKE_ERR, + ERR_MSG_FAILED_TO_VALIDATE_POLICY_KEYS); + } + else if (retval == 0) { + return err->code; + } + } + as_status retval = as_policy_base_set_from_pyobject( self, err, py_policy, &policy->base, true); if (retval != AEROSPIKE_OK) { From fe1e2a637af4f763091f37d06a7d05dfd541b497 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:02:38 -0800 Subject: [PATCH 75/94] fix comment --- src/include/policy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/policy.h b/src/include/policy.h index 4b9735aa54..50e6907463 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -341,7 +341,7 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, // These methods are used for setting transaction level policies // 1. Copies an config-level as_policy_* *src* to a txn-level *dst* policy // 2. Sets an as_policy_* *dst* from a Python object *py_policy*. to override the defaults from *src* -// *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_write instances. +// *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_* instances. // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, From 84882df6097c7b05aeb1273fd4db169de0cf6b4a Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:06:53 -0800 Subject: [PATCH 76/94] Comment better --- src/include/policy.h | 2 ++ src/include/policy_config.h | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/include/policy.h b/src/include/policy.h index 50e6907463..e95f26b33d 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -342,6 +342,8 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, // 1. Copies an config-level as_policy_* *src* to a txn-level *dst* policy // 2. Sets an as_policy_* *dst* from a Python object *py_policy*. to override the defaults from *src* // *py_policy* must be a Python dictionary, and *src* and *dst* must point to valid as_policy_* instances. +// We only need the AerospikeClient *self* for parsing Python client expressions and to tell whether to validate keys. +// // Returns AEROSPIKE_OK on success or another status code on error. On error, the err argument is populated. as_status as_policy_write_copy_and_set_from_pyobject(AerospikeClient *self, diff --git a/src/include/policy_config.h b/src/include/policy_config.h index 2f3127bfa7..5349cd1a39 100644 --- a/src/include/policy_config.h +++ b/src/include/policy_config.h @@ -40,6 +40,11 @@ as_status set_optional_gen(as_policy_gen *target_ptr, PyObject *py_policy, as_status set_optional_exists(as_policy_exists *target_ptr, PyObject *py_policy, const char *name); +// Although set_subpolicies is called by AerospikeClient's init method +// it should be safe to use because set_subpolicies only reads from self->validate_keys in this case +// We know that self->validate_keys is initialized by the time we call this. +// TODO: refactor set_subpolicies to not depend on AerospikeClient +// // This only sets the err object if an invalid dictionary key is passed // On error, return an error code as_status set_subpolicies(AerospikeClient *self, as_error *err, From c5088d0ccaff95ab8aa57c7b1ae0b6a2de978755 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:07:38 -0800 Subject: [PATCH 77/94] comment --- src/include/policy_config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/policy_config.h b/src/include/policy_config.h index 5349cd1a39..a788cd5299 100644 --- a/src/include/policy_config.h +++ b/src/include/policy_config.h @@ -40,7 +40,7 @@ as_status set_optional_gen(as_policy_gen *target_ptr, PyObject *py_policy, as_status set_optional_exists(as_policy_exists *target_ptr, PyObject *py_policy, const char *name); -// Although set_subpolicies is called by AerospikeClient's init method +// Although set_subpolicies is called by AerospikeClient's init method, and it takes in AerospikeClient as a param, // it should be safe to use because set_subpolicies only reads from self->validate_keys in this case // We know that self->validate_keys is initialized by the time we call this. // TODO: refactor set_subpolicies to not depend on AerospikeClient From 9074b469d04a62c6f008b12e58dd1588d0ee1086 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:37:15 -0800 Subject: [PATCH 78/94] Dont need to wrap this around do while since it's not going to be called inside an if statement block --- src/main/policy.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 6bc19ee033..8a3bbcae79 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -41,11 +41,9 @@ #define POLICY_UPDATE() *policy_p = policy; -// TODO: replace and remove // Don't make an inline function because // we want to return early if the policy is NULL or Py_None. // If we make this an inline function, we have to check again outside the inline function -// TODO: wrap around do while? #define RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ From df9a960503df8eac4b584cfeff20e40a1fc74a56 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:38:59 -0800 Subject: [PATCH 79/94] Remove unused inline func --- src/main/policy.c | 16 ++++------------ src/main/policy_config.c | 8 ++++---- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 8a3bbcae79..e249f7194b 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -44,6 +44,10 @@ // Don't make an inline function because // we want to return early if the policy is NULL or Py_None. // If we make this an inline function, we have to check again outside the inline function +// +// For transaction level policies, before calling this, we convert policy = None to NULL and this never gets called. +// But for config-level policies, we do not check whether py_policy is NULL or None. +// Either way, we want as_policy_*_set_from_pyobject to be a no-op. #define RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ @@ -53,18 +57,6 @@ "policy must be a dict"); \ } -// For transaction level policies, before calling this, we convert policy = None to NULL and this never gets called. -// But for config-level policies, we do not check whether py_policy is NULL or None. -// Either way, we want as_policy_*_set_from_pyobject to be a no-op. -static inline as_status is_pyobject_valid_policy_type(as_error *err, - PyObject *py_object) -{ - if (py_object == NULL || py_object == Py_None || PyDict_Check(py_object)) { - return AEROSPIKE_OK; - } - return as_error_update(err, AEROSPIKE_ERR_PARAM, "policy must be a dict"); -} - // TODO: Python exceptions should be propagated up instead of being cleared // but the policy helper functions don't handle this case and they only populate // an as_error object and return a status code. diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 1d26e1c32c..eb07702a68 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -47,16 +47,16 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, as_error_reset(err); as_status set_policy_status = AEROSPIKE_OK; - PyObject *read_policy = PyDict_GetItemString(py_policies_dict, "read"); + PyObject *py_read_policy = PyDict_GetItemString(py_policies_dict, "read"); set_policy_status = as_policy_read_set_from_pyobject( - self, err, read_policy, &config->policies.read, false); + self, err, py_read_policy, &config->policies.read, false); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } - PyObject *write_policy = PyDict_GetItemString(py_policies_dict, "write"); + PyObject *py_write_policy = PyDict_GetItemString(py_policies_dict, "write"); set_policy_status = as_policy_write_set_from_pyobject( - self, err, write_policy, &config->policies.write, false); + self, err, py_write_policy, &config->policies.write, false); if (set_policy_status != AEROSPIKE_OK) { return set_policy_status; } From 4dce0e6b44a8f71eb90022a1e437d540d5012d53 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:44:26 -0800 Subject: [PATCH 80/94] Use helper function from compatibility header --- src/main/policy.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index e249f7194b..845a9798f4 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -17,6 +17,8 @@ #include #include +#include "pythoncapi_compat.h" + #include #include #include @@ -73,7 +75,7 @@ "Unable to create Python unicode object"); \ } \ PyObject *py_field = \ - PyDict_GetItemWithError(py_policy, py_field_name); \ + PyDict_GetItemStringWithError(py_policy, py_field_name); \ if (py_field == NULL && PyErr_Occurred()) { \ PyErr_Clear(); \ Py_DECREF(py_field_name); \ @@ -103,23 +105,16 @@ #define POLICY_SET_EXPRESSIONS_FIELD() \ { \ - PyObject *py_field_name = PyUnicode_FromString("expressions"); \ - if (py_field_name == NULL) { \ - PyErr_Clear(); \ - return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ - "Unable to create Python unicode object"); \ - } \ - PyObject *py_exp_list = \ - PyDict_GetItemWithError(py_policy, py_field_name); \ - if (py_exp_list == NULL && PyErr_Occurred()) { \ + int retval = \ + PyDict_GetItemStringRef(py_policy, "expressions", py_exp_list); \ + if (retval == -1) { \ PyErr_Clear(); \ Py_DECREF(py_field_name); \ return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ "Unable to fetch expressions field " \ "from policy dictionary"); \ } \ - Py_DECREF(py_field_name); \ - if (py_exp_list) { \ + else if (py_exp_list) { \ as_exp *exp_list = NULL; \ if (as_exp_new_from_pyobject(self, py_exp_list, &exp_list, err, \ false) == AEROSPIKE_OK) { \ From b0b1c1d18606e0d0d4f741e88194d7bc95e646b1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:51:15 -0800 Subject: [PATCH 81/94] improve naming. --- src/main/policy.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 845a9798f4..7a41eabe5a 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -50,7 +50,7 @@ // For transaction level policies, before calling this, we convert policy = None to NULL and this never gets called. // But for config-level policies, we do not check whether py_policy is NULL or None. // Either way, we want as_policy_*_set_from_pyobject to be a no-op. -#define RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() \ +#define RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() \ if (py_policy == NULL || py_policy == Py_None) { \ return AEROSPIKE_OK; \ } \ @@ -242,7 +242,7 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, as_policy_admin *config_admin_policy) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); //Initialize policy with global defaults as_policy_admin_copy(config_admin_policy, policy); @@ -346,7 +346,7 @@ as_status as_policy_apply_set_from_pyobject(AerospikeClient *self, as_policy_apply *policy, bool is_this_txn_policy) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); if (self->validate_keys) { as_status retval = does_py_dict_contain_valid_keys( @@ -391,7 +391,7 @@ pyobject_to_policy_info(as_error *err, PyObject *py_policy, as_policy_info *config_info_policy, bool validate_keys, as_policy_with_extra_keys_allowed other_policy) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); //Initialize policy with global defaults as_policy_info_copy(config_info_policy, policy); @@ -449,7 +449,7 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query *config_query_policy, as_exp *exp_list, as_exp **exp_list_p) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); //Initialize policy with global defaults as_policy_query_copy(config_query_policy, policy); @@ -514,7 +514,7 @@ as_status as_policy_read_set_from_pyobject(AerospikeClient *self, as_error *err, as_policy_read *policy, bool is_policy_txn_level) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); if (self->validate_keys) { as_status retval = does_py_dict_contain_valid_keys( @@ -561,7 +561,7 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, as_policy_remove *config_remove_policy, as_exp *exp_list, as_exp **exp_list_p) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); //Initialize policy with global defaults as_policy_remove_copy(config_remove_policy, policy); @@ -614,7 +614,7 @@ as_status pyobject_to_policy_scan( as_policy_scan *config_scan_policy, as_exp *exp_list, as_exp **exp_list_p, bool py_policy_also_supports_info_policy_fields) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); //Initialize policy with global defaults as_policy_scan_copy(config_scan_policy, policy); @@ -675,7 +675,7 @@ as_status as_policy_write_set_from_pyobject(AerospikeClient *self, as_policy_write *policy, bool is_policy_txn_level) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() if (self->validate_keys) { as_status retval = does_py_dict_contain_valid_keys( @@ -727,7 +727,7 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, as_policy_operate *config_operate_policy, as_exp *exp_list, as_exp **exp_list_p) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE(); + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); //Initialize policy with global defaults as_policy_operate_copy(config_operate_policy, policy); @@ -786,7 +786,7 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, as_policy_batch *config_batch_policy, as_exp *exp_list, as_exp **exp_list_p) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() //Initialize policy with global defaults as_policy_batch_copy(config_batch_policy, policy); @@ -840,7 +840,7 @@ as_status pyobject_to_batch_write_policy(AerospikeClient *self, as_error *err, as_policy_batch_write **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_policy_batch_write_init(policy); if (self->validate_keys) { @@ -881,7 +881,7 @@ as_status pyobject_to_batch_read_policy(AerospikeClient *self, as_error *err, as_policy_batch_read **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_policy_batch_read_init(policy); if (self->validate_keys) { @@ -918,7 +918,7 @@ as_status pyobject_to_batch_apply_policy(AerospikeClient *self, as_error *err, as_policy_batch_apply **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_policy_batch_apply_init(policy); if (self->validate_keys) { @@ -957,7 +957,7 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, as_policy_batch_remove **policy_p, as_exp *exp_list, as_exp **exp_list_p) { - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_policy_batch_remove_init(policy); if (self->validate_keys) { @@ -993,7 +993,7 @@ as_status pyobject_to_bit_policy(as_error *err, PyObject *py_policy, as_bit_policy *policy, bool validate_keys) { as_bit_policy_init(policy); - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() if (validate_keys) { as_status retval = does_py_dict_contain_valid_keys( @@ -1030,7 +1030,7 @@ as_status pyobject_to_map_policy(as_error *err, PyObject *py_policy, as_map_policy *policy, bool validate_keys) { // Initialize Policy - RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE() + RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_map_policy_init(policy); if (validate_keys) { From 82195cad2f6373f32184a17433cfb765f01a3b5e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:52:09 -0800 Subject: [PATCH 82/94] fix --- src/main/policy.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/policy.c b/src/main/policy.c index 7a41eabe5a..790b8ba90f 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -105,8 +105,9 @@ #define POLICY_SET_EXPRESSIONS_FIELD() \ { \ + PyObject *py_exp_list = NULL; \ int retval = \ - PyDict_GetItemStringRef(py_policy, "expressions", py_exp_list); \ + PyDict_GetItemStringRef(py_policy, "expressions", &py_exp_list); \ if (retval == -1) { \ PyErr_Clear(); \ Py_DECREF(py_field_name); \ From 8ddb54c104c6311c2b88ff195546100a85551ba9 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:57:24 -0800 Subject: [PATCH 83/94] Fix --- src/main/policy.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/policy.c b/src/main/policy.c index 790b8ba90f..e9c7122b02 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -110,7 +110,6 @@ PyDict_GetItemStringRef(py_policy, "expressions", &py_exp_list); \ if (retval == -1) { \ PyErr_Clear(); \ - Py_DECREF(py_field_name); \ return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ "Unable to fetch expressions field " \ "from policy dictionary"); \ From 0cb1da51aa9ffe69d6c842f1a0b6ae4f7e098682 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 10:04:53 -0800 Subject: [PATCH 84/94] not needed unless something before is not resetting error --- src/main/policy_config.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index eb07702a68..5c9b3bf7dc 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -44,8 +44,6 @@ as_status set_optional_int_property(int *property_ptr, PyObject *py_policy, as_status set_subpolicies(AerospikeClient *self, as_error *err, as_config *config, PyObject *py_policies_dict) { - as_error_reset(err); - as_status set_policy_status = AEROSPIKE_OK; PyObject *py_read_policy = PyDict_GetItemString(py_policies_dict, "read"); set_policy_status = as_policy_read_set_from_pyobject( From bb4ce901261632c2362585f6a5eaec9e2977959d Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 10:06:59 -0800 Subject: [PATCH 85/94] Refactor --- src/main/policy_config.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 5c9b3bf7dc..ebe66499de 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -95,13 +95,6 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, return set_policy_status; } - PyObject *batch_policy = PyDict_GetItemString(py_policies_dict, "batch"); - set_policy_status = set_batch_policy(err, &config->policies.batch, - batch_policy, self->validate_keys); - if (set_policy_status != AEROSPIKE_OK) { - return set_policy_status; - } - PyObject *info_policy = PyDict_GetItemString(py_policies_dict, "info"); set_policy_status = set_info_policy(err, &config->policies.info, info_policy, self->validate_keys); @@ -143,17 +136,8 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, return set_policy_status; } - PyObject *batch_parent_write_policy = - PyDict_GetItemString(py_policies_dict, "batch_parent_write"); - set_policy_status = - set_batch_policy(err, &config->policies.batch_parent_write, - batch_parent_write_policy, self->validate_keys); - if (set_policy_status != AEROSPIKE_OK) { - return set_policy_status; - } - - // TODO: combine with other batch policies - const char *batch_policy_names[] = {"txn_verify", "txn_roll"}; + const char *batch_policy_names[] = {"batch", "batch_parent_write", + "txn_verify", "txn_roll"}; as_policy_batch *batch_policies[] = {&config->policies.txn_verify, &config->policies.txn_roll}; for (unsigned long i = 0; From 277612c34a4a5610cf030678b55140e890574940 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 10:07:59 -0800 Subject: [PATCH 86/94] fix --- src/main/policy_config.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/policy_config.c b/src/main/policy_config.c index ebe66499de..5c5875ad4f 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -138,8 +138,12 @@ as_status set_subpolicies(AerospikeClient *self, as_error *err, const char *batch_policy_names[] = {"batch", "batch_parent_write", "txn_verify", "txn_roll"}; - as_policy_batch *batch_policies[] = {&config->policies.txn_verify, - &config->policies.txn_roll}; + as_policy_batch *batch_policies[] = { + &config->policies.batch, + &config->policies.batch_parent_write, + &config->policies.txn_verify, + &config->policies.txn_roll, + }; for (unsigned long i = 0; i < sizeof(batch_policy_names) / sizeof(batch_policy_names[0]); i++) { PyObject *py_batch_policy = From 6aa561f9f81cb0668a5a1a9f89a21d15877926dc Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 10:15:04 -0800 Subject: [PATCH 87/94] Fix --- src/main/policy.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index e9c7122b02..61eab40ede 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -68,23 +68,14 @@ // exception that was thrown #define POLICY_SET_FIELD(__field, __type) \ { \ - PyObject *py_field_name = PyUnicode_FromString(#__field); \ - if (py_field_name == NULL) { \ - PyErr_Clear(); \ - return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ - "Unable to create Python unicode object"); \ - } \ - PyObject *py_field = \ - PyDict_GetItemStringWithError(py_policy, py_field_name); \ - if (py_field == NULL && PyErr_Occurred()) { \ + PyObject *py_field = NULL; \ + int retval = PyDict_GetItemStringRef(py_policy, #__field, &py_field); \ + if (retval == -1) { \ PyErr_Clear(); \ - Py_DECREF(py_field_name); \ return as_error_update( \ err, AEROSPIKE_ERR_CLIENT, \ "Unable to fetch field from policy dictionary"); \ } \ - Py_DECREF(py_field_name); \ - \ if (py_field) { \ if (PyLong_Check(py_field)) { \ long field_val = PyLong_AsLong(py_field); \ @@ -262,7 +253,6 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, } } - // Set policy fields POLICY_SET_FIELD(timeout, uint32_t); } // Update the policy From c28386073bc9cc06139d5ebd123fabed00eb95ed Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 10:30:54 -0800 Subject: [PATCH 88/94] It should be fine to change the error message. the same exception is being raised. --- test/new_tests/test_new_constructor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index d9233c5b1e..d5229e4872 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -498,9 +498,8 @@ def test_query_client_default_ttl(self, config_ttl_setup): def test_invalid_read_touch_ttl_percent(self, policy_name: str): config = copy.deepcopy(gconfig) config["policies"][policy_name]["read_touch_ttl_percent"] = "fail" - with pytest.raises(e.ParamError) as excinfo: + with pytest.raises(e.ParamError): aerospike.client(config) - assert excinfo.value.msg == "Invalid Policy setting value" @pytest.mark.parametrize( "config, context", From 7565cbccf44dbbd5be29bed42856bf892c87dbc7 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 13:42:52 -0800 Subject: [PATCH 89/94] fix mem leaks --- src/main/policy.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/policy.c b/src/main/policy.c index 61eab40ede..d2cad98b80 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -79,6 +79,7 @@ if (py_field) { \ if (PyLong_Check(py_field)) { \ long field_val = PyLong_AsLong(py_field); \ + Py_DECREF(py_field); \ if (field_val == -1 && PyErr_Occurred()) { \ PyErr_Clear(); \ return as_error_update( \ @@ -88,6 +89,7 @@ policy->__field = (__type)field_val; \ } \ else { \ + Py_DECREF(py_field); \ return as_error_update(err, AEROSPIKE_ERR_PARAM, \ "%s is invalid", #__field); \ } \ @@ -107,8 +109,10 @@ } \ else if (py_exp_list) { \ as_exp *exp_list = NULL; \ - if (as_exp_new_from_pyobject(self, py_exp_list, &exp_list, err, \ - false) == AEROSPIKE_OK) { \ + retval = as_exp_new_from_pyobject(self, py_exp_list, &exp_list, \ + err, false); \ + Py_DECREF(py_exp_list); \ + if (retval == AEROSPIKE_OK) { \ policy->filter_exp = exp_list; \ } \ else { \ From 7243fd539196af62c6b3dcd9ac7e94087c30e9c3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 14:43:52 -0800 Subject: [PATCH 90/94] TODO WIP: remove exp_list_p and exp_list parameters since we can access the exp_list directly from the policy object. --- src/include/policy.h | 26 +++++++++++--------------- src/main/policy.c | 35 ++++++++++++++--------------------- 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index e95f26b33d..da805bf8d2 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -215,8 +215,7 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_apply *policy, as_policy_apply **policy_p, - as_policy_apply *config_apply_policy, - as_exp *exp_list, as_exp **exp_list_p); + as_policy_apply *config_apply_policy); typedef enum { SECOND_AS_POLICY_WRITE, @@ -234,35 +233,32 @@ pyobject_to_policy_info(as_error *err, PyObject *py_policy, as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_query *policy, as_policy_query **policy_p, - as_policy_query *config_query_policy, - as_exp *exp_list, as_exp **exp_list_p); + as_policy_query *config_query_policy); as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_remove *policy, as_policy_remove **policy_p, - as_policy_remove *config_remove_policy, - as_exp *exp_list, as_exp **exp_list_p); + as_policy_remove *config_remove_policy); // py_policy_also_supports_info_policy_fields only applies if self->validate_keys is true -as_status pyobject_to_policy_scan( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_scan *policy, as_policy_scan **policy_p, - as_policy_scan *config_scan_policy, as_exp *exp_list, as_exp **exp_list_p, - bool py_policy_also_supports_info_policy_fields); +as_status +pyobject_to_policy_scan(AerospikeClient *self, as_error *err, + PyObject *py_policy, as_policy_scan *policy, + as_policy_scan **policy_p, + as_policy_scan *config_scan_policy, + bool py_policy_also_supports_info_policy_fields); as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_operate *policy, as_policy_operate **policy_p, - as_policy_operate *config_operate_policy, - as_exp *exp_list, as_exp **exp_list_p); + as_policy_operate *config_operate_policy); as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch *policy, as_policy_batch **policy_p, - as_policy_batch *config_batch_policy, - as_exp *exp_list, as_exp **exp_list_p); + as_policy_batch *config_batch_policy); as_status pyobject_to_map_policy(as_error *err, PyObject *py_policy, as_map_policy *policy, bool validate_keys); diff --git a/src/main/policy.c b/src/main/policy.c index d2cad98b80..a0c7abd462 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -440,8 +440,7 @@ pyobject_to_policy_info(as_error *err, PyObject *py_policy, as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_query *policy, as_policy_query **policy_p, - as_policy_query *config_query_policy, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_query *config_query_policy) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); @@ -552,8 +551,7 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_remove *policy, as_policy_remove **policy_p, - as_policy_remove *config_remove_policy, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_remove *config_remove_policy) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); //Initialize policy with global defaults @@ -602,11 +600,12 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, * We assume that the error object and the policy object are already allocated * and initialized (although, we do reset the error object here). */ -as_status pyobject_to_policy_scan( - AerospikeClient *self, as_error *err, PyObject *py_policy, - as_policy_scan *policy, as_policy_scan **policy_p, - as_policy_scan *config_scan_policy, as_exp *exp_list, as_exp **exp_list_p, - bool py_policy_also_supports_info_policy_fields) +as_status +pyobject_to_policy_scan(AerospikeClient *self, as_error *err, + PyObject *py_policy, as_policy_scan *policy, + as_policy_scan **policy_p, + as_policy_scan *config_scan_policy, + bool py_policy_also_supports_info_policy_fields) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); @@ -718,8 +717,7 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_operate *policy, as_policy_operate **policy_p, - as_policy_operate *config_operate_policy, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_operate *config_operate_policy) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL(); @@ -777,8 +775,7 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch *policy, as_policy_batch **policy_p, - as_policy_batch *config_batch_policy, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_batch *config_batch_policy) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() @@ -831,8 +828,7 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, as_status pyobject_to_batch_write_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_write *policy, - as_policy_batch_write **policy_p, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_batch_write **policy_p) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_policy_batch_write_init(policy); @@ -872,8 +868,7 @@ as_status pyobject_to_batch_write_policy(AerospikeClient *self, as_error *err, as_status pyobject_to_batch_read_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_read *policy, - as_policy_batch_read **policy_p, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_batch_read **policy_p) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_policy_batch_read_init(policy); @@ -909,8 +904,7 @@ as_status pyobject_to_batch_read_policy(AerospikeClient *self, as_error *err, as_status pyobject_to_batch_apply_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_apply *policy, - as_policy_batch_apply **policy_p, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_batch_apply **policy_p) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_policy_batch_apply_init(policy); @@ -948,8 +942,7 @@ as_status pyobject_to_batch_apply_policy(AerospikeClient *self, as_error *err, as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_remove *policy, - as_policy_batch_remove **policy_p, - as_exp *exp_list, as_exp **exp_list_p) + as_policy_batch_remove **policy_p) { RETURN_IF_PY_POLICY_IS_INVALID_OR_NONE_OR_NULL() as_policy_batch_remove_init(policy); From 07e367ddd65b1ffce2bb06ca14bf156fa081a65d Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 7 Nov 2025 14:57:32 -0800 Subject: [PATCH 91/94] WIP. todo check that this is good practice.. --- src/include/policy.h | 13 ++++--------- src/main/client/batch_apply.c | 8 ++++---- src/main/client/batch_operate.c | 8 ++++---- src/main/client/batch_read.c | 4 ++-- src/main/client/batch_write.c | 2 -- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/include/policy.h b/src/include/policy.h index da805bf8d2..56a805abc6 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -280,27 +280,22 @@ as_status pyobject_to_hll_policy(as_error *err, PyObject *py_policy, as_status pyobject_to_batch_write_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_write *policy, - as_policy_batch_write **policy_p, - as_exp *exp_list, as_exp **exp_list_p); + as_policy_batch_write **policy_p); as_status pyobject_to_batch_read_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_read *policy, - as_policy_batch_read **policy_p, - as_exp *exp_list, as_exp **exp_list_p); + as_policy_batch_read **policy_p); as_status pyobject_to_batch_apply_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_apply *policy, - as_policy_batch_apply **policy_p, - as_exp *exp_list, as_exp **exp_list_p); + as_policy_batch_apply **policy_p); as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_batch_remove *policy, - as_policy_batch_remove **policy_p, - as_exp *exp_list, - as_exp **exp_list_p); + as_policy_batch_remove **policy_p); // These methods are used for setting config-level policies and are re-used for the helper methods // that set transaction-level policies. diff --git a/src/main/client/batch_apply.c b/src/main/client/batch_apply.c index dcbfde5cde..91d099d628 100644 --- a/src/main/client/batch_apply.c +++ b/src/main/client/batch_apply.c @@ -272,12 +272,12 @@ static PyObject *AerospikeClient_Batch_Apply_Invoke( as_list_destroy(arglist); } - if (batch_exp_list_p) { - as_exp_destroy(batch_exp_list_p); + if (policy_batch_p) { + as_exp_destroy(policy_batch_p->base.filter_exp); } - if (batch_apply_exp_list_p) { - as_exp_destroy(batch_apply_exp_list_p); + if (policy_batch_apply_p) { + as_exp_destroy(policy_batch_apply_p->filter_exp); } as_batch_destroy(&batch); diff --git a/src/main/client/batch_operate.c b/src/main/client/batch_operate.c index 7fba363e71..e80d5e84e2 100644 --- a/src/main/client/batch_operate.c +++ b/src/main/client/batch_operate.c @@ -301,12 +301,12 @@ static PyObject *AerospikeClient_Batch_Operate_Invoke( free(as_vector_get_ptr(unicodeStrVector, i)); } - if (batch_exp_list_p) { - as_exp_destroy(batch_exp_list_p); + if (policy_batch_p) { + as_exp_destroy(policy_batch_p->base.filter_exp); } - if (batch_write_exp_list_p) { - as_exp_destroy(batch_write_exp_list_p); + if (policy_batch_write_p) { + as_exp_destroy(policy_batch_write_p->filter_exp); } as_vector_destroy(unicodeStrVector); diff --git a/src/main/client/batch_read.c b/src/main/client/batch_read.c index 2784a7fc4f..e86b2f44bb 100644 --- a/src/main/client/batch_read.c +++ b/src/main/client/batch_read.c @@ -287,8 +287,8 @@ PyObject *AerospikeClient_BatchRead(AerospikeClient *self, PyObject *args, as_batch_destroy(&batch); - if (batch_exp_list_p) { - as_exp_destroy(batch_exp_list_p); + if (policy_batch_p) { + as_exp_destroy(policy_batch_p->base.filter_exp); } CLEANUP2: diff --git a/src/main/client/batch_write.c b/src/main/client/batch_write.c index 29ef4b10d8..74093e7ef9 100644 --- a/src/main/client/batch_write.c +++ b/src/main/client/batch_write.c @@ -135,8 +135,6 @@ static PyObject *AerospikeClient_BatchWriteInvoke(AerospikeClient *self, as_policy_batch batch_policy; as_policy_batch *batch_policy_p = NULL; - as_exp exp_list; - as_exp *exp_list_p = NULL; PyObject *py_batch_type = NULL; PyObject *py_key = NULL; From ebaef29e6a5ddf73f120b36d30df20f5494e81fd Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 10 Nov 2025 08:40:31 -0800 Subject: [PATCH 92/94] WIP removing filter_exp arguments and variables. TODO not sure if free'ing the filter_exp by dereferencing the policy pointer will cause performance costs --- src/main/client/batch_apply.c | 11 ++--------- src/main/client/batch_operate.c | 13 ++----------- src/main/client/batch_read.c | 7 +------ src/main/client/batch_remove.c | 21 ++++++--------------- src/main/client/batch_write.c | 18 +++++++----------- src/main/client/operate.c | 22 ++++++---------------- src/main/client/remove.c | 14 +++++--------- src/main/client/scan.c | 11 +++-------- src/main/query/foreach.c | 14 +++++--------- src/main/query/results.c | 14 +++++--------- src/main/scan/execute_background.c | 17 ++++++----------- 11 files changed, 48 insertions(+), 114 deletions(-) diff --git a/src/main/client/batch_apply.c b/src/main/client/batch_apply.c index 91d099d628..aa558c57f6 100644 --- a/src/main/client/batch_apply.c +++ b/src/main/client/batch_apply.c @@ -123,11 +123,6 @@ static PyObject *AerospikeClient_Batch_Apply_Invoke( as_batch_init(&batch, 0); // For expressions conversion. - as_exp batch_exp_list; - as_exp *batch_exp_list_p = NULL; - - as_exp batch_apply_exp_list; - as_exp *batch_apply_exp_list_p = NULL; PyObject *br_instance = NULL; @@ -181,8 +176,7 @@ static PyObject *AerospikeClient_Batch_Apply_Invoke( if (py_policy_batch) { if (pyobject_to_policy_batch( self, err, py_policy_batch, &policy_batch, &policy_batch_p, - &self->as->config.policies.batch, &batch_exp_list, - &batch_exp_list_p) != AEROSPIKE_OK) { + &self->as->config.policies.batch) != AEROSPIKE_OK) { goto CLEANUP; } } @@ -190,8 +184,7 @@ static PyObject *AerospikeClient_Batch_Apply_Invoke( if (py_policy_batch_apply) { if (pyobject_to_batch_apply_policy( self, err, py_policy_batch_apply, &policy_batch_apply, - &policy_batch_apply_p, &batch_apply_exp_list, - &batch_apply_exp_list_p) != AEROSPIKE_OK) { + &policy_batch_apply_p) != AEROSPIKE_OK) { goto CLEANUP; } } diff --git a/src/main/client/batch_operate.c b/src/main/client/batch_operate.c index e80d5e84e2..50c109ab37 100644 --- a/src/main/client/batch_operate.c +++ b/src/main/client/batch_operate.c @@ -129,13 +129,6 @@ static PyObject *AerospikeClient_Batch_Operate_Invoke( as_batch batch; as_batch_init(&batch, 0); - // For expressions conversion. - as_exp batch_exp_list; - as_exp *batch_exp_list_p = NULL; - - as_exp batch_write_exp_list; - as_exp *batch_write_exp_list_p = NULL; - as_vector *unicodeStrVector = as_vector_create(sizeof(char *), 128); as_static_pool static_pool; @@ -210,8 +203,7 @@ static PyObject *AerospikeClient_Batch_Operate_Invoke( if (py_policy_batch) { if (pyobject_to_policy_batch( self, err, py_policy_batch, &policy_batch, &policy_batch_p, - &self->as->config.policies.batch, &batch_exp_list, - &batch_exp_list_p) != AEROSPIKE_OK) { + &self->as->config.policies.batch) != AEROSPIKE_OK) { goto CLEANUP; } } @@ -219,8 +211,7 @@ static PyObject *AerospikeClient_Batch_Operate_Invoke( if (py_policy_batch_write) { if (pyobject_to_batch_write_policy( self, err, py_policy_batch_write, &policy_batch_write, - &policy_batch_write_p, &batch_write_exp_list, - &batch_write_exp_list_p) != AEROSPIKE_OK) { + &policy_batch_write_p) != AEROSPIKE_OK) { goto CLEANUP; } } diff --git a/src/main/client/batch_read.c b/src/main/client/batch_read.c index e86b2f44bb..49107fae83 100644 --- a/src/main/client/batch_read.c +++ b/src/main/client/batch_read.c @@ -152,15 +152,10 @@ PyObject *AerospikeClient_BatchRead(AerospikeClient *self, PyObject *args, as_policy_batch policy_batch; as_policy_batch *policy_batch_p = NULL; - // For expressions conversion. - as_exp batch_exp_list; - as_exp *batch_exp_list_p = NULL; - if (py_policy_batch) { if (pyobject_to_policy_batch( self, &err, py_policy_batch, &policy_batch, &policy_batch_p, - &self->as->config.policies.batch, &batch_exp_list, - &batch_exp_list_p) != AEROSPIKE_OK) { + &self->as->config.policies.batch) != AEROSPIKE_OK) { goto CLEANUP3; } } diff --git a/src/main/client/batch_remove.c b/src/main/client/batch_remove.c index 24bcb0085f..d6f696a010 100644 --- a/src/main/client/batch_remove.c +++ b/src/main/client/batch_remove.c @@ -121,13 +121,6 @@ static PyObject *AerospikeClient_Batch_Remove_Invoke( as_batch batch; as_batch_init(&batch, 0); - // For expressions conversion. - as_exp batch_exp_list; - as_exp *batch_exp_list_p = NULL; - - as_exp batch_remove_exp_list; - as_exp *batch_remove_exp_list_p = NULL; - PyObject *br_instance = NULL; Py_ssize_t keys_size = PyList_Size(py_keys); @@ -175,8 +168,7 @@ static PyObject *AerospikeClient_Batch_Remove_Invoke( if (py_policy_batch) { if (pyobject_to_policy_batch( self, err, py_policy_batch, &policy_batch, &policy_batch_p, - &self->as->config.policies.batch, &batch_exp_list, - &batch_exp_list_p) != AEROSPIKE_OK) { + &self->as->config.policies.batch) != AEROSPIKE_OK) { goto CLEANUP; } } @@ -184,8 +176,7 @@ static PyObject *AerospikeClient_Batch_Remove_Invoke( if (py_policy_batch_remove) { if (pyobject_to_batch_remove_policy( self, err, py_policy_batch_remove, &policy_batch_remove, - &policy_batch_remove_p, &batch_remove_exp_list, - &batch_remove_exp_list_p) != AEROSPIKE_OK) { + &policy_batch_remove_p) != AEROSPIKE_OK) { goto CLEANUP; } } @@ -253,12 +244,12 @@ static PyObject *AerospikeClient_Batch_Remove_Invoke( as_error_reset(err); CLEANUP: - if (batch_exp_list_p) { - as_exp_destroy(batch_exp_list_p); + if (policy_batch_p) { + as_exp_destroy(policy_batch_p->base.filter_exp); } - if (batch_remove_exp_list_p) { - as_exp_destroy(batch_remove_exp_list_p); + if (policy_batch_remove_p) { + as_exp_destroy(policy_batch_remove_p->filter_exp); } as_batch_destroy(&batch); diff --git a/src/main/client/batch_write.c b/src/main/client/batch_write.c index 74093e7ef9..90e430f868 100644 --- a/src/main/client/batch_write.c +++ b/src/main/client/batch_write.c @@ -44,14 +44,11 @@ PyObject *py___policy = \ PyObject_GetAttrString(py_batch_record, FIELD_NAME_BATCH_POLICY); \ if (py___policy != Py_None) { \ - as_exp *expr = NULL; \ - as_exp *expr_p = expr; \ if (py___policy != NULL) { \ __policy = (__policy_type *)malloc(sizeof(__policy_type)); \ garb->policy_to_free = __policy; \ if (__conversion_func(self, err, py___policy, __policy, \ - &__policy, expr, \ - &expr_p) != AEROSPIKE_OK) { \ + &__policy) != AEROSPIKE_OK) { \ /* Don't call strstr unless we have to. It is a linear time operation */ \ /* Also, not bothering to use POSIX regex library in this case */ \ if (!(self->validate_keys && \ @@ -66,7 +63,7 @@ Py_DECREF(py___policy); \ goto CLEANUP_ON_ERROR; \ } \ - garb->expressions_to_free = expr_p; \ + garb->expressions_to_free = __policy->filter_exp; \ } \ else { \ as_error_update(err, AEROSPIKE_ERR_PARAM, \ @@ -166,10 +163,9 @@ static PyObject *AerospikeClient_BatchWriteInvoke(AerospikeClient *self, } if (py_policy != NULL) { - if (pyobject_to_policy_batch(self, err, py_policy, &batch_policy, - &batch_policy_p, - &self->as->config.policies.batch, - &exp_list, &exp_list_p) != AEROSPIKE_OK) { + if (pyobject_to_policy_batch( + self, err, py_policy, &batch_policy, &batch_policy_p, + &self->as->config.policies.batch) != AEROSPIKE_OK) { goto CLEANUP4; } } @@ -554,8 +550,8 @@ static PyObject *AerospikeClient_BatchWriteInvoke(AerospikeClient *self, as_vector_destroy(unicodeStrVector); - if (exp_list_p != NULL) { - as_exp_destroy(exp_list_p); + if (batch_policy_p != NULL) { + as_exp_destroy(batch_policy_p->base.filter_exp); } if (err->code != AEROSPIKE_OK) { diff --git a/src/main/client/operate.c b/src/main/client/operate.c index 2910558c8a..b71d442a4b 100644 --- a/src/main/client/operate.c +++ b/src/main/client/operate.c @@ -922,10 +922,6 @@ static PyObject *AerospikeClient_Operate_Invoke(AerospikeClient *self, as_policy_operate operate_policy; as_policy_operate *operate_policy_p = NULL; - // For expressions conversion. - as_exp exp_list; - as_exp *exp_list_p = NULL; - as_vector *unicodeStrVector = as_vector_create(sizeof(char *), 128); as_operations ops; @@ -935,8 +931,7 @@ static PyObject *AerospikeClient_Operate_Invoke(AerospikeClient *self, if (py_policy) { if (pyobject_to_policy_operate( self, err, py_policy, &operate_policy, &operate_policy_p, - &self->as->config.policies.operate, &exp_list, - &exp_list_p) != AEROSPIKE_OK) { + &self->as->config.policies.operate) != AEROSPIKE_OK) { goto CLEANUP; } } @@ -984,8 +979,8 @@ static PyObject *AerospikeClient_Operate_Invoke(AerospikeClient *self, free(as_vector_get_ptr(unicodeStrVector, i)); } - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (operate_policy_p) { + as_exp_destroy(operate_policy_p->base.filter_exp); } as_vector_destroy(unicodeStrVector); @@ -1097,10 +1092,6 @@ AerospikeClient_OperateOrdered_Invoke(AerospikeClient *self, as_error *err, Py_ssize_t ops_list_size = PyList_Size(py_list); as_operations_inita(&ops, ops_list_size); - // For expressions conversion. - as_exp exp_list; - as_exp *exp_list_p = NULL; - /* These are the values which will be returned in a 3 element list */ PyObject *py_return_key = NULL; PyObject *py_return_meta = NULL; @@ -1111,8 +1102,7 @@ AerospikeClient_OperateOrdered_Invoke(AerospikeClient *self, as_error *err, if (py_policy) { if (pyobject_to_policy_operate( self, err, py_policy, &operate_policy, &operate_policy_p, - &self->as->config.policies.operate, &exp_list, - &exp_list_p) != AEROSPIKE_OK) { + &self->as->config.policies.operate) != AEROSPIKE_OK) { goto CLEANUP; } } @@ -1198,8 +1188,8 @@ AerospikeClient_OperateOrdered_Invoke(AerospikeClient *self, as_error *err, as_vector_destroy(unicodeStrVector); - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (operate_policy_p) { + as_exp_destroy(operate_policy_p->base.filter_exp); } if (rec && operation_succeeded) { diff --git a/src/main/client/remove.c b/src/main/client/remove.c index 0b76b6b358..58eb00246f 100644 --- a/src/main/client/remove.c +++ b/src/main/client/remove.c @@ -50,10 +50,6 @@ PyObject *AerospikeClient_Remove_Invoke(AerospikeClient *self, PyObject *py_key, as_policy_remove *remove_policy_p = NULL; as_key key; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - // Initialisation flags bool key_initialised = false; @@ -81,9 +77,9 @@ PyObject *AerospikeClient_Remove_Invoke(AerospikeClient *self, PyObject *py_key, // Convert python policy object to as_policy_exists if (py_policy) { - pyobject_to_policy_remove( - self, &err, py_policy, &remove_policy, &remove_policy_p, - &self->as->config.policies.remove, &exp_list, &exp_list_p); + pyobject_to_policy_remove(self, &err, py_policy, &remove_policy, + &remove_policy_p, + &self->as->config.policies.remove); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } @@ -123,8 +119,8 @@ PyObject *AerospikeClient_Remove_Invoke(AerospikeClient *self, PyObject *py_key, CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (remove_policy_p) { + as_exp_destroy(remove_policy_p->base.filter_exp); } if (key_initialised == true) { diff --git a/src/main/client/scan.c b/src/main/client/scan.c index 9409df4828..5041d1a17b 100644 --- a/src/main/client/scan.c +++ b/src/main/client/scan.c @@ -80,10 +80,6 @@ static PyObject *AerospikeClient_ScanApply_Invoke( PyObject *py_ustr2 = NULL; PyObject *py_ustr3 = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - as_static_pool static_pool; memset(&static_pool, 0, sizeof(static_pool)); @@ -132,7 +128,7 @@ static PyObject *AerospikeClient_ScanApply_Invoke( if (py_policy) { pyobject_to_policy_scan(self, &err, py_policy, &scan_policy, &scan_policy_p, &self->as->config.policies.scan, - &exp_list, &exp_list_p, true); + true); if (err.code != AEROSPIKE_OK) { goto CLEANUP; @@ -211,9 +207,8 @@ static PyObject *AerospikeClient_ScanApply_Invoke( CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); - ; + if (scan_policy_p) { + as_exp_destroy(scan_policy_p->base.filter_exp); } if (py_ustr1) { diff --git a/src/main/query/foreach.c b/src/main/query/foreach.c index 643131e120..373fd0a0e6 100644 --- a/src/main/query/foreach.c +++ b/src/main/query/foreach.c @@ -164,10 +164,6 @@ PyObject *AerospikeQuery_Foreach(AerospikeQuery *self, PyObject *args, as_policy_query query_policy; as_policy_query *query_policy_p = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - as_partition_filter partition_filter = {0}; as_partition_filter *partition_filter_p = NULL; as_partitions_status *ps = NULL; @@ -186,9 +182,9 @@ PyObject *AerospikeQuery_Foreach(AerospikeQuery *self, PyObject *args, } // Convert python policy object to as_policy_exists - pyobject_to_policy_query( - self->client, &err, py_policy, &query_policy, &query_policy_p, - &self->client->as->config.policies.query, &exp_list, &exp_list_p); + pyobject_to_policy_query(self->client, &err, py_policy, &query_policy, + &query_policy_p, + &self->client->as->config.policies.query); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } @@ -244,8 +240,8 @@ PyObject *AerospikeQuery_Foreach(AerospikeQuery *self, PyObject *args, } CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (query_policy_p) { + as_exp_destroy(query_policy_p->base.filter_exp); } if (self->query.apply.arglist) { diff --git a/src/main/query/results.c b/src/main/query/results.c index bc0994cb42..0b4241a9bc 100644 --- a/src/main/query/results.c +++ b/src/main/query/results.c @@ -87,10 +87,6 @@ PyObject *AerospikeQuery_Results(AerospikeQuery *self, PyObject *args, as_policy_query query_policy; as_policy_query *query_policy_p = NULL; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - as_partition_filter partition_filter = {0}; as_partition_filter *partition_filter_p = NULL; as_partitions_status *ps = NULL; @@ -107,9 +103,9 @@ PyObject *AerospikeQuery_Results(AerospikeQuery *self, PyObject *args, } // Convert python policy object to as_policy_query - pyobject_to_policy_query( - self->client, &err, py_policy, &query_policy, &query_policy_p, - &self->client->as->config.policies.query, &exp_list, &exp_list_p); + pyobject_to_policy_query(self->client, &err, py_policy, &query_policy, + &query_policy_p, + &self->client->as->config.policies.query); if (err.code != AEROSPIKE_OK) { goto CLEANUP; } @@ -160,8 +156,8 @@ PyObject *AerospikeQuery_Results(AerospikeQuery *self, PyObject *args, Py_END_ALLOW_THREADS CLEANUP: /*??trace()*/ - if (exp_list_p) { - as_exp_destroy(exp_list_p); + if (query_policy_p) { + as_exp_destroy(query_policy_p->base.filter_exp); } if (err.code != AEROSPIKE_OK) { diff --git a/src/main/scan/execute_background.c b/src/main/scan/execute_background.c index 0c2e7affa9..d5dcc4703b 100644 --- a/src/main/scan/execute_background.c +++ b/src/main/scan/execute_background.c @@ -38,10 +38,6 @@ PyObject *AerospikeScan_ExecuteBackground(AerospikeScan *self, PyObject *args, uint64_t scan_id = 0; static char *kwlist[] = {"policy", NULL}; - // For converting expressions. - as_exp exp_list; - as_exp *exp_list_p = NULL; - if (PyArg_ParseTupleAndKeywords(args, kwds, "|O:execute_background", kwlist, &py_policy) == false) { return NULL; @@ -61,10 +57,10 @@ PyObject *AerospikeScan_ExecuteBackground(AerospikeScan *self, PyObject *args, } if (py_policy) { - if (pyobject_to_policy_scan( - self->client, &err, py_policy, &scan_policy, &scan_policy_p, - &self->client->as->config.policies.scan, &exp_list, &exp_list_p, - false) != AEROSPIKE_OK) { + if (pyobject_to_policy_scan(self->client, &err, py_policy, &scan_policy, + &scan_policy_p, + &self->client->as->config.policies.scan, + false) != AEROSPIKE_OK) { goto CLEANUP; } } @@ -76,9 +72,8 @@ PyObject *AerospikeScan_ExecuteBackground(AerospikeScan *self, PyObject *args, CLEANUP: - if (exp_list_p) { - as_exp_destroy(exp_list_p); - ; + if (scan_policy_p) { + as_exp_destroy(scan_policy_p->base.filter_exp); } if (err.code != AEROSPIKE_OK) { From 4ffdf955d8194b5ca09376be1d34369c808d7b56 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:37:37 -0700 Subject: [PATCH 93/94] Address compiler error. --- src/main/scan/foreach.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scan/foreach.c b/src/main/scan/foreach.c index 2d82eab761..2c1b384d08 100644 --- a/src/main/scan/foreach.c +++ b/src/main/scan/foreach.c @@ -172,9 +172,9 @@ PyObject *AerospikeScan_Foreach(AerospikeScan *self, PyObject *args, } // Convert python policy object to as_policy_exists - pyobject_to_policy_scan( - self->client, &data.error, py_policy, &scan_policy, &scan_policy_p, - &self->client->as->config.policies.scan, &exp_list_p, false); + pyobject_to_policy_scan(self->client, &data.error, py_policy, &scan_policy, + &scan_policy_p, + &self->client->as->config.policies.scan, false); if (data.error.code != AEROSPIKE_OK) { goto CLEANUP; From 23f9cb9a21f94844e47f6d15e525c8a45eabe13b Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:39:55 -0700 Subject: [PATCH 94/94] Address compiler error. This should be the last of the errors from pyobject_to_policy_scan --- src/main/scan/results.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scan/results.c b/src/main/scan/results.c index 9113621559..8a9bf8cc78 100644 --- a/src/main/scan/results.c +++ b/src/main/scan/results.c @@ -109,9 +109,9 @@ PyObject *AerospikeScan_Results(AerospikeScan *self, PyObject *args, } // Convert python policy object to as_policy_scan - pyobject_to_policy_scan( - self->client, &err, py_policy, &scan_policy, &scan_policy_p, - &self->client->as->config.policies.scan, &exp_list_p, false); + pyobject_to_policy_scan(self->client, &err, py_policy, &scan_policy, + &scan_policy_p, + &self->client->as->config.policies.scan, false); if (err.code != AEROSPIKE_OK) { as_error_update(&err, err.code, NULL); goto CLEANUP;