diff --git a/NEWS.md b/NEWS.md index a306e59..5cc91b5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -88,6 +88,17 @@ ## Bug fixes +* `setup_pointblank_agent()` cleanup: the `cols_reference` argument was never + read (verified by grep since its introduction) and is now deprecated + (warning when supplied, removal planned); the local equality steps validate + a derived `__eq` boolean with the shared NA semantics instead of + embedding the whole reference vector in each step (O(rows) per step, + serialised with the report, and unable to express the one-sided/two-sided + NA distinction); the roxygen example is now executable and representative + (it used to target a `__ok` column that did not exist); `get_col_names()` + is hoisted out of the per-column loop; the dead `cols_reference`/ + `cols_candidate` locals of the caller are gone (issue #25). + * `equal_mode: normalized` now does what the vignette always said: it implies `case_insensitive = TRUE` and `trim = TRUE` for character columns unless those flags are set explicitly (an explicit value wins over the mode). It diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 259d554..0970389 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -540,8 +540,6 @@ compare_datasets_from_yaml <- function(data_reference, # Analyze columns col_analysis <- analyze_columns(data_reference, data_candidate, ignore_columns = ignore_columns) - cols_reference <- col_analysis$cols_reference - cols_candidate <- col_analysis$cols_candidate missing_in_candidate <- col_analysis$missing_in_candidate extra_in_candidate <- col_analysis$extra_in_candidate common_cols <- col_analysis$common_cols @@ -864,17 +862,16 @@ compare_datasets_from_yaml <- function(data_reference, } agent <- setup_pointblank_agent( cmp_for_agent, - cols_reference, - fail$eq, - fail$tol, - row_validation_info, - ref_suffix, - warn_at, - stop_at, - label, - na_equal, - lang, - locale, + common_cols = fail$eq, + tol_cols = fail$tol, + row_validation_info = row_validation_info, + ref_suffix = ref_suffix, + warn_at = warn_at, + stop_at = stop_at, + label = label, + na_equal = na_equal, + lang = lang, + locale = locale, missing_in_candidate = missing_in_candidate, type_mismatch_cols = type_mismatch_cols, add_col_exists_steps = FALSE diff --git a/R/pointblank_setup.R b/R/pointblank_setup.R index b3dc16c..2173113 100644 --- a/R/pointblank_setup.R +++ b/R/pointblank_setup.R @@ -3,9 +3,19 @@ #' Creates and configures a pointblank validation agent with all necessary validation steps #' including column existence checks, exact value comparisons, and tolerance validations. #' +#' The equality steps validate a `__eq` boolean (one-sided NA fails, +#' two-sided NA follows `na_equal`, matching the tolerance kernel and the lazy +#' SQL): when `cmp` does not already carry it, it is derived on the fly for a +#' local data.frame. Tolerance steps expect the `__ok` booleans to be +#' precomputed (see [add_tolerance_columns()]). A lazy `cmp` must carry every +#' boolean column already. +#' #' @param cmp Comparison dataframe with candidate and reference data -#' @param cols_reference Character vector of reference column names -#' @param common_cols Character vector of columns present in both datasets +#' @param cols_reference Deprecated and unused; supplying any non-NULL value +#' raises a warning (an explicit NULL is silent) and the argument will be +#' removed in a future release +#' @param common_cols Character vector of equality columns to validate (the +#' internal pipeline passes only the failing ones on its failure path) #' @param tol_cols Character vector of columns with tolerance rules #' @param row_validation_info List with row validation information from validate_row_counts #' @param ref_suffix Suffix for reference columns @@ -29,21 +39,36 @@ #' data columns. #' @return Configured pointblank agent ready for interrogation #' @examples -#' cmp <- data.frame(a = 1:3, a__reference = 1:3, b = c(1.1, 2.2, 3.3), -#' b__reference = c(1.0, 2.0, 3.0)) +#' cmp <- data.frame( +#' a = 1:3, a__reference = 1:3, +#' b = c(1.1, 2.2, 3.3), b__reference = c(1.0, 2.0, 3.0), +#' b__ok = c(FALSE, FALSE, FALSE) +#' ) #' row_info <- list(check_count = FALSE) -#' setup_pointblank_agent(cmp, c("a", "b"), c("a", "b"), "b", row_info, -#' "__reference", 0.1, 0.1, "Test", TRUE) +#' agent <- setup_pointblank_agent( +#' cmp, +#' common_cols = "a", tol_cols = "b", row_validation_info = row_info, +#' ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, +#' label = "Test", na_equal = TRUE +#' ) +#' pointblank::all_passed(pointblank::interrogate(agent)) #' @importFrom pointblank create_agent col_exists col_vals_equal action_levels #' @importFrom tidyselect all_of #' @importFrom rlang := #' @export -setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, +setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_cols, row_validation_info = NULL, ref_suffix, warn_at, stop_at, label, na_equal, lang = "fr", locale = "fr_FR", missing_in_candidate = character(0), type_mismatch_cols = character(0), add_col_exists_steps = TRUE) { + if (!is.null(cols_reference)) { + warning( + "The 'cols_reference' argument of setup_pointblank_agent() is deprecated and unused; it will be removed in a future release.", + call. = FALSE + ) + } + # Add dummy columns for missing columns BEFORE creating the agent # These columns are set to FALSE and we'll check they equal TRUE (will fail) for (c in missing_in_candidate) { @@ -66,6 +91,25 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, } } + # Derive the missing __eq booleans BEFORE creating the agent, so every + # equality step validates the same boolean the verdict logic uses (shared NA + # semantics) instead of embedding the whole reference vector in the step + # (O(n_rows) per step, serialised with the report, and na_pass cannot + # express the one-sided/two-sided NA distinction). + eq_cols <- setdiff(x = common_cols, y = tol_cols) + if (!is_non_local(cmp)) { + for (c in eq_cols) { + eq_col <- datadiff_eq_col(c) + if (!(eq_col %in% get_col_names(cmp))) { + cmp[[eq_col]] <- eq_col_bool( + cmp, col = c, ref_suffix = ref_suffix, na_equal = na_equal + ) + } + } + } + # Hoisted once, AFTER every cmp mutation above + col_names <- get_col_names(cmp) + agent <- create_agent(tbl = cmp, label = label, actions = action_levels(warn_at = warn_at, stop_at = stop_at), lang = lang, @@ -105,22 +149,19 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, ) } - eq_cols <- setdiff(x = common_cols, y = tol_cols) for (c in eq_cols) { - eq_precomputed <- datadiff_eq_col(c) - if (eq_precomputed %in% get_col_names(cmp)) { - # Lazy path: use pre-computed boolean equality column - agent <- agent %>% - col_vals_equal(columns = all_of(eq_precomputed), value = TRUE, na_pass = FALSE) - } else { - # Local path: compare directly against reference column values - agent <- agent %>% - col_vals_equal( - columns = all_of(c), - value = cmp[[paste0(c, ref_suffix)]], - na_pass = na_equal - ) + eq_col <- datadiff_eq_col(c) + if (!(eq_col %in% col_names)) { + # Only reachable with a lazy cmp missing its precomputed boolean: + # the documented contract requires it (deriving it here would need + # the raw columns, absent from the slim lazy table) + stop(sprintf( + "equality boolean column '%s' missing from cmp (a lazy cmp must carry the precomputed __eq columns)", + eq_col + ), call. = FALSE) } + agent <- agent %>% + col_vals_equal(columns = all_of(eq_col), value = TRUE, na_pass = FALSE) } for (c in tol_cols) { @@ -130,7 +171,7 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, # Add row count validation if needed # Use colnames() instead of names() to work correctly with lazy tables - if (!is.null(row_validation_info) && isTRUE(row_validation_info$check_count) && "row_count_ok" %in% get_col_names(cmp)) { + if (!is.null(row_validation_info) && isTRUE(row_validation_info$check_count) && "row_count_ok" %in% col_names) { agent <- agent %>% col_vals_equal(columns = all_of("row_count_ok"), value = TRUE, na_pass = FALSE) } diff --git a/man/setup_pointblank_agent.Rd b/man/setup_pointblank_agent.Rd index 0f22844..7ef1bc3 100644 --- a/man/setup_pointblank_agent.Rd +++ b/man/setup_pointblank_agent.Rd @@ -6,7 +6,7 @@ \usage{ setup_pointblank_agent( cmp, - cols_reference, + cols_reference = NULL, common_cols, tol_cols, row_validation_info = NULL, @@ -25,9 +25,12 @@ setup_pointblank_agent( \arguments{ \item{cmp}{Comparison dataframe with candidate and reference data} -\item{cols_reference}{Character vector of reference column names} +\item{cols_reference}{Deprecated and unused; supplying any non-NULL value +raises a warning (an explicit NULL is silent) and the argument will be +removed in a future release} -\item{common_cols}{Character vector of columns present in both datasets} +\item{common_cols}{Character vector of equality columns to validate (the +internal pipeline passes only the failing ones on its failure path)} \item{tol_cols}{Character vector of columns with tolerance rules} @@ -69,10 +72,26 @@ Configured pointblank agent ready for interrogation Creates and configures a pointblank validation agent with all necessary validation steps including column existence checks, exact value comparisons, and tolerance validations. } +\details{ +The equality steps validate a \verb{__eq} boolean (one-sided NA fails, +two-sided NA follows \code{na_equal}, matching the tolerance kernel and the lazy +SQL): when \code{cmp} does not already carry it, it is derived on the fly for a +local data.frame. Tolerance steps expect the \verb{__ok} booleans to be +precomputed (see \code{\link[=add_tolerance_columns]{add_tolerance_columns()}}). A lazy \code{cmp} must carry every +boolean column already. +} \examples{ -cmp <- data.frame(a = 1:3, a__reference = 1:3, b = c(1.1, 2.2, 3.3), - b__reference = c(1.0, 2.0, 3.0)) +cmp <- data.frame( + a = 1:3, a__reference = 1:3, + b = c(1.1, 2.2, 3.3), b__reference = c(1.0, 2.0, 3.0), + b__ok = c(FALSE, FALSE, FALSE) +) row_info <- list(check_count = FALSE) -setup_pointblank_agent(cmp, c("a", "b"), c("a", "b"), "b", row_info, - "__reference", 0.1, 0.1, "Test", TRUE) +agent <- setup_pointblank_agent( + cmp, + common_cols = "a", tol_cols = "b", row_validation_info = row_info, + ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, + label = "Test", na_equal = TRUE +) +pointblank::all_passed(pointblank::interrogate(agent)) } diff --git a/tests/testthat/test-pointblank-setup.R b/tests/testthat/test-pointblank-setup.R index 3b28d8b..ec7611b 100644 --- a/tests/testthat/test-pointblank-setup.R +++ b/tests/testthat/test-pointblank-setup.R @@ -12,14 +12,14 @@ test_that("setup_pointblank_agent creates valid agent", { text__reference = c("a", "b", "c") ) - cols_reference <- c("id", "value", "text") common_cols <- c("id", "value", "text") tol_cols <- c("value") rules <- list() ref_suffix <- "__reference" # Test agent creation - agent <- setup_pointblank_agent(cmp, cols_reference, common_cols, tol_cols, rules, ref_suffix, + agent <- setup_pointblank_agent(cmp, common_cols = common_cols, tol_cols = tol_cols, + row_validation_info = rules, ref_suffix = ref_suffix, warn_at = 0.1, stop_at = 0.1, label = "Test", na_equal = TRUE) # Check that agent was created @@ -40,13 +40,13 @@ test_that("setup_pointblank_agent handles different column types", { text__reference = c("a", "b") ) - cols_reference <- c("id", "text") common_cols <- c("id", "text") tol_cols <- character(0) # No tolerance columns rules <- list() ref_suffix <- "__reference" - agent <- setup_pointblank_agent(cmp, cols_reference, common_cols, tol_cols, rules, ref_suffix, + agent <- setup_pointblank_agent(cmp, common_cols = common_cols, tol_cols = tol_cols, + row_validation_info = rules, ref_suffix = ref_suffix, warn_at = 0.5, stop_at = 0.5, label = "Test Exact", na_equal = FALSE) expect_s3_class(agent, "ptblank_agent") @@ -57,7 +57,8 @@ test_that("setup_pointblank_agent handles empty inputs", { # Test with minimal data cmp <- data.frame(id = 1, id__reference = 1) - agent <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 1.0, stop_at = 1.0, label = "Minimal", na_equal = TRUE) expect_s3_class(agent, "ptblank_agent") @@ -67,7 +68,8 @@ test_that("setup_pointblank_agent handles empty inputs", { test_that("setup_pointblank_agent uses French language by default", { cmp <- data.frame(id = 1, id__reference = 1) - agent <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test", na_equal = TRUE) expect_s3_class(agent, "ptblank_agent") @@ -79,7 +81,8 @@ test_that("setup_pointblank_agent accepts custom language parameters", { cmp <- data.frame(id = 1, id__reference = 1) # Test English - agent_en <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_en <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test EN", na_equal = TRUE, lang = "en", locale = "en_US") @@ -88,7 +91,8 @@ test_that("setup_pointblank_agent accepts custom language parameters", { expect_equal(agent_en$locale, "en_US") # Test German - agent_de <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_de <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test DE", na_equal = TRUE, lang = "de", locale = "de_DE") @@ -101,21 +105,24 @@ test_that("setup_pointblank_agent accepts various supported languages", { cmp <- data.frame(id = 1, id__reference = 1) # Test Spanish - agent_es <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_es <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test ES", na_equal = TRUE, lang = "es", locale = "es_ES") expect_equal(agent_es$lang, "es") expect_equal(agent_es$locale, "es_ES") # Test Portuguese - agent_pt <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_pt <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test PT", na_equal = TRUE, lang = "pt", locale = "pt_BR") expect_equal(agent_pt$lang, "pt") expect_equal(agent_pt$locale, "pt_BR") # Test Italian - agent_it <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_it <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test IT", na_equal = TRUE, lang = "it", locale = "it_IT") expect_equal(agent_it$lang, "it") @@ -135,7 +142,6 @@ test_that("setup_pointblank_agent handles lazy table with missing_in_candidate a # Passing a lazy table exercises the dplyr::mutate() path (lines 52 and 63) agent <- setup_pointblank_agent( lazy_tbl, - cols_reference = character(0), common_cols = character(0), tol_cols = c("val"), row_validation_info = list(check_count = FALSE), @@ -150,3 +156,46 @@ test_that("setup_pointblank_agent handles lazy table with missing_in_candidate a expect_s3_class(agent, "ptblank_agent") }) + +test_that("cols_reference is deprecated and unused", { + cmp <- data.frame(a = 1:3, a__reference = 1:3) + row_info <- list(check_count = FALSE) + expect_warning( + setup_pointblank_agent(cmp, cols_reference = c("a"), common_cols = "a", + tol_cols = character(0), row_validation_info = row_info, + ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, + label = "x", na_equal = TRUE), + regexp = "deprecated" + ) + # Omitting it works, produces the same agent shape and stays silent + expect_no_warning( + agent <- setup_pointblank_agent(cmp, common_cols = "a", + tol_cols = character(0), row_validation_info = row_info, + ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, + label = "x", na_equal = TRUE) + ) + expect_s3_class(agent, "ptblank_agent") +}) + +test_that("direct equality steps use the shared NA semantics via a derived boolean", { + # One-sided NA with na_equal = TRUE: must FAIL (kernel/lazy semantics), + # not pass through col_vals_equal's na_pass + cmp <- data.frame( + s = c("a", NA, "c"), + s__reference = c("a", "b", "c"), + stringsAsFactors = FALSE + ) + row_info <- list(check_count = FALSE) + agent <- setup_pointblank_agent(cmp, common_cols = "s", + tol_cols = character(0), row_validation_info = row_info, + ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, + label = "na semantics", na_equal = TRUE) + reponse <- pointblank::interrogate(agent) + expect_false(pointblank::all_passed(reponse)) + + # The step no longer embeds the reference VECTOR in the agent: the equality + # is validated through a derived __eq boolean column + vs <- reponse$validation_set + step_cols <- vapply(vs$column, function(cc) cc[1], character(1)) + expect_true("s__eq" %in% step_cols) +})