From b853baacdca52228bb06ede5afac879c991c70ce Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 17:42:16 +0200 Subject: [PATCH 01/40] fix: erreur explicite quand une colonne cle est absente d'un dataset (issue #16) Le retour anticipe silencieux (liste tronquee a 6 champs sans coverage ni summary) est remplace par un stop() qui nomme les colonnes cles manquantes et le ou les datasets concernes. Le test qui verrouillait l'ancien comportement est remplace par des expect_error/expect_match couvrant : cle absente des deux cotes, cote candidat seul, sans YAML explicite, et cle multi-colonnes partiellement absente. --- DESCRIPTION | 2 +- NEWS.md | 11 ++++++ R/compare_datasets_from_yaml.R | 33 +++++++++++------ man/compare_datasets_from_yaml.Rd | 4 ++- tests/testthat/test-key-parameter.R | 55 +++++++++++++++++++++++------ 5 files changed, 82 insertions(+), 23 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 9447044..f8fa1f1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: datadiff Title: Data Validation Based on 'YAML' Rules -Version: 0.4.9 +Version: 0.4.9.9000 Authors@R: c( person("Vincent", "Guyader", , "vincent@thinkr.fr", role = c("cre", "aut"), comment = c(ORCID = "0000-0003-0671-9270")), diff --git a/NEWS.md b/NEWS.md index b36e3b8..86b698c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,14 @@ +# datadiff (development version) + +## Breaking changes + +* `compare_datasets_from_yaml()` now raises an explicit error when one or more + key columns are absent from either dataset, naming the missing column(s) and + the dataset(s) concerned. It previously emitted a vague message + ("could not find key in both data") and returned a truncated 6-field list + (no `coverage`, no `summary`) that broke downstream consumers such as + `datadiff_report_html()` (issue #16). + # datadiff 0.4.9 ## Bug fixes diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 65c9f1b..4d83a2f 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -119,7 +119,9 @@ read_rules <- function(path) { #' #' @param data_reference Reference dataframe, tibble, or lazy table (tbl_lazy) #' @param data_candidate Candidate dataframe to validate against reference -#' @param key Optional character vector of column names to use as join keys for ordered comparison +#' @param key Optional character vector of column names to use as join keys for +#' ordered comparison. Every key column must exist in both datasets; otherwise +#' an error is raised naming the missing column(s) and the dataset(s) concerned. #' @param path Path to YAML file containing validation rules. If NULL, default rules are #' generated automatically based on the reference dataset structure. #' @param warn_at Warning threshold as fraction of failing tests (default: 1e-14) @@ -403,18 +405,27 @@ compare_datasets_from_yaml <- function(data_reference, row_validation_info <- validate_row_counts(data_reference_p, data_candidate_p, rules) if (!is.null(key)) { - if (isFALSE(all(key %in% get_col_names(data_reference_p))) | isFALSE(all(key %in% get_col_names(data_candidate_p)))) { - message("could not find key in both data") - return( - list( - all_passed = FALSE, - agent = NULL, - reponse = NULL, - missing_in_candidate = NULL, - extra_in_candidate = NULL, - applied_rules = NULL + missing_key_ref <- setdiff(key, get_col_names(data_reference_p)) + missing_key_cand <- setdiff(key, get_col_names(data_candidate_p)) + if (length(missing_key_ref) > 0 || length(missing_key_cand) > 0) { + describe_missing <- function(cols, dataset_name) { + if (length(cols) == 0) { + return(NULL) + } + sprintf( + "%s missing in %s", + paste(sprintf("'%s'", cols), collapse = ", "), + dataset_name ) + } + details <- c( + describe_missing(missing_key_ref, dataset_name = "data_reference"), + describe_missing(missing_key_cand, dataset_name = "data_candidate") ) + stop(sprintf( + "Key column(s) not found: %s.", + paste(details, collapse = "; ") + ), call. = FALSE) } # Join candidate to reference on key to handle different row counts diff --git a/man/compare_datasets_from_yaml.Rd b/man/compare_datasets_from_yaml.Rd index 71bc074..6fc85ab 100644 --- a/man/compare_datasets_from_yaml.Rd +++ b/man/compare_datasets_from_yaml.Rd @@ -29,7 +29,9 @@ compare_datasets_from_yaml( \item{data_candidate}{Candidate dataframe to validate against reference} -\item{key}{Optional character vector of column names to use as join keys for ordered comparison} +\item{key}{Optional character vector of column names to use as join keys for +ordered comparison. Every key column must exist in both datasets; otherwise +an error is raised naming the missing column(s) and the dataset(s) concerned.} \item{path}{Path to YAML file containing validation rules. If NULL, default rules are generated automatically based on the reference dataset structure.} diff --git a/tests/testthat/test-key-parameter.R b/tests/testthat/test-key-parameter.R index 346b5a7..d511428 100644 --- a/tests/testthat/test-key-parameter.R +++ b/tests/testthat/test-key-parameter.R @@ -72,7 +72,7 @@ by_type: unlink('test_key_null.yaml') }) -test_that("compare_datasets_from_yaml key parameter validation works", { +test_that("compare_datasets_from_yaml errors when key columns are absent", { ref <- data.frame(id = 1:3, value = c(10.0, 20.0, 30.0)) cand <- data.frame(id = 1:3, value = c(10.1, 20.1, 30.1)) @@ -85,17 +85,52 @@ by_type: abs: 0.5 ' - writeLines(yaml_content, 'test_key_validation.yaml') + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(yaml_content, con = yaml_path) - # Test: Invalid key should return an empty output with all_passed FALSE - - res <- compare_datasets_from_yaml(ref, cand, key = "nonexistent", path = 'test_key_validation.yaml') + # Key absent from both datasets: explicit error naming both datasets + err_both <- tryCatch( + compare_datasets_from_yaml(ref, cand, key = "nonexistent", path = yaml_path), + error = function(e) { + conditionMessage(e) + } + ) + expect_match(err_both, "Key column\\(s\\) not found") + expect_match(err_both, "'nonexistent'", fixed = TRUE) + expect_match(err_both, "data_reference", fixed = TRUE) + expect_match(err_both, "data_candidate", fixed = TRUE) + + # Key present in the reference but absent from the candidate: + # only the candidate is named + cand_no_id <- data.frame(idx = 1:3, value = c(10.1, 20.1, 30.1)) + err_cand <- tryCatch( + compare_datasets_from_yaml(ref, cand_no_id, key = "id", path = yaml_path), + error = function(e) { + conditionMessage(e) + } + ) + expect_match(err_cand, "'id'", fixed = TRUE) + expect_match(err_cand, "data_candidate", fixed = TRUE) + expect_no_match(err_cand, "data_reference", fixed = TRUE) + + # Same scenario without an explicit YAML path (auto-generated template) + expect_error( + compare_datasets_from_yaml(ref, cand_no_id, key = "id"), + regexp = "Key column\\(s\\) not found" + ) - expect_false(res$all_passed) - expect_null(res$agent) - expect_null(res$reponse) - # Clean up - unlink('test_key_validation.yaml') + # Multi-column key with a single absent column: only that column is reported + ref_multi <- data.frame(id = 1:3, grp = 1:3, value = 1:3) + cand_multi <- data.frame(id = 1:3, value = 1:3) + err_multi <- tryCatch( + compare_datasets_from_yaml(ref_multi, cand_multi, key = c("id", "grp"), path = yaml_path), + error = function(e) { + conditionMessage(e) + } + ) + expect_match(err_multi, "'grp'", fixed = TRUE) + expect_no_match(err_multi, "'id'", fixed = TRUE) }) test_that("key parameter takes precedence over YAML in edge cases", { From cd96477b3447fb314c63364bb099199d36db3751 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 18:09:17 +0200 Subject: [PATCH 02/40] fix: validation canonique de la cle avant la branche template auto Reponse a la review Copilot de la PR : - la cle passee en argument est validee contre les DEUX datasets avant la branche path = NULL, qui atteignait write_rules_template() en premier et levait une erreur ne validant que la reference, dans un format different - une cle vide (character(0)) ou non-character est rejetee proprement au lieu de tomber dans un cross join dplyr deprecie - la validation est factorisee dans validate_comparison_key(), reutilisee par le check tardif qui couvre les cles issues du YAML --- NEWS.md | 8 ++- R/compare_datasets_from_yaml.R | 84 +++++++++++++++++++++-------- tests/testthat/test-key-parameter.R | 42 ++++++++++++++- 3 files changed, 110 insertions(+), 24 deletions(-) diff --git a/NEWS.md b/NEWS.md index 86b698c..7601979 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,11 +4,17 @@ * `compare_datasets_from_yaml()` now raises an explicit error when one or more key columns are absent from either dataset, naming the missing column(s) and - the dataset(s) concerned. It previously emitted a vague message + the dataset(s) concerned, on every code path (with or without an explicit + YAML rules file). It previously emitted a vague message ("could not find key in both data") and returned a truncated 6-field list (no `coverage`, no `summary`) that broke downstream consumers such as `datadiff_report_html()` (issue #16). +* `compare_datasets_from_yaml()` rejects an empty (`character(0)`) or + non-character `key` with a clear error. An empty key previously fell through + to a keyless cross join (deprecated dplyr behavior producing a cartesian + product). + # datadiff 0.4.9 ## Bug fixes diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 4d83a2f..ee74ba9 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -111,6 +111,49 @@ read_rules <- function(path) { r } +#' Validate a comparison key against both datasets +#' +#' Shared guard for every code path that consumes a key: checks the type and +#' emptiness of the key, then its presence in both datasets, and raises an +#' explicit error naming the missing column(s) and the dataset(s) concerned. +#' +#' @param key Character vector of key column names. +#' @param ref_cols Column names of the reference dataset. +#' @param cand_cols Column names of the candidate dataset. +#' @return \code{NULL}, invisibly. Called for its side effect (error). +#' @noRd +validate_comparison_key <- function(key, ref_cols, cand_cols) { + if (!is.character(key) || length(key) == 0) { + stop( + "Parameter 'key' must be a non-empty character vector specifying column name(s) to use as join key(s).", + call. = FALSE + ) + } + missing_key_ref <- setdiff(key, ref_cols) + missing_key_cand <- setdiff(key, cand_cols) + if (length(missing_key_ref) > 0 || length(missing_key_cand) > 0) { + describe_missing <- function(cols, dataset_name) { + if (length(cols) == 0) { + return(NULL) + } + sprintf( + "%s missing in %s", + paste(sprintf("'%s'", cols), collapse = ", "), + dataset_name + ) + } + details <- c( + describe_missing(missing_key_ref, dataset_name = "data_reference"), + describe_missing(missing_key_cand, dataset_name = "data_candidate") + ) + stop(sprintf( + "Key column(s) not found: %s.", + paste(details, collapse = "; ") + ), call. = FALSE) + } + invisible(NULL) +} + #' Compare datasets using YAML validation rules #' #' Main function for comparing reference and candidate datasets using configurable @@ -266,6 +309,18 @@ compare_datasets_from_yaml <- function(data_reference, data_candidate <- arrow_dataset_to_duckdb(data_candidate, fresh_con, "datadiff_cand") } + # Validate the key argument against BOTH datasets before any other consumer: + # the auto-generated-template path (path = NULL) would otherwise reach + # write_rules_template() first, which validates the reference only and with + # a less specific error. + if (!is.null(key)) { + validate_comparison_key( + key, + ref_cols = get_col_names(data_reference), + cand_cols = get_col_names(data_candidate) + ) + } + # If no path provided, create a temporary YAML with default rules if (is.null(path)) { path <- tempfile(fileext = ".yaml") @@ -405,28 +460,13 @@ compare_datasets_from_yaml <- function(data_reference, row_validation_info <- validate_row_counts(data_reference_p, data_candidate_p, rules) if (!is.null(key)) { - missing_key_ref <- setdiff(key, get_col_names(data_reference_p)) - missing_key_cand <- setdiff(key, get_col_names(data_candidate_p)) - if (length(missing_key_ref) > 0 || length(missing_key_cand) > 0) { - describe_missing <- function(cols, dataset_name) { - if (length(cols) == 0) { - return(NULL) - } - sprintf( - "%s missing in %s", - paste(sprintf("'%s'", cols), collapse = ", "), - dataset_name - ) - } - details <- c( - describe_missing(missing_key_ref, dataset_name = "data_reference"), - describe_missing(missing_key_cand, dataset_name = "data_candidate") - ) - stop(sprintf( - "Key column(s) not found: %s.", - paste(details, collapse = "; ") - ), call. = FALSE) - } + # Re-validated here because the key may come from the YAML rules + # (defaults$keys), which the argument-level check upstream cannot see. + validate_comparison_key( + key, + ref_cols = get_col_names(data_reference_p), + cand_cols = get_col_names(data_candidate_p) + ) # Join candidate to reference on key to handle different row counts cmp <- left_join(data_candidate_p, data_reference_p, by = key, suffix = c("", ref_suffix)) diff --git a/tests/testthat/test-key-parameter.R b/tests/testthat/test-key-parameter.R index d511428..86fb923 100644 --- a/tests/testthat/test-key-parameter.R +++ b/tests/testthat/test-key-parameter.R @@ -117,9 +117,24 @@ by_type: # Same scenario without an explicit YAML path (auto-generated template) expect_error( compare_datasets_from_yaml(ref, cand_no_id, key = "id"), - regexp = "Key column\\(s\\) not found" + regexp = "missing in data_candidate" ) + # Without an explicit YAML path, a key absent from the *reference* must get + # the same explicit error format (not the reference-only error of + # write_rules_template on the auto-generated-template path) + ref_no_id <- data.frame(idx = 1:3, value = c(10.0, 20.0, 30.0)) + cand_id <- data.frame(id = 1:3, value = c(10.1, 20.1, 30.1)) + err_ref <- tryCatch( + compare_datasets_from_yaml(ref_no_id, cand_id, key = "id"), + error = function(e) { + conditionMessage(e) + } + ) + expect_match(err_ref, "'id'", fixed = TRUE) + expect_match(err_ref, "missing in data_reference", fixed = TRUE) + expect_no_match(err_ref, "data_candidate", fixed = TRUE) + # Multi-column key with a single absent column: only that column is reported ref_multi <- data.frame(id = 1:3, grp = 1:3, value = 1:3) cand_multi <- data.frame(id = 1:3, value = 1:3) @@ -133,6 +148,31 @@ by_type: expect_no_match(err_multi, "'id'", fixed = TRUE) }) +test_that("compare_datasets_from_yaml rejects empty or non-character keys", { + ref <- data.frame(id = 1:3, value = c(10.0, 20.0, 30.0)) + cand <- data.frame(id = 1:3, value = c(10.1, 20.1, 30.1)) + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + write_rules_template(ref, key = "id", path = yaml_path) + + # An empty key must not silently fall through to a keyless cross join + expect_error( + compare_datasets_from_yaml(ref, cand, key = character(0), path = yaml_path), + regexp = "non-empty character vector" + ) + expect_error( + compare_datasets_from_yaml(ref, cand, key = character(0)), + regexp = "non-empty character vector" + ) + + # A non-character key gets a clear type error, not a missing-column error + expect_error( + compare_datasets_from_yaml(ref, cand, key = 1L, path = yaml_path), + regexp = "non-empty character vector" + ) +}) + test_that("key parameter takes precedence over YAML in edge cases", { # Create data where YAML key would cause issues but parameter key works ref <- data.frame( From 4f4b2de18ac1cb3fdccfa60cde023010d119d019 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 17:52:27 +0200 Subject: [PATCH 03/40] fix: erreur franche sur comparaison positionnelle a effectifs inegaux (issue #15) Sans cle et a effectifs differents, error_msg_no_key n'alimentait qu'un message() informatif juste avant un crash R opaque sur l'affectation des colonnes de reference. L'erreur porte desormais le texte de error_msg_no_key complete des deux effectifs. Sur le chemin positionnel valide, les colonnes de reference sont ajoutees en un bind unique au lieu d'une boucle d'affectation par colonne. --- NEWS.md | 10 +++++++ R/compare_datasets_from_yaml.R | 17 +++++++++--- man/compare_datasets_from_yaml.Rd | 4 ++- tests/testthat/test-edge-cases.R | 43 +++++++++++++------------------ 4 files changed, 44 insertions(+), 30 deletions(-) diff --git a/NEWS.md b/NEWS.md index 7601979..3573cfa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,15 @@ # datadiff (development version) +## Bug fixes + +* A positional (key-less) comparison of datasets with different row counts now + raises a single clear error built from `error_msg_no_key` and mentioning both + row counts. It previously emitted `error_msg_no_key` as an informational + message and then crashed on an opaque internal column assignment + ("replacement has X rows, data has Y"). On the valid positional path, the + reference columns are now appended with a single bind instead of a + per-column assignment loop (issue #15). + ## Breaking changes * `compare_datasets_from_yaml()` now raises an explicit error when one or more diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index ee74ba9..4ff70a2 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -171,7 +171,9 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' @param stop_at Stop threshold as fraction of failing tests (default: 1e-14) #' @param ref_suffix Suffix for reference columns in comparison dataframe (default: "__reference") #' @param label Descriptive label for the validation report -#' @param error_msg_no_key Error message when datasets have different row counts without keys +#' @param error_msg_no_key Text of the error raised when a positional (key-less) +#' comparison receives datasets with different row counts; the actual row +#' counts of both datasets are appended to this text. #' @param lang Language code for pointblank reports. Defaults to the #' \code{datadiff.lang} option if set, otherwise \code{"fr"}. Override globally #' with \code{options(datadiff.lang = "en")}. Supported values include @@ -479,11 +481,18 @@ compare_datasets_from_yaml <- function(data_reference, } # Use pre-computed counts from validate_row_counts (avoids a second nrow() on lazy) if (row_validation_info$ref_count != row_validation_info$cand_count) { - message(error_msg_no_key) + stop(sprintf( + "%s (data_reference: %s rows, data_candidate: %s rows).", + error_msg_no_key, + row_validation_info$ref_count, + row_validation_info$cand_count + ), call. = FALSE) } cmp <- data_candidate_p - for (c in common_cols) { - cmp[[paste0(c, ref_suffix)]] <- data_reference_p[[c]] + if (length(common_cols) > 0) { + ref_block <- data_reference_p[, common_cols, drop = FALSE] + names(ref_block) <- paste0(common_cols, ref_suffix) + cmp <- dplyr::bind_cols(cmp, ref_block) } } diff --git a/man/compare_datasets_from_yaml.Rd b/man/compare_datasets_from_yaml.Rd index 6fc85ab..c3998d8 100644 --- a/man/compare_datasets_from_yaml.Rd +++ b/man/compare_datasets_from_yaml.Rd @@ -44,7 +44,9 @@ generated automatically based on the reference dataset structure.} \item{label}{Descriptive label for the validation report} -\item{error_msg_no_key}{Error message when datasets have different row counts without keys} +\item{error_msg_no_key}{Text of the error raised when a positional (key-less) +comparison receives datasets with different row counts; the actual row +counts of both datasets are appended to this text.} \item{lang}{Language code for pointblank reports. Defaults to the \code{datadiff.lang} option if set, otherwise \code{"fr"}. Override globally diff --git a/tests/testthat/test-edge-cases.R b/tests/testthat/test-edge-cases.R index b2735f4..f4c8101 100644 --- a/tests/testthat/test-edge-cases.R +++ b/tests/testthat/test-edge-cases.R @@ -791,34 +791,27 @@ test_that("no crash and no spurious warning when all column types match", { unlink(template_path) }) -test_that("message is shown for positional comparison when row counts differ", { +test_that("positional comparison with unequal row counts raises a clear error", { ref <- data.frame(value = 1:5) cand <- data.frame(value = 1:3) - # No key -> positional comparison; row counts differ -> message emitted then - # R errors on column assignment. Capture every message before the error. - capture_messages <- function(expr) { - msgs <- character(0) - tryCatch( - withCallingHandlers( - suppressWarnings(expr), - message = function(m) { - msgs <<- c(msgs, conditionMessage(m)) - invokeRestart("muffleMessage") - } - ), - error = function(e) NULL - ) - msgs - } - - # Default: the row-count message uses error_msg_no_key's default text. - msgs <- capture_messages(compare_datasets_from_yaml(ref, cand)) - expect_true(any(grepl("same number of rows", msgs))) + # No key -> positional comparison; row counts differ -> a single clear error + # built from error_msg_no_key, mentioning both row counts. + err <- tryCatch( + suppressMessages(compare_datasets_from_yaml(ref, cand)), + error = function(e) { + conditionMessage(e) + } + ) + expect_match(err, "same number of rows", fixed = TRUE) + expect_match(err, "data_reference: 5 rows", fixed = TRUE) + expect_match(err, "data_candidate: 3 rows", fixed = TRUE) - # A custom error_msg_no_key must actually reach the emitted message. - msgs_custom <- capture_messages( - compare_datasets_from_yaml(ref, cand, error_msg_no_key = "CUSTOM_NO_KEY_MSG") + # A custom error_msg_no_key must be the text of the raised error. + expect_error( + suppressMessages( + compare_datasets_from_yaml(ref, cand, error_msg_no_key = "CUSTOM_NO_KEY_MSG") + ), + regexp = "CUSTOM_NO_KEY_MSG" ) - expect_true(any(grepl("CUSTOM_NO_KEY_MSG", msgs_custom))) }) From a445cb060e10baa3c151c48236702441111ef161 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 18:38:38 +0200 Subject: [PATCH 04/40] fix: abort avant collect et garde non-local sur les deux cotes (chemin positionnel) Reponse a la review Copilot de la PR : - le stop() effectifs inegaux est remonte avant le collect des tables non-locales (les comptes sont deja connus) : plus de materialisation inutile ni de note collecting trompeuse juste avant l'erreur - la garde de collect ne testait que data_reference_p : une comparaison positionnelle melant reference locale et candidat lazy sautait la collecte du candidat et echouait en aval (bug preexistant revele par la review) - le test positionnel capture desormais les messages et verifie que le texte de l'erreur ne fuit pas sur le flux message --- NEWS.md | 8 ++- R/compare_datasets_from_yaml.R | 21 +++++--- tests/testthat/test-edge-cases.R | 83 ++++++++++++++++++++++++++------ 3 files changed, 89 insertions(+), 23 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3573cfa..f236e49 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,8 +6,12 @@ raises a single clear error built from `error_msg_no_key` and mentioning both row counts. It previously emitted `error_msg_no_key` as an informational message and then crashed on an opaque internal column assignment - ("replacement has X rows, data has Y"). On the valid positional path, the - reference columns are now appended with a single bind instead of a + ("replacement has X rows, data has Y"). The error is raised before any + materialisation of non-local tables (the counts are already known), and the + collect guard now covers both sides: a positional comparison mixing a local + reference with a lazy candidate (or vice versa) previously skipped the + candidate collection and failed downstream. On the valid positional path, + the reference columns are now appended with a single bind instead of a per-column assignment loop (issue #15). ## Breaking changes diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 4ff70a2..270d021 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -473,13 +473,9 @@ compare_datasets_from_yaml <- function(data_reference, # Join candidate to reference on key to handle different row counts cmp <- left_join(data_candidate_p, data_reference_p, by = key, suffix = c("", ref_suffix)) } else { - # For non-keyed comparison, collect non-local tables (positional join requires local data) - if (is_non_local(data_reference_p)) { - message("Note: positional comparison requires collecting non-local tables into memory.") - data_reference_p <- dplyr::collect(data_reference_p) - data_candidate_p <- dplyr::collect(data_candidate_p) - } - # Use pre-computed counts from validate_row_counts (avoids a second nrow() on lazy) + # Row-count mismatch is decidable from the counts precomputed by + # validate_row_counts(): abort before the potentially expensive collect + # of non-local tables below. if (row_validation_info$ref_count != row_validation_info$cand_count) { stop(sprintf( "%s (data_reference: %s rows, data_candidate: %s rows).", @@ -488,6 +484,17 @@ compare_datasets_from_yaml <- function(data_reference, row_validation_info$cand_count ), call. = FALSE) } + # For non-keyed comparison, collect non-local tables on BOTH sides + # (positional binding requires local data on each side). + if (is_non_local(data_reference_p) || is_non_local(data_candidate_p)) { + message("Note: positional comparison requires collecting non-local tables into memory.") + if (is_non_local(data_reference_p)) { + data_reference_p <- dplyr::collect(data_reference_p) + } + if (is_non_local(data_candidate_p)) { + data_candidate_p <- dplyr::collect(data_candidate_p) + } + } cmp <- data_candidate_p if (length(common_cols) > 0) { ref_block <- data_reference_p[, common_cols, drop = FALSE] diff --git a/tests/testthat/test-edge-cases.R b/tests/testthat/test-edge-cases.R index f4c8101..cc984eb 100644 --- a/tests/testthat/test-edge-cases.R +++ b/tests/testthat/test-edge-cases.R @@ -791,27 +791,82 @@ test_that("no crash and no spurious warning when all column types match", { unlink(template_path) }) +# Helper for the positional-path tests: run expr, return the error message +# and every message emitted (the "key is missing" design note is expected; +# the error text must never leak on the message stream). +catch_error_and_messages <- function(expr) { + msgs <- character(0) + err <- tryCatch( + withCallingHandlers( + expr, + message = function(m) { + msgs <<- c(msgs, conditionMessage(m)) + invokeRestart("muffleMessage") + } + ), + error = function(e) { + conditionMessage(e) + } + ) + list(error = err, messages = msgs) +} + test_that("positional comparison with unequal row counts raises a clear error", { ref <- data.frame(value = 1:5) cand <- data.frame(value = 1:3) # No key -> positional comparison; row counts differ -> a single clear error # built from error_msg_no_key, mentioning both row counts. - err <- tryCatch( - suppressMessages(compare_datasets_from_yaml(ref, cand)), - error = function(e) { - conditionMessage(e) - } - ) - expect_match(err, "same number of rows", fixed = TRUE) - expect_match(err, "data_reference: 5 rows", fixed = TRUE) - expect_match(err, "data_candidate: 3 rows", fixed = TRUE) + out <- catch_error_and_messages(compare_datasets_from_yaml(ref, cand)) + expect_match(out$error, "same number of rows", fixed = TRUE) + expect_match(out$error, "data_reference: 5 rows", fixed = TRUE) + expect_match(out$error, "data_candidate: 3 rows", fixed = TRUE) + # The error text must be raised, not emitted as an informational message + expect_false(any(grepl("same number of rows", out$messages, fixed = TRUE))) # A custom error_msg_no_key must be the text of the raised error. - expect_error( - suppressMessages( - compare_datasets_from_yaml(ref, cand, error_msg_no_key = "CUSTOM_NO_KEY_MSG") - ), - regexp = "CUSTOM_NO_KEY_MSG" + out_custom <- catch_error_and_messages( + compare_datasets_from_yaml(ref, cand, error_msg_no_key = "CUSTOM_NO_KEY_MSG") + ) + expect_match(out_custom$error, "CUSTOM_NO_KEY_MSG", fixed = TRUE) + expect_false(any(grepl("CUSTOM_NO_KEY_MSG", out_custom$messages, fixed = TRUE))) +}) + +test_that("unequal-count positional comparison on lazy tables aborts before collecting", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + duckdb::dbWriteTable(con, "ref5", data.frame(value = 1:5)) + duckdb::dbWriteTable(con, "cand3", data.frame(value = 1:3)) + + out <- catch_error_and_messages( + compare_datasets_from_yaml(dplyr::tbl(con, "ref5"), dplyr::tbl(con, "cand3")) + ) + expect_match(out$error, "same number of rows", fixed = TRUE) + # The mismatch is decidable from the precomputed counts: no table + # materialisation (and no misleading collecting note) before the stop + expect_false(any(grepl("collecting non-local tables", out$messages, fixed = TRUE))) +}) + +test_that("mixed local/lazy inputs work on the positional path", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + df <- data.frame(value = c(1.0, 2.0, 3.0)) + duckdb::dbWriteTable(con, "t3", df) + + # Local reference + lazy candidate: the candidate must be collected too + res <- suppressMessages( + compare_datasets_from_yaml(df, dplyr::tbl(con, "t3")) + ) + expect_true(res$all_passed) + + # Lazy reference + local candidate + res2 <- suppressMessages( + compare_datasets_from_yaml(dplyr::tbl(con, "t3"), df) ) + expect_true(res2$all_passed) }) From c25bcd1505999329926ac22f8ede3333b25c602e Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 18:15:46 +0200 Subject: [PATCH 05/40] fix: precedence deterministe argument > YAML > defaut pour key et label (issue #20) - label explicite honore meme quand path est fourni (etait ecrase inconditionnellement par le YAML) - champ keys lu explicitement via [[ au lieu du partial matching $ sur key ; fallback retrocompatible sur un champ key singulier ; keys gagne quand les deux sont presents - cles issues du YAML coercees en character (unlist) avant validation - table de precedence dans la doc roxygen et note dans la vignette --- NEWS.md | 10 ++ R/compare_datasets_from_yaml.R | 38 +++++- man/compare_datasets_from_yaml.Rd | 25 +++- tests/testthat/test-yaml-arg-precedence.R | 138 ++++++++++++++++++++++ vignettes/datadiff.Rmd | 5 + 5 files changed, 209 insertions(+), 7 deletions(-) create mode 100644 tests/testthat/test-yaml-arg-precedence.R diff --git a/NEWS.md b/NEWS.md index f236e49..6a79756 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,16 @@ ## Bug fixes +* Argument vs YAML resolution is now deterministic and documented: explicit + argument > YAML `defaults` > built-in default, for both `key` and `label`. + Previously the `label` argument was silently overwritten by the YAML label + (or the built-in default) whenever `path` was provided, and the YAML `keys` + field was only reached through `$` partial matching on a singular `key` + lookup, an accident waiting to break. `keys` is now read explicitly, with a + legacy singular `key` field honored as fallback; when both are present, + `keys` (the canonical field written by `write_rules_template()`) wins + (issue #20). + * A positional (key-less) comparison of datasets with different row counts now raises a single clear error built from `error_msg_no_key` and mentioning both row counts. It previously emitted `error_msg_no_key` as an informational diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 270d021..aa048ab 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -160,17 +160,36 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' validation rules defined in a YAML file. Supports exact matching, tolerance-based #' comparisons, text normalization, and row count validation. #' +#' @section Argument vs YAML precedence: +#' Some settings can come both from an explicit argument and from the YAML +#' rules file. The resolution is always: explicit argument first, then the +#' YAML `defaults` section, then the built-in default. +#' +#' | Setting | Explicit argument | YAML `defaults` field | Built-in default | +#' |---|---|---|---| +#' | join key | `key` | `keys` (legacy alias: `key`) | none (positional) | +#' | report label | `label` | `label` | "Comparing candidate vs reference" | +#' +#' When the YAML contains both `keys` and the legacy singular `key` field, +#' `keys` (the canonical field written by [write_rules_template()]) wins. +#' +#' The `key` argument must be a character vector (a non-character value is an +#' error), while YAML-sourced key values are coerced to character (YAML being +#' stringly typed, `keys: [2024]` designates the column named "2024"). +#' #' @param data_reference Reference dataframe, tibble, or lazy table (tbl_lazy) #' @param data_candidate Candidate dataframe to validate against reference #' @param key Optional character vector of column names to use as join keys for #' ordered comparison. Every key column must exist in both datasets; otherwise #' an error is raised naming the missing column(s) and the dataset(s) concerned. +#' See the "Argument vs YAML precedence" section. #' @param path Path to YAML file containing validation rules. If NULL, default rules are #' generated automatically based on the reference dataset structure. #' @param warn_at Warning threshold as fraction of failing tests (default: 1e-14) #' @param stop_at Stop threshold as fraction of failing tests (default: 1e-14) #' @param ref_suffix Suffix for reference columns in comparison dataframe (default: "__reference") -#' @param label Descriptive label for the validation report +#' @param label Descriptive label for the validation report. See the +#' "Argument vs YAML precedence" section. #' @param error_msg_no_key Text of the error raised when a positional (key-less) #' comparison receives datasets with different row counts; the actual row #' counts of both datasets are appended to this text. @@ -360,12 +379,21 @@ compare_datasets_from_yaml <- function(data_reference, na_equal <- isTRUE(rules$defaults$na_equal) ignore_columns <- rules$defaults$ignore_columns %||% character(0) - label <- rules$defaults$label - if (is.null(label) || label == "") {label <- "Comparing candidate vs reference"} - # Use key parameter if provided, otherwise fall back to rules + # Precedence: explicit argument > YAML defaults > built-in default + label <- label %||% rules$defaults[["label"]] + if (is.null(label) || label == "") { + label <- "Comparing candidate vs reference" + } + + # Precedence: explicit argument > YAML defaults > none. The canonical YAML + # field is "keys" (what write_rules_template() writes); a legacy singular + # "key" field is honored as fallback. [[ avoids $ partial matching. if (is.null(key)) { - key <- rules$defaults$key + key <- rules$defaults[["keys"]] %||% rules$defaults[["key"]] + if (!is.null(key)) { + key <- as.character(unlist(key, use.names = FALSE)) + } } if (is.null(key)) {message("key is missing")} diff --git a/man/compare_datasets_from_yaml.Rd b/man/compare_datasets_from_yaml.Rd index c3998d8..b6225b7 100644 --- a/man/compare_datasets_from_yaml.Rd +++ b/man/compare_datasets_from_yaml.Rd @@ -31,7 +31,8 @@ compare_datasets_from_yaml( \item{key}{Optional character vector of column names to use as join keys for ordered comparison. Every key column must exist in both datasets; otherwise -an error is raised naming the missing column(s) and the dataset(s) concerned.} +an error is raised naming the missing column(s) and the dataset(s) concerned. +See the "Argument vs YAML precedence" section.} \item{path}{Path to YAML file containing validation rules. If NULL, default rules are generated automatically based on the reference dataset structure.} @@ -42,7 +43,8 @@ generated automatically based on the reference dataset structure.} \item{ref_suffix}{Suffix for reference columns in comparison dataframe (default: "__reference")} -\item{label}{Descriptive label for the validation report} +\item{label}{Descriptive label for the validation report. See the +"Argument vs YAML precedence" section.} \item{error_msg_no_key}{Text of the error raised when a positional (key-less) comparison receives datasets with different row counts; the actual row @@ -105,6 +107,25 @@ Main function for comparing reference and candidate datasets using configurable validation rules defined in a YAML file. Supports exact matching, tolerance-based comparisons, text normalization, and row count validation. } +\section{Argument vs YAML precedence}{ + +Some settings can come both from an explicit argument and from the YAML +rules file. The resolution is always: explicit argument first, then the +YAML \code{defaults} section, then the built-in default.\tabular{llll}{ + Setting \tab Explicit argument \tab YAML \code{defaults} field \tab Built-in default \cr + join key \tab \code{key} \tab \code{keys} (legacy alias: \code{key}) \tab none (positional) \cr + report label \tab \code{label} \tab \code{label} \tab "Comparing candidate vs reference" \cr +} + + +When the YAML contains both \code{keys} and the legacy singular \code{key} field, +\code{keys} (the canonical field written by \code{\link[=write_rules_template]{write_rules_template()}}) wins. + +The \code{key} argument must be a character vector (a non-character value is an +error), while YAML-sourced key values are coerced to character (YAML being +stringly typed, \verb{keys: [2024]} designates the column named "2024"). +} + \examples{ # Create test data ref <- data.frame(id = 1:3, value = c(1.0, 2.0, 3.0)) diff --git a/tests/testthat/test-yaml-arg-precedence.R b/tests/testthat/test-yaml-arg-precedence.R new file mode 100644 index 0000000..517d94f --- /dev/null +++ b/tests/testthat/test-yaml-arg-precedence.R @@ -0,0 +1,138 @@ +# Precedence between explicit arguments and YAML rules (issue #20): +# explicit argument > YAML defaults > built-in default. + +test_that("explicit label argument wins over the YAML label", { + ref <- data.frame(id = 1:2, x = c(1.0, 2.0)) + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + write_rules_template(ref, key = "id", label = "yaml label", path = yaml_path) + + res <- compare_datasets_from_yaml( + ref, ref, + key = "id", path = yaml_path, label = "explicit label" + ) + expect_identical(res$reponse$label, "explicit label") + + # Without an explicit argument, the YAML label applies + res_yaml <- compare_datasets_from_yaml(ref, ref, key = "id", path = yaml_path) + expect_identical(res_yaml$reponse$label, "yaml label") +}) + +test_that("YAML 'keys' field is read without partial matching", { + ref <- data.frame(id = 1:3, value = c(1.0, 2.0, 3.0)) + # Shuffled candidate: only a comparison joined on the YAML key passes + cand_shuffled <- ref[c(3, 1, 2), , drop = FALSE] + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(' +version: 1 +defaults: + keys: [id] + na_equal: yes +row_validation: + check_count: no +by_type: + numeric: + abs: 0.000000001 +', con = yaml_path) + + warns <- character(0) + withCallingHandlers( + { + res <- compare_datasets_from_yaml(ref, cand_shuffled, path = yaml_path) + }, + warning = function(w) { + warns <<- c(warns, conditionMessage(w)) + invokeRestart("muffleWarning") + } + ) + expect_true(res$all_passed) + expect_false(any(grepl("partial match", warns))) +}) + +test_that("YAML 'keys' field is honored under warnPartialMatchDollar", { + ref <- data.frame(id = 1:3, value = c(1.0, 2.0, 3.0)) + cand_shuffled <- ref[c(2, 3, 1), , drop = FALSE] + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(' +version: 1 +defaults: + keys: [id] +row_validation: + check_count: no +', con = yaml_path) + + old <- options(warnPartialMatchDollar = TRUE) + on.exit(options(old), add = TRUE) + + # The keys resolution itself must not rely on $ partial matching + rules <- read_rules(yaml_path) + expect_no_warning(rules$defaults[["keys"]] %||% rules$defaults[["key"]]) + + res <- suppressWarnings( + compare_datasets_from_yaml(ref, cand_shuffled, path = yaml_path) + ) + expect_true(res$all_passed) +}) + +test_that("YAML with both 'keys' and legacy 'key' fields: 'keys' wins", { + ref <- data.frame(id = 1:3, value = c(1.0, 2.0, 3.0)) + cand_shuffled <- ref[c(3, 1, 2), , drop = FALSE] + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(' +version: 1 +defaults: + keys: [id] + key: [nonexistent] +row_validation: + check_count: no +', con = yaml_path) + + # If the legacy singular field won, the comparison would error on a + # missing key column + res <- compare_datasets_from_yaml(ref, cand_shuffled, path = yaml_path) + expect_true(res$all_passed) +}) + +test_that("legacy singular 'key' YAML field is still honored", { + ref <- data.frame(id = 1:3, value = c(1.0, 2.0, 3.0)) + cand_shuffled <- ref[c(3, 1, 2), , drop = FALSE] + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(' +version: 1 +defaults: + key: [id] +row_validation: + check_count: no +', con = yaml_path) + + res <- compare_datasets_from_yaml(ref, cand_shuffled, path = yaml_path) + expect_true(res$all_passed) +}) + +test_that("key argument wins over the YAML keys field", { + ref <- data.frame(id = 1:3, other = c(10L, 20L, 30L), value = c(1.0, 2.0, 3.0)) + cand_shuffled <- ref[c(3, 1, 2), , drop = FALSE] + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(' +version: 1 +defaults: + keys: [nonexistent] +row_validation: + check_count: no +', con = yaml_path) + + # The explicit argument must shadow the (broken) YAML keys entirely + res <- compare_datasets_from_yaml(ref, cand_shuffled, key = "id", path = yaml_path) + expect_true(res$all_passed) +}) diff --git a/vignettes/datadiff.Rmd b/vignettes/datadiff.Rmd index 45e30ed..7650d8a 100644 --- a/vignettes/datadiff.Rmd +++ b/vignettes/datadiff.Rmd @@ -228,6 +228,11 @@ defaults: keys: id # join key (single or composite) label: ref vs cand # label shown in the pointblank report +# Note: when a setting exists both as a function argument and in the YAML, +# the explicit argument wins: key > defaults$keys, label > defaults$label. +# The canonical YAML field for the join key is "keys" (a legacy singular +# "key" field is honored as fallback when "keys" is absent). + row_validation: check_count: yes expected_count: ~ # null = use reference row count From a8a20cdc422808a5bbd00255ef866ea9240c1c15 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 19:06:01 +0200 Subject: [PATCH 06/40] fix: validation du label et tests de non-partial-matching renforces Reponse a la review Copilot de la PR : - label NA ou non scalaire rejete a l'entree avec un message clair (validate_label, aussi branche dans write_rules_template qui portait le meme pattern latent if (label == "")) - le test keys-sans-partial-matching active warnPartialMatchDollar, sans quoi il passait aussi avec l'implementation regressee - le doublon de test affaibli par un suppressWarnings global est fusionne dans ce test unique (capture de tous les warnings + assertion ciblee) --- R/compare_datasets_from_yaml.R | 23 ++++++++++ tests/testthat/test-yaml-arg-precedence.R | 53 +++++++++++++---------- 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index aa048ab..340ee24 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -62,6 +62,7 @@ write_rules_template <- function(data_reference, ) } } + validate_label(label) if (is.null(label) || label == "") {label <- paste("comparaison", deparse1(substitute(data_reference))) } @@ -111,6 +112,27 @@ read_rules <- function(path) { r } +#' Validate a report label argument +#' +#' A label must be NULL or a single non-NA character string: anything else +#' would crash later on the scalar `if (label == "")` fallback checks. +#' +#' @param label The label value to validate. +#' @return \code{NULL}, invisibly. Called for its side effect (error). +#' @noRd +validate_label <- function(label) { + if (is.null(label)) { + return(invisible(NULL)) + } + if (!is.character(label) || length(label) != 1 || is.na(label)) { + stop( + "Parameter 'label' must be a single character string (or NULL).", + call. = FALSE + ) + } + invisible(NULL) +} + #' Validate a comparison key against both datasets #' #' Shared guard for every code path that consumes a key: checks the type and @@ -280,6 +302,7 @@ compare_datasets_from_yaml <- function(data_reference, if (!inherits(data_candidate, valid_classes)) { stop("data_candidate must be a data.frame, tibble, lazy table, or Arrow object") } + validate_label(label) # Guard: ensure dbplyr is available when lazy tables are used if (inherits(data_reference, "tbl_lazy") || inherits(data_candidate, "tbl_lazy")) { diff --git a/tests/testthat/test-yaml-arg-precedence.R b/tests/testthat/test-yaml-arg-precedence.R index 517d94f..8774cfc 100644 --- a/tests/testthat/test-yaml-arg-precedence.R +++ b/tests/testthat/test-yaml-arg-precedence.R @@ -19,7 +19,7 @@ test_that("explicit label argument wins over the YAML label", { expect_identical(res_yaml$reponse$label, "yaml label") }) -test_that("YAML 'keys' field is read without partial matching", { +test_that("YAML 'keys' field is read without $ partial matching", { ref <- data.frame(id = 1:3, value = c(1.0, 2.0, 3.0)) # Shuffled candidate: only a comparison joined on the YAML key passes cand_shuffled <- ref[c(3, 1, 2), , drop = FALSE] @@ -38,6 +38,12 @@ by_type: abs: 0.000000001 ', con = yaml_path) + # warnPartialMatchDollar makes any $ partial-match resolution audible: + # without it, a regressed rules$defaults$key implementation would pass + # this test silently (both implementations resolve the key) + old <- options(warnPartialMatchDollar = TRUE) + on.exit(options(old), add = TRUE) + warns <- character(0) withCallingHandlers( { @@ -52,31 +58,34 @@ by_type: expect_false(any(grepl("partial match", warns))) }) -test_that("YAML 'keys' field is honored under warnPartialMatchDollar", { - ref <- data.frame(id = 1:3, value = c(1.0, 2.0, 3.0)) - cand_shuffled <- ref[c(2, 3, 1), , drop = FALSE] +test_that("an invalid label argument is rejected with a clear error", { + ref <- data.frame(id = 1:2, x = c(1.0, 2.0)) yaml_path <- tempfile(fileext = ".yaml") on.exit(unlink(yaml_path), add = TRUE) - writeLines(' -version: 1 -defaults: - keys: [id] -row_validation: - check_count: no -', con = yaml_path) - - old <- options(warnPartialMatchDollar = TRUE) - on.exit(options(old), add = TRUE) - - # The keys resolution itself must not rely on $ partial matching - rules <- read_rules(yaml_path) - expect_no_warning(rules$defaults[["keys"]] %||% rules$defaults[["key"]]) - - res <- suppressWarnings( - compare_datasets_from_yaml(ref, cand_shuffled, path = yaml_path) + write_rules_template(ref, key = "id", path = yaml_path) + + # NA and non-scalar labels must be rejected upfront, not crash later on + # an if (label == "") condition + expect_error( + compare_datasets_from_yaml(ref, ref, key = "id", path = yaml_path, + label = NA_character_), + regexp = "single character string" + ) + expect_error( + compare_datasets_from_yaml(ref, ref, key = "id", label = NA_character_), + regexp = "single character string" + ) + expect_error( + compare_datasets_from_yaml(ref, ref, key = "id", path = yaml_path, + label = c("a", "b")), + regexp = "single character string" + ) + expect_error( + write_rules_template(ref, key = "id", label = NA_character_, + path = tempfile(fileext = ".yaml")), + regexp = "single character string" ) - expect_true(res$all_passed) }) test_that("YAML with both 'keys' and legacy 'key' fields: 'keys' wins", { From 00d574972391561b1f8ae343e07bd6aa69492ad3 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 19:16:38 +0200 Subject: [PATCH 07/40] fix: label vide n'ecrase plus le label YAML, doc du label auto-template, muffle cible Reponse a la seconde review Copilot de la PR : - un label "" vaut absence de label explicite (meme convention que write_rules_template) et laisse gagner le label YAML dans la chaine de precedence - la doc de precedence precise que sur path = NULL c'est le label du template auto (Comparison with default rules) qui s'applique, pas le defaut interne - le test de non-partial-matching ne muffle plus que les warnings partial match, les autres restent audibles --- R/compare_datasets_from_yaml.R | 13 ++++++++++++- man/compare_datasets_from_yaml.Rd | 6 ++++++ tests/testthat/test-yaml-arg-precedence.R | 17 +++++++++++++---- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 340ee24..8e9ae62 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -199,6 +199,12 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' error), while YAML-sourced key values are coerced to character (YAML being #' stringly typed, `keys: [2024]` designates the column named "2024"). #' +#' Note for `path = NULL`: the auto-generated rules template itself carries a +#' label (`"Comparison with default rules"`, or the explicit `label` argument), +#' so that is the label effectively used without a YAML file; the built-in +#' default above only applies when a YAML file is supplied with an empty or +#' missing `defaults$label`. +#' #' @param data_reference Reference dataframe, tibble, or lazy table (tbl_lazy) #' @param data_candidate Candidate dataframe to validate against reference #' @param key Optional character vector of column names to use as join keys for @@ -403,7 +409,12 @@ compare_datasets_from_yaml <- function(data_reference, na_equal <- isTRUE(rules$defaults$na_equal) ignore_columns <- rules$defaults$ignore_columns %||% character(0) - # Precedence: explicit argument > YAML defaults > built-in default + # Precedence: explicit argument > YAML defaults > built-in default. + # An empty string means "no explicit label" (same convention as + # write_rules_template), so it must not shadow the YAML label. + if (!is.null(label) && label == "") { + label <- NULL + } label <- label %||% rules$defaults[["label"]] if (is.null(label) || label == "") { label <- "Comparing candidate vs reference" diff --git a/man/compare_datasets_from_yaml.Rd b/man/compare_datasets_from_yaml.Rd index b6225b7..07ba477 100644 --- a/man/compare_datasets_from_yaml.Rd +++ b/man/compare_datasets_from_yaml.Rd @@ -124,6 +124,12 @@ When the YAML contains both \code{keys} and the legacy singular \code{key} field The \code{key} argument must be a character vector (a non-character value is an error), while YAML-sourced key values are coerced to character (YAML being stringly typed, \verb{keys: [2024]} designates the column named "2024"). + +Note for \code{path = NULL}: the auto-generated rules template itself carries a +label (\code{"Comparison with default rules"}, or the explicit \code{label} argument), +so that is the label effectively used without a YAML file; the built-in +default above only applies when a YAML file is supplied with an empty or +missing \code{defaults$label}. } \examples{ diff --git a/tests/testthat/test-yaml-arg-precedence.R b/tests/testthat/test-yaml-arg-precedence.R index 8774cfc..41d7a5d 100644 --- a/tests/testthat/test-yaml-arg-precedence.R +++ b/tests/testthat/test-yaml-arg-precedence.R @@ -17,6 +17,11 @@ test_that("explicit label argument wins over the YAML label", { # Without an explicit argument, the YAML label applies res_yaml <- compare_datasets_from_yaml(ref, ref, key = "id", path = yaml_path) expect_identical(res_yaml$reponse$label, "yaml label") + + # An empty string means "no explicit label": the YAML label still applies + res_empty <- compare_datasets_from_yaml(ref, ref, key = "id", path = yaml_path, + label = "") + expect_identical(res_empty$reponse$label, "yaml label") }) test_that("YAML 'keys' field is read without $ partial matching", { @@ -44,18 +49,22 @@ by_type: old <- options(warnPartialMatchDollar = TRUE) on.exit(options(old), add = TRUE) - warns <- character(0) + # Muffle ONLY partial-match warnings (the tested regression signal); + # any other warning stays audible and would surface in the test output + partial_warns <- character(0) withCallingHandlers( { res <- compare_datasets_from_yaml(ref, cand_shuffled, path = yaml_path) }, warning = function(w) { - warns <<- c(warns, conditionMessage(w)) - invokeRestart("muffleWarning") + if (grepl("partial match", conditionMessage(w))) { + partial_warns <<- c(partial_warns, conditionMessage(w)) + invokeRestart("muffleWarning") + } } ) expect_true(res$all_passed) - expect_false(any(grepl("partial match", warns))) + expect_length(partial_warns, 0L) }) test_that("an invalid label argument is rejected with a clear error", { From 7aa060463d7c30acd0d077b7301b42daf8859c9c Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 18:26:29 +0200 Subject: [PATCH 08/40] fix: comparaison lazy correcte sur tables avec une colonne nommee n (issue #18) find_duplicate_keys() lazy comptait via dplyr::count() sans nom explicite : avec une cle nommee n, count() stocke le compte dans nn et le filter(n > 1) lisait les valeurs de la cle, faussant silencieusement la detection de doublons et les comptes du warning. Les deux helpers lazy utilisent desormais un nom de compte reserve (..datadiff_n) et globalVariables(n) est supprime. Note : contrairement au constat initial de l'issue, lazy_nrow() ne cassait pas (count() sans groupe ne renomme pas son compte quand une colonne n existe) ; le nom explicite y est applique par robustesse et le comportement verrouille par test. --- NAMESPACE | 1 + NEWS.md | 8 +++ R/duplicate_keys.R | 13 +++-- R/globals.R | 1 - R/validation.R | 4 +- tests/testthat/test-duplicate-keys.R | 73 ++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 5 deletions(-) delete mode 100644 R/globals.R diff --git a/NAMESPACE b/NAMESPACE index 7cf9d48..9234e31 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -26,6 +26,7 @@ importFrom(pointblank,col_vals_equal) importFrom(pointblank,create_agent) importFrom(pointblank,interrogate) importFrom(rlang,":=") +importFrom(rlang,.data) importFrom(stats,setNames) importFrom(tidyselect,all_of) importFrom(utils,modifyList) diff --git a/NEWS.md b/NEWS.md index 6a79756..6037a1d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,14 @@ ## Bug fixes +* Lazy comparisons now work on tables containing a column named `n`. The + duplicate-key detection used `dplyr::count()` with its default output name: + with a key column named `n`, the filter and the aggregates silently read the + key values instead of the counts, producing a wrong duplicate diagnosis. + Both lazy helpers now use a reserved count name, and the + `globalVariables("n")` declaration that masked the pattern is gone + (issue #18). + * Argument vs YAML resolution is now deterministic and documented: explicit argument > YAML `defaults` > built-in default, for both `key` and `label`. Previously the `label` argument was silently overwritten by the YAML label diff --git a/R/duplicate_keys.R b/R/duplicate_keys.R index 2fd5997..bb60c73 100644 --- a/R/duplicate_keys.R +++ b/R/duplicate_keys.R @@ -20,18 +20,25 @@ format_key_examples <- function(uniq_keys, key) { } } +#' @importFrom rlang .data +#' @noRd find_duplicate_keys <- function(data, key) { if (is_non_local(data)) { + # Reserved count name: count()'s default "n" collides with a user column + # named "n" (count() then stores its result in "nn", and a key named "n" + # would be filtered on its own values instead of the count). Assumes no + # key column is itself named "..datadiff_n". + count_col <- "..datadiff_n" dups <- data %>% - dplyr::count(dplyr::across(dplyr::all_of(key))) %>% - dplyr::filter(n > 1L) %>% + dplyr::count(dplyr::across(dplyr::all_of(key)), name = count_col) %>% + dplyr::filter(.data[[count_col]] > 1L) %>% dplyr::collect() if (nrow(dups) == 0L) { return(NULL) } return(list( n_dup_keys = nrow(dups), - n_dup_rows = sum(dups$n), + n_dup_rows = sum(dups[[count_col]]), examples = format_key_examples(dups[, key, drop = FALSE], key) )) } diff --git a/R/globals.R b/R/globals.R deleted file mode 100644 index d07754e..0000000 --- a/R/globals.R +++ /dev/null @@ -1 +0,0 @@ -globalVariables("n") diff --git a/R/validation.R b/R/validation.R index 38ca862..80e4fd0 100644 --- a/R/validation.R +++ b/R/validation.R @@ -1,6 +1,8 @@ lazy_nrow <- function(x) { if (is_non_local(x)) { - dplyr::pull(dplyr::collect(dplyr::count(x)), n) + # Explicit count name: do not rely on count()'s implicit "n"/"nn" naming + count_col <- "..datadiff_n" + dplyr::pull(dplyr::collect(dplyr::count(x, name = count_col)), var = count_col) } else { nrow(x) } diff --git a/tests/testthat/test-duplicate-keys.R b/tests/testthat/test-duplicate-keys.R index ad0f651..f6bd2af 100644 --- a/tests/testthat/test-duplicate-keys.R +++ b/tests/testthat/test-duplicate-keys.R @@ -55,3 +55,76 @@ test_that("lazy tables produce the same structure (SQL path)", { expect_equal(info$n_dup_keys, 1L) expect_equal(info$n_dup_rows, 2L) }) + +# A user column named "n" collides with dplyr::count()'s default output name: +# count() then names its result "nn" (documented count()/tally() behavior). +# The lazy helpers must use a reserved name instead of relying on "n". + +test_that("lazy duplicate detection works when the key column is named 'n'", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + # Key values are large on purpose: if filter()/sum() read the key column + # instead of the count, n_dup_keys / n_dup_rows become absurd. + duckdb::dbWriteTable(con, "t_n", data.frame(n = c(100, 100, 200), v = 1:3)) + info <- find_duplicate_keys(dplyr::tbl(con, "t_n"), "n") + expect_equal(info$n_dup_keys, 1L) + expect_equal(info$n_dup_rows, 2L) + expect_equal(info$examples, "n = 100") + + # No duplicates: values > 1 in the key column must not be mistaken + # for duplicate counts + duckdb::dbWriteTable(con, "t_n_uniq", data.frame(n = c(100, 200, 300), v = 1:3)) + expect_null(find_duplicate_keys(dplyr::tbl(con, "t_n_uniq"), "n")) +}) + +test_that("lazy_nrow works on a table with a column named 'n'", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + duckdb::dbWriteTable(con, "t_nrow", data.frame(n = c(10, 20, 30), v = 1:3)) + expect_equal(lazy_nrow(dplyr::tbl(con, "t_nrow")), 3) +}) + +test_that("full lazy comparison works on tables with a column named 'n'", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + df <- data.frame(n = c(1, 1, 2), v = c(10, 10, 20)) + duckdb::dbWriteTable(con, "ref_n", df) + duckdb::dbWriteTable(con, "cand_n", df) + + # Key named "n" with genuine duplicates: the warning must carry the real + # counts (1 duplicated key value affecting 2 rows), not sums of key values + warns <- character(0) + withCallingHandlers( + { + res <- compare_datasets_from_yaml( + dplyr::tbl(con, "ref_n"), dplyr::tbl(con, "cand_n"), + key = "n" + ) + }, + warning = function(w) { + warns <<- c(warns, conditionMessage(w)) + invokeRestart("muffleWarning") + } + ) + dup_warns <- grep("Duplicate keys detected", warns, value = TRUE) + expect_length(dup_warns, 1L) + expect_match(dup_warns, "1 duplicate key value\\(s\\) affecting 2 rows") + + # Column "n" as a plain compared (non-key) column + duckdb::dbWriteTable(con, "ref_n2", data.frame(id = 1:3, n = c(10, 20, 30))) + duckdb::dbWriteTable(con, "cand_n2", data.frame(id = 1:3, n = c(10, 20, 30))) + res2 <- compare_datasets_from_yaml( + dplyr::tbl(con, "ref_n2"), dplyr::tbl(con, "cand_n2"), + key = "id" + ) + expect_true(res2$all_passed) +}) From 770aa1771a28ada943cc6ca5066a7512e4976626 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 19:35:58 +0200 Subject: [PATCH 09/40] fix: le nom de compte reserve ne peut plus entrer en collision avec une cle Reponse a la review Copilot de la PR : count(name = ) remplace silencieusement la colonne de groupement par le compte, reproduisant la confusion valeur/compte sur le nom reserve lui-meme (les exemples du warning montraient le compte au lieu de la valeur de cle). Le nom reserve est etendu jusqu'a differer de toutes les colonnes cles. --- R/duplicate_keys.R | 9 +++++++-- tests/testthat/test-duplicate-keys.R | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/R/duplicate_keys.R b/R/duplicate_keys.R index bb60c73..ac1125d 100644 --- a/R/duplicate_keys.R +++ b/R/duplicate_keys.R @@ -26,9 +26,14 @@ find_duplicate_keys <- function(data, key) { if (is_non_local(data)) { # Reserved count name: count()'s default "n" collides with a user column # named "n" (count() then stores its result in "nn", and a key named "n" - # would be filtered on its own values instead of the count). Assumes no - # key column is itself named "..datadiff_n". + # would be filtered on its own values instead of the count). The reserved + # name is extended until it differs from every key column, because + # count(name = ) silently REPLACES that grouping column + # with the count. count_col <- "..datadiff_n" + while (count_col %in% key) { + count_col <- paste0(count_col, "_") + } dups <- data %>% dplyr::count(dplyr::across(dplyr::all_of(key)), name = count_col) %>% dplyr::filter(.data[[count_col]] > 1L) %>% diff --git a/tests/testthat/test-duplicate-keys.R b/tests/testthat/test-duplicate-keys.R index f6bd2af..aa9a6ce 100644 --- a/tests/testthat/test-duplicate-keys.R +++ b/tests/testthat/test-duplicate-keys.R @@ -80,6 +80,25 @@ test_that("lazy duplicate detection works when the key column is named 'n'", { expect_null(find_duplicate_keys(dplyr::tbl(con, "t_n_uniq"), "n")) }) +test_that("the reserved count name itself cannot collide with a key column", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + # A key literally named like the reserved count column must not reproduce + # the value-vs-count confusion on a different name + df <- data.frame(a = c(100, 100, 200), v = 1:3) + names(df)[1] <- "..datadiff_n" + duckdb::dbWriteTable(con, "t_reserved", df) + info <- find_duplicate_keys(dplyr::tbl(con, "t_reserved"), "..datadiff_n") + expect_equal(info$n_dup_keys, 1L) + expect_equal(info$n_dup_rows, 2L) + # The example must show the duplicated KEY VALUE (100), not the count (2): + # count(name = ) silently replaces the key column + expect_equal(info$examples, "..datadiff_n = 100") +}) + test_that("lazy_nrow works on a table with a column named 'n'", { skip_if_not_installed("duckdb") skip_if_not_installed("dbplyr") From b327fa7b20a701d89e7a8263d47f1306d320367d Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 18:45:14 +0200 Subject: [PATCH 10/40] fix: temp table lazy nettoyee et nom unique par processus (issue #19) Sur le chemin lazy avec connexion fournie par l'utilisateur, la table slim booleenne materialisee par compute() n'etait jamais droppee et s'accumulait jusqu'a la fermeture de la connexion. Elle est desormais supprimee en on.exit (y compris en cas d'erreur en aval du compute). Le nom de table, derive de l'horloge (%H%M%OS3), pouvait entrer en collision dans la meme milliseconde ; il vient maintenant d'un compteur par processus + PID via datadiff_tmp_table_name(). --- NEWS.md | 6 ++++ R/compare_datasets_from_yaml.R | 16 ++++++++- R/utils.R | 12 +++++++ tests/testthat/test-temp-tables.R | 58 +++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/testthat/test-temp-tables.R diff --git a/NEWS.md b/NEWS.md index 6037a1d..13b5918 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,12 @@ ## Bug fixes +* The lazy path no longer leaks its internal temp table on a user-supplied + connection: the slim boolean table is dropped at the end of each call (the + Arrow path already closed its private connection). Its name is now derived + from a per-process counter plus the PID instead of the wall clock, removing + a collision window for two calls in the same millisecond (issue #19). + * Lazy comparisons now work on tables containing a column named `n`. The duplicate-key detection used `dplyr::count()` with its default output name: with a key column named `n`, the filter and the aggregates silently read the diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 8e9ae62..bc2db62 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -631,13 +631,27 @@ compare_datasets_from_yaml <- function(data_reference, if (isTRUE(row_validation_info$check_count)) "row_count_ok" else character(0) ) cmp_slim <- dplyr::select(cmp, dplyr::any_of(val_cols)) - tmp_tbl_name <- paste0("datadiff_", gsub("[^0-9]", "", format(Sys.time(), "%H%M%OS3"))) + tmp_tbl_name <- datadiff_tmp_table_name() # compute() sends CREATE TEMP TABLE AS SELECT ... to DuckDB: all computation # (join, boolean expressions) happens inside DuckDB's process, with disk # spilling available for the large join. We then collect() the slim boolean # result into R so that pointblank receives a plain data.frame - avoiding # DuckDB connection-state issues (is_tbl_mssql crash) during interrogation. cmp_slim_computed <- dplyr::compute(cmp_slim, name = tmp_tbl_name, temporary = TRUE) + # The slim table only feeds the collect() below. Drop it at exit so that + # repeated calls on a user-supplied connection do not accumulate temp + # tables for the lifetime of that connection. after = FALSE runs the drop + # BEFORE the exit handlers registered earlier, in particular before the + # dbDisconnect of the private connection on the Arrow path (on.exit + # add = TRUE fires FIFO by default, which would drop on a closed + # connection); the try() then only masks genuine failures. + on.exit( + try( + DBI::dbRemoveTable(dbplyr::remote_con(cmp_slim_computed), tmp_tbl_name), + silent = TRUE + ), + add = TRUE, after = FALSE + ) cmp_for_agent <- dplyr::collect(cmp_slim_computed) } diff --git a/R/utils.R b/R/utils.R index f3fc49e..43a8953 100644 --- a/R/utils.R +++ b/R/utils.R @@ -17,6 +17,18 @@ is_non_local <- function(x) { inherits(x, "tbl_lazy") || inherits(x, c("ArrowObject", "arrow_dplyr_query")) } +# Package-local mutable state (temp-table counter). +.datadiff_state <- new.env(parent = emptyenv()) + +# Unique temp-table name: per-process counter + PID, so that two calls in the +# same session never collide (a clock-derived name could repeat within the +# same millisecond) and two R processes sharing a database cannot either. +datadiff_tmp_table_name <- function() { + counter <- get0("tmp_tbl_counter", envir = .datadiff_state, ifnotfound = 0L) + 1L + assign("tmp_tbl_counter", value = counter, envir = .datadiff_state) + sprintf("datadiff_tmp_%d_%d", Sys.getpid(), counter) +} + is_arrow <- function(x) { inherits(x, c("ArrowObject", "arrow_dplyr_query")) } diff --git a/tests/testthat/test-temp-tables.R b/tests/testthat/test-temp-tables.R new file mode 100644 index 0000000..d036b24 --- /dev/null +++ b/tests/testthat/test-temp-tables.R @@ -0,0 +1,58 @@ +# Lazy path temp-table hygiene: the slim boolean table computed on +# the user's connection must not outlive the call, and its name must be unique +# per process (no clock-derived names). + +test_that("no temp tables leak on a user-supplied connection", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + duckdb::dbWriteTable(con, "ref_leak", data.frame(id = 1:3, x = c(1.0, 2.0, 3.0))) + duckdb::dbWriteTable(con, "cand_leak", data.frame(id = 1:3, x = c(1.0, 2.0, 3.0))) + + for (i in 1:3) { + res <- compare_datasets_from_yaml( + dplyr::tbl(con, "ref_leak"), dplyr::tbl(con, "cand_leak"), + key = "id" + ) + expect_true(res$all_passed) + } + + leftover <- DBI::dbGetQuery(con, paste0( + "SELECT table_name FROM information_schema.tables ", + "WHERE table_name LIKE 'datadiff%'" + )) + expect_equal(nrow(leftover), 0L) +}) + +test_that("temp tables are cleaned up even when the comparison fails", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + duckdb::dbWriteTable(con, "ref_red", data.frame(id = 1:3, x = c(1.0, 2.0, 3.0))) + duckdb::dbWriteTable(con, "cand_red", data.frame(id = 1:3, x = c(9.0, 9.0, 9.0))) + + res <- compare_datasets_from_yaml( + dplyr::tbl(con, "ref_red"), dplyr::tbl(con, "cand_red"), + key = "id" + ) + expect_false(res$all_passed) + + leftover <- DBI::dbGetQuery(con, paste0( + "SELECT table_name FROM information_schema.tables ", + "WHERE table_name LIKE 'datadiff%'" + )) + expect_equal(nrow(leftover), 0L) +}) + +test_that("temp table names are unique within a process", { + n1 <- datadiff_tmp_table_name() + n2 <- datadiff_tmp_table_name() + expect_false(identical(n1, n2)) + expect_match(n1, "^datadiff_tmp_") + # The name embeds the PID so two R processes sharing a database cannot collide + expect_match(n1, as.character(Sys.getpid()), fixed = TRUE) +}) From 795211046b0b1548d7bcadab097989f05b9756f3 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 19:12:58 +0200 Subject: [PATCH 11/40] fix!: NA unilateral = difference sur les colonnes d'egalite du chemin local (issue #14) Le chemin local passait tout NA (unilateral inclus) avec na_equal = TRUE via na_pass, la ou le chemin lazy et le kernel tolerance echouent sur un NA unilateral : memes donnees, verdicts opposes selon le backend. Les trois implementations sont alignees sur la semantique du kernel : unilateral = difference, bilateral = na_equal. - eq_col_bool : semantique cible, idiome vectorise sans ifelse - chemin d'echec local : __eq materialise pour les colonnes d'egalite en echec, le step pointblank valide le meme booleen que le verdict (na_pass ne peut pas exprimer cette semantique) - garde d'equivalence enc.mco mis a jour : le NA unilateral character echoue desormais quel que soit na_equal - nouveau test-na-equality-semantics.R : equivalence local/DuckDB sur NA unilateral, bilateral, ligne candidate sans correspondance, na_equal TRUE/FALSE --- NEWS.md | 13 ++++ R/compare_datasets_from_yaml.R | 12 +++ R/fast_path.R | 25 ++++--- tests/testthat/test-equivalence-guard.R | 62 +++++++++------- tests/testthat/test-na-equality-semantics.R | 82 +++++++++++++++++++++ 5 files changed, 157 insertions(+), 37 deletions(-) create mode 100644 tests/testthat/test-na-equality-semantics.R diff --git a/NEWS.md b/NEWS.md index 13b5918..ed76837 100644 --- a/NEWS.md +++ b/NEWS.md @@ -40,6 +40,19 @@ ## Breaking changes +* The local (data.frame) path now applies the same NA semantics as the lazy + path and the numeric tolerance kernel to **equality** columns: a one-sided + NA (a value facing a missing value, including candidate rows with no + reference match after the key join) is always a difference; a two-sided NA + follows `na_equal`. Previously, with `na_equal = TRUE` (the default), any NA + on either side passed on the local path (`na_pass` semantics), while the + same data failed on the lazy path: identical datasets could get opposite + verdicts depending on the backend. Comparisons that relied on one-sided NA + passing locally will now fail; the two-sided NA behavior is unchanged. + On the local failure path, `pointblank::get_data_extracts()` output for a + failing equality column now also carries its `__eq` boolean column + (as the lazy path always did) (issue #14). + * `compare_datasets_from_yaml()` now raises an explicit error when one or more key columns are absent from either dataset, naming the missing column(s) and the dataset(s) concerned, on every code path (with or without an explicit diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index bc2db62..8350de4 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -700,6 +700,18 @@ compare_datasets_from_yaml <- function(data_reference, tbl = cmp_for_agent, tol_cols = tol_cols, eq_cols = eq_cols, ref_suffix = ref_suffix, na_equal = na_equal ) + # Local path: materialise the __eq boolean for the failing equality + # columns so the pointblank step validates the exact boolean the verdict + # used (one-sided NA fails, two-sided NA follows na_equal). The lazy path + # already carries __eq columns; col_vals_equal(na_pass = ...) alone cannot + # express these semantics. + if (!is_lazy && length(fail$eq) > 0) { + for (c in fail$eq) { + cmp_for_agent[[paste0(c, "__eq")]] <- eq_col_bool( + cmp_for_agent, col = c, ref_suffix = ref_suffix, na_equal = na_equal + ) + } + } agent <- setup_pointblank_agent( cmp_for_agent, cols_reference, diff --git a/R/fast_path.R b/R/fast_path.R index 0e94d14..956c279 100644 --- a/R/fast_path.R +++ b/R/fast_path.R @@ -1,11 +1,12 @@ # Per-column pass predicates, shared by the all-pass short-circuit and the -# failing-column selection. They reproduce pointblank's verdict exactly: +# failing-column selection. # - a tolerance column passes iff every __ok value is TRUE (NA counts as # a failure, matching col_vals_equal(..., na_pass = FALSE)); -# - an equality column passes iff its pre-computed __eq column is all TRUE -# (lazy path) or, when no __eq exists (local path), the raw comparison holds -# under the same na_pass = na_equal semantics pointblank uses (any NA on -# either side passes iff na_equal). +# - an equality column passes iff its pre-computed __eq column is all +# TRUE (lazy path) or, when no __eq exists (local path), the raw comparison +# holds under the shared NA semantics: a one-sided NA is always a +# difference, a two-sided NA follows na_equal. This matches both the +# numeric tolerance kernel and the lazy SQL CASE WHEN. # Per-row boolean outcome of a tolerance column: the pre-computed __ok # vector. NA (never produced by construction) would count as a failure, matching @@ -15,9 +16,8 @@ tol_col_bool <- function(tbl, col) { } # Per-row boolean outcome of an equality column, resolved to TRUE/FALSE (no NA): -# the pre-computed __eq vector (lazy path) or, when absent (local path), the -# raw comparison under the same na_pass = na_equal semantics pointblank uses -# (any NA on either side passes iff na_equal). +# the pre-computed __eq vector (lazy path) or, when absent (local path), +# the raw comparison with one-sided NA = FALSE and two-sided NA = na_equal. eq_col_bool <- function(tbl, col, ref_suffix, na_equal) { eq_precomputed <- tbl[[paste0(col, "__eq")]] if (!is.null(eq_precomputed)) { @@ -25,8 +25,15 @@ eq_col_bool <- function(tbl, col, ref_suffix, na_equal) { } cand_vals <- tbl[[col]] ref_vals <- tbl[[paste0(col, ref_suffix)]] + cand_na <- is.na(cand_vals) + ref_na <- is.na(ref_vals) cmp_res <- cand_vals == ref_vals - ifelse(is.na(cmp_res), na_equal, cmp_res) + # Both values present and equal; any NA resolves below + out <- !is.na(cmp_res) & cmp_res + # Two-sided NA follows na_equal; a one-sided NA stays FALSE (a value facing + # a missing value is a difference) + out[cand_na & ref_na] <- na_equal + out } tol_col_passes <- function(tbl, col) { diff --git a/tests/testthat/test-equivalence-guard.R b/tests/testthat/test-equivalence-guard.R index 7db51a9..4a80f2f 100644 --- a/tests/testthat/test-equivalence-guard.R +++ b/tests/testthat/test-equivalence-guard.R @@ -1,10 +1,11 @@ -# Equivalence guard for the speed optimization of compare_datasets_from_yaml. +# Equivalence guard for compare_datasets_from_yaml. # # These tests pin BOTH the verdict (all_passed) AND the exact set of failing # cells (as recovered through pointblank::get_data_extracts, the way the -# enc.mco oracle consumes the result). They must stay green before and after -# the optimization: the optimized code may change HOW the verdict is produced, -# never WHAT it produces. +# enc.mco oracle consumes the result). Optimizations may change HOW the +# verdict is produced, never WHAT it produces; local and lazy backends must +# agree. A documented semantic fix (with a NEWS "Breaking changes" entry) MAY +# re-pin a verdict here, deliberately and never silently. KEY <- c("id", ".row") @@ -101,31 +102,36 @@ test_that("multiple failing cells across columns are all surfaced", { expect_equal(out$cells, c("a@3|1", "n@5|1", "txt@2|1")) }) -test_that("character NA (any side) passes under na_equal = TRUE", { - for (mutate in list( - function(r, c) { r$txt[2] <- NA; c$txt[2] <- NA; list(r, c) }, - function(r, c) { c$txt[2] <- NA; list(r, c) }, - function(r, c) { r$txt[2] <- NA; list(r, c) } - )) { - ref <- make_ref(); cand <- ref - m <- mutate(ref, cand); ref <- m[[1]]; cand <- m[[2]] - out <- run_compare(ref, cand, key = KEY, na_equal_default = TRUE) - expect_true(out$all_passed) - expect_equal(out$cells, character(0)) - } -}) +# NA semantics for equality columns (aligned with the tolerance kernel and the +# lazy SQL path): a two-sided NA follows na_equal, a one-sided NA is always a +# difference. -test_that("character NA (any side) fails under na_equal = FALSE", { - for (mutate in list( - function(r, c) { r$txt[2] <- NA; c$txt[2] <- NA; list(r, c) }, - function(r, c) { c$txt[2] <- NA; list(r, c) }, - function(r, c) { r$txt[2] <- NA; list(r, c) } - )) { - ref <- make_ref(); cand <- ref - m <- mutate(ref, cand); ref <- m[[1]]; cand <- m[[2]] - out <- run_compare(ref, cand, key = KEY, na_equal_default = FALSE) - expect_false(out$all_passed) - expect_equal(out$cells, "txt@2|1") +test_that("character NA on both sides follows na_equal", { + ref <- make_ref(); cand <- ref + ref$txt[2] <- NA + cand$txt[2] <- NA + + out_true <- run_compare(ref, cand, key = KEY, na_equal_default = TRUE) + expect_true(out_true$all_passed) + expect_equal(out_true$cells, character(0)) + + out_false <- run_compare(ref, cand, key = KEY, na_equal_default = FALSE) + expect_false(out_false$all_passed) + expect_equal(out_false$cells, "txt@2|1") +}) + +test_that("character NA on one side only fails regardless of na_equal", { + for (na_equal in c(TRUE, FALSE)) { + for (mutate in list( + function(r, c) { c$txt[2] <- NA; list(r, c) }, + function(r, c) { r$txt[2] <- NA; list(r, c) } + )) { + ref <- make_ref(); cand <- ref + m <- mutate(ref, cand); ref <- m[[1]]; cand <- m[[2]] + out <- run_compare(ref, cand, key = KEY, na_equal_default = na_equal) + expect_false(out$all_passed) + expect_equal(out$cells, "txt@2|1") + } } }) diff --git a/tests/testthat/test-na-equality-semantics.R b/tests/testthat/test-na-equality-semantics.R new file mode 100644 index 0000000..f5fe6c7 --- /dev/null +++ b/tests/testthat/test-na-equality-semantics.R @@ -0,0 +1,82 @@ +# NA semantics for equality (non-tolerance) columns, aligned across the local +# and lazy paths: a one-sided NA is always a difference; a two-sided NA follows +# na_equal. Same semantics as the numeric tolerance kernel. + +compare_both_paths <- function(ref, cand, key, na_equal) { + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(sprintf(' +version: 1 +defaults: + keys: [%s] + na_equal: %s +row_validation: + check_count: no +', key, if (na_equal) "yes" else "no"), con = yaml_path) + + res_local <- suppressWarnings(suppressMessages( + compare_datasets_from_yaml(ref, cand, key = key, path = yaml_path) + )) + + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + duckdb::dbWriteTable(con, "ref_na", ref) + duckdb::dbWriteTable(con, "cand_na", cand) + res_lazy <- suppressWarnings(suppressMessages( + compare_datasets_from_yaml( + dplyr::tbl(con, "ref_na"), dplyr::tbl(con, "cand_na"), + key = key, path = yaml_path + ) + )) + list(local = res_local$all_passed, lazy = res_lazy$all_passed) +} + +test_that("one-sided NA on an equality column fails identically on both paths", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + ref <- data.frame(id = 1:2, s = c("a", NA), stringsAsFactors = FALSE) + cand <- data.frame(id = 1:2, s = c("a", "b"), stringsAsFactors = FALSE) + + for (na_equal in c(TRUE, FALSE)) { + out <- compare_both_paths(ref, cand, key = "id", na_equal = na_equal) + expect_false(out$local) + expect_identical(out$local, out$lazy) + } + + # Mirror case: NA on the candidate side + for (na_equal in c(TRUE, FALSE)) { + out <- compare_both_paths(cand, ref, key = "id", na_equal = na_equal) + expect_false(out$local) + expect_identical(out$local, out$lazy) + } +}) + +test_that("two-sided NA on an equality column follows na_equal on both paths", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + ref <- data.frame(id = 1:2, s = c("a", NA), stringsAsFactors = FALSE) + cand <- data.frame(id = 1:2, s = c("a", NA), stringsAsFactors = FALSE) + + out_true <- compare_both_paths(ref, cand, key = "id", na_equal = TRUE) + expect_true(out_true$local) + expect_identical(out_true$local, out_true$lazy) + + out_false <- compare_both_paths(ref, cand, key = "id", na_equal = FALSE) + expect_false(out_false$local) + expect_identical(out_false$local, out_false$lazy) +}) + +test_that("a candidate row with no reference match fails on both paths", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + ref <- data.frame(id = 1:2, s = c("a", "b"), stringsAsFactors = FALSE) + cand <- data.frame(id = 1:3, s = c("a", "b", "c"), stringsAsFactors = FALSE) + + # The left join fills the reference side with NA: one-sided NA, hence a + # failure, even under na_equal = TRUE + for (na_equal in c(TRUE, FALSE)) { + out <- compare_both_paths(ref, cand, key = "id", na_equal = na_equal) + expect_false(out$local) + expect_identical(out$local, out$lazy) + } +}) From a60689037a6a8f57f9c5c731093b911324f009f4 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 19:33:50 +0200 Subject: [PATCH 12/40] fix: le SQL temple reproduit la semantique NaN/Inf du kernel R (issue #13) Le CASE WHEN ne gerait que NULL ; or NaN est un float ordinaire dans DuckDB (NaN = NaN vrai, NaN > tout) : des Parquet contenant NaN/Inf obtenaient des verdicts inverses en lazy (faux PASS sur NaN/Inf unilateral, faux FAIL sur infinis de meme signe). - case_tol : bilateral NA/NaN -> na_equal, unilateral NA/NaN -> FALSE, infinis de meme signe -> TRUE, infini restant -> FALSE, sinon comparaison finie (le cap du fp_correction non fini est structurel : la branche finie n'est atteinte qu'avec une reference finie) - detection backend : isnan()/isinf() sur DuckDB ; SQLite (pas de stockage NaN, insere NULL) garde les regles NULL et detecte l'infini par comparaison a DBL_MAX - colonnes d'egalite numeriques : regles NA nan-aware (eq_num_cols), alignees sur le chemin local ou is.na(NaN) est TRUE - tests d'equivalence avec le kernel R comme oracle (grille complete NaN/Inf/NA x na_equal, DuckDB + SQLite, egalite numerique NaN, bout-en-bout reprex de l'issue) --- NEWS.md | 13 ++ R/compare_datasets_from_yaml.R | 9 +- R/tolerance.R | 86 ++++++++--- tests/testthat/test-nan-inf-sql-equivalence.R | 135 ++++++++++++++++++ 4 files changed, 226 insertions(+), 17 deletions(-) create mode 100644 tests/testthat/test-nan-inf-sql-equivalence.R diff --git a/NEWS.md b/NEWS.md index ed76837..996a3b3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,19 @@ ## Bug fixes +* The lazy SQL booleans now reproduce the R tolerance kernel's NaN/Inf + semantics: same-sign infinities pass, a one-sided NA/NaN/Inf fails, NaN on + both sides follows `na_equal`. The previous CASE WHEN only handled SQL NULL, + and NaN is a regular float in DuckDB (`NaN = NaN` is true, NaN sorts above + everything): Parquet data containing NaN or infinities could silently get + the opposite verdict on the lazy path (false PASS on one-sided NaN/Inf, + false FAIL on matching infinities). NaN detection uses `isnan()` on DuckDB; + backends without NaN storage (SQLite) keep the NULL rules, and infinity + detection falls back to a `> DBL_MAX` comparison there. Numeric equality + (non-tolerance) columns get the same NaN-aware NA rules, matching the local + path. The new equivalence tests use the R kernel as oracle on a full + NaN/Inf/NA grid, on DuckDB and SQLite (issue #13). + * The lazy path no longer leaks its internal temp table on a user-supplied connection: the slim boolean table is dropped at the end of each call (the Arrow path already closed its private connection). Its name is now derived diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 8350de4..1646a02 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -595,8 +595,15 @@ compare_datasets_from_yaml <- function(data_reference, # construction + SQL rendering), the dominant cost on wide tables; the # templated SQL is O(1) dbplyr work and lets the database do the rest. cmp <- if (is_non_local(cmp)) { + # Numeric equality columns get NaN-aware NA rules in the SQL (matching + # the local path, where is.na(NaN) is TRUE); isnan() on a non-numeric + # column would be a SQL type error. + eq_num_cols <- eq_cols[vapply(X = eq_cols, FUN = function(nm) { + is.numeric(schema_ref[[nm]]) + }, FUN.VALUE = logical(1))] add_bool_cols_sql(cmp, tol_cols, eq_cols, - col_rules, ref_suffix, na_equal) + col_rules, ref_suffix, na_equal, + eq_num_cols = eq_num_cols) } else { add_ok_columns(cmp, tol_cols, col_rules, ref_suffix, na_equal) } diff --git a/R/tolerance.R b/R/tolerance.R index b1022b7..671ead3 100644 --- a/R/tolerance.R +++ b/R/tolerance.R @@ -105,19 +105,31 @@ add_ok_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { #' On the lazy path, building the per-column tolerance/equality booleans with #' `dplyr::mutate()` is O(expressions) on the R side (dbplyr query construction #' and SQL rendering dominate; e.g. ~60 s of 64 s for 300 columns). This builds -#' the same boolean columns in a single templated SQL `SELECT`, leaving DuckDB to -#' execute. The CASE WHEN logic reproduces exactly the `dplyr::case_when` used -#' previously (NULL handling and IEEE 754 fp correction inlined), so the lazy -#' verdict is unchanged. +#' the same boolean columns in a single templated SQL `SELECT`, leaving the +#' database to execute. +#' +#' The CASE WHEN logic reproduces the R tolerance kernel +#' (`compute_tolerance_col`) including its NaN/Inf semantics: same-sign +#' infinities pass, a one-sided NA/NaN/Inf fails, NA/NaN on both sides follows +#' `na_equal`. SQL NULL only covers R's NA; NaN is a regular float in DuckDB +#' (ordered above everything, `NaN = NaN` true), so NaN detection uses +#' `isnan()` there. Backends without NaN storage (SQLite turns NaN into NULL +#' on insert) fall back to the NULL rules; infinity detection uses a +#' `> DBL_MAX` comparison where `isinf()` is unavailable. #' #' @param cmp A lazy table (the join of candidate and reference). #' @param tol_cols,eq_cols Tolerance / equality column names. #' @param col_rules Per-column rules (abs / rel). #' @param ref_suffix Reference-column suffix. #' @param na_equal Logical; NA equality semantics. +#' @param eq_num_cols Subset of `eq_cols` holding numeric data: their equality +#' CASE gets the NaN-aware NA rules (matching the local path, where +#' `is.na(NaN)` is TRUE). Non-numeric columns must not receive `isnan()` +#' (type error in SQL). #' @return A lazy table with the `__ok` / `__eq` columns added. #' @noRd -add_bool_cols_sql <- function(cmp, tol_cols, eq_cols, col_rules, ref_suffix, na_equal) { +add_bool_cols_sql <- function(cmp, tol_cols, eq_cols, col_rules, ref_suffix, + na_equal, eq_num_cols = character(0)) { if (length(tol_cols) == 0 && length(eq_cols) == 0) { return(cmp) } @@ -127,18 +139,60 @@ add_bool_cols_sql <- function(cmp, tol_cols, eq_cols, col_rules, ref_suffix, na_ num <- function(x) sprintf("%.17g", x) fp_eps <- 8 * .Machine$double.eps - case_bool <- function(cc, rc, cond) { - if (na_equal) { - sprintf(paste0("CASE WHEN %s IS NULL AND %s IS NULL THEN TRUE ", - "WHEN %s IS NULL OR %s IS NULL THEN FALSE ", - "WHEN %s THEN TRUE ELSE FALSE END"), - cc, rc, cc, rc, cond) + is_duckdb <- inherits(con, "duckdb_connection") + nan_sql <- function(x) { + if (is_duckdb) { + sprintf("isnan(%s)", x) + } else { + "FALSE" + } + } + inf_sql <- function(x) { + if (is_duckdb) { + sprintf("isinf(%s)", x) } else { - sprintf(paste0("CASE WHEN %s IS NULL OR %s IS NULL THEN FALSE ", - "WHEN %s THEN TRUE ELSE FALSE END"), - cc, rc, cond) + sprintf("(%s > 1.7976931348623157e308 OR %s < -1.7976931348623157e308)", + x, x) } } + # R's is.na(): SQL NULL or NaN + na_like <- function(x) { + sprintf("(%s IS NULL OR %s)", x, nan_sql(x)) + } + na_equal_lit <- if (na_equal) "TRUE" else "FALSE" + + # Tolerance kernel semantics: two-sided NA/NaN -> na_equal; one-sided + # NA/NaN -> FALSE; same-sign infinities -> TRUE; remaining infinity + # (one-sided or opposite signs) -> FALSE; else the finite comparison. + case_tol <- function(cc, rc, cond) { + sprintf(paste0( + "CASE WHEN %s AND %s THEN %s ", + "WHEN %s OR %s THEN FALSE ", + "WHEN %s AND %s AND ((%s > 0) = (%s > 0)) THEN TRUE ", + "WHEN %s OR %s THEN FALSE ", + "WHEN %s THEN TRUE ELSE FALSE END"), + na_like(cc), na_like(rc), na_equal_lit, + na_like(cc), na_like(rc), + inf_sql(cc), inf_sql(rc), cc, rc, + inf_sql(cc), inf_sql(rc), + cond + ) + } + + # Equality semantics: two-sided NA(-like) -> na_equal; one-sided -> FALSE; + # else plain SQL equality (infinities compare correctly there). NaN joins + # the NA rules only for numeric columns (nan_aware). + case_eq <- function(cc, rc, nan_aware) { + n <- if (nan_aware) na_like else function(x) sprintf("%s IS NULL", x) + sprintf(paste0( + "CASE WHEN %s AND %s THEN %s ", + "WHEN %s OR %s THEN FALSE ", + "WHEN %s = %s THEN TRUE ELSE FALSE END"), + n(cc), n(rc), na_equal_lit, + n(cc), n(rc), + cc, rc + ) + } tol_exprs <- vapply(X = tol_cols, FUN = function(c) { cc <- q(c) @@ -147,14 +201,14 @@ add_bool_cols_sql <- function(cmp, tol_cols, eq_cols, col_rules, ref_suffix, na_ rt <- col_rules[[c]][["rel"]] %||% 0 within <- sprintf("ABS(%s - %s) <= (%s + %s * ABS(%s)) + %s * ABS(%s)", cc, rc, num(at), num(rt), rc, num(fp_eps), rc) - sprintf("%s AS %s", case_bool(cc, rc, within), q(paste0(c, "__ok"))) + sprintf("%s AS %s", case_tol(cc, rc, cond = within), q(paste0(c, "__ok"))) }, FUN.VALUE = character(1), USE.NAMES = FALSE) eq_exprs <- vapply(X = eq_cols, FUN = function(c) { cc <- q(c) rc <- q(paste0(c, ref_suffix)) sprintf("%s AS %s", - case_bool(cc, rc, sprintf("%s = %s", cc, rc)), + case_eq(cc, rc, nan_aware = c %in% eq_num_cols), q(paste0(c, "__eq"))) }, FUN.VALUE = character(1), USE.NAMES = FALSE) diff --git a/tests/testthat/test-nan-inf-sql-equivalence.R b/tests/testthat/test-nan-inf-sql-equivalence.R new file mode 100644 index 0000000..2e885b5 --- /dev/null +++ b/tests/testthat/test-nan-inf-sql-equivalence.R @@ -0,0 +1,135 @@ +# NaN/Inf semantics of the templated SQL booleans (lazy path), with the R +# tolerance kernel as oracle (NOT the historical dplyr path, which shared the +# same NULL-only blind spot): same-sign infinities pass, a one-sided +# NA/NaN/Inf fails, NA/NaN on both sides follows na_equal. + +# Boolean vector produced by the templated SQL for one tolerance column. +lazy_tol_ok <- function(con, cand, ref, abs_tol, rel_tol, na_equal) { + cmp <- data.frame(id = seq_along(cand), x = cand, x__reference = ref) + DBI::dbWriteTable(con, "cmp_nan_inf", cmp, overwrite = TRUE) + rules <- list(x = list(abs = abs_tol, rel = rel_tol)) + out <- add_bool_cols_sql( + dplyr::tbl(con, "cmp_nan_inf"), + tol_cols = "x", eq_cols = character(0), + col_rules = rules, ref_suffix = "__reference", na_equal = na_equal + ) + res <- dplyr::collect(dplyr::arrange(out, id)) + as.logical(res$x__ok) +} + +test_that("DuckDB tolerance booleans match the R kernel on NaN/Inf/NA", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + vals <- c(1, 2.5, NA, NaN, Inf, -Inf, 0) + grid <- expand.grid(cand = vals, ref = vals) + + for (na_equal in c(TRUE, FALSE)) { + expected <- compute_tolerance_col( + grid$cand, grid$ref, + abs_tol = 0.1, rel_tol = 0, na_equal = na_equal + )$ok + got <- lazy_tol_ok(con, grid$cand, grid$ref, + abs_tol = 0.1, rel_tol = 0, na_equal = na_equal) + expect_identical(got, expected) + } + + # Relative tolerance: an infinite reference must not produce an infinite + # threshold that lets a finite candidate pass + for (na_equal in c(TRUE, FALSE)) { + expected_rel <- compute_tolerance_col( + grid$cand, grid$ref, + abs_tol = 0, rel_tol = 0.5, na_equal = na_equal + )$ok + got_rel <- lazy_tol_ok(con, grid$cand, grid$ref, + abs_tol = 0, rel_tol = 0.5, na_equal = na_equal) + expect_identical(got_rel, expected_rel) + } +}) + +test_that("SQLite tolerance booleans match the R kernel on Inf/NA", { + skip_if_not_installed("RSQLite") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(RSQLite::SQLite(), ":memory:") + on.exit(DBI::dbDisconnect(con), add = TRUE) + + # No NaN in the grid: SQLite has no NaN storage (it arrives as NULL, + # which the NA rules then cover) + vals <- c(1, 2.5, NA, Inf, -Inf, 0) + grid <- expand.grid(cand = vals, ref = vals) + + for (na_equal in c(TRUE, FALSE)) { + expected <- compute_tolerance_col( + grid$cand, grid$ref, + abs_tol = 0.1, rel_tol = 0, na_equal = na_equal + )$ok + got <- lazy_tol_ok(con, grid$cand, grid$ref, + abs_tol = 0.1, rel_tol = 0, na_equal = na_equal) + expect_identical(got, expected) + } +}) + +test_that("numeric equality columns treat NaN as NA-like on DuckDB", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + run_eq <- function(cand, ref, na_equal) { + cmp <- data.frame(id = seq_along(cand), x = cand, x__reference = ref) + DBI::dbWriteTable(con, "cmp_eq_nan", cmp, overwrite = TRUE) + out <- add_bool_cols_sql( + dplyr::tbl(con, "cmp_eq_nan"), + tol_cols = character(0), eq_cols = "x", + col_rules = list(), ref_suffix = "__reference", na_equal = na_equal, + eq_num_cols = "x" + ) + res <- dplyr::collect(dplyr::arrange(out, id)) + as.logical(res$x__eq) + } + + cand <- c(NaN, NaN, NaN, 1) + ref <- c(NaN, 1, NA, 1) + + # Local-path oracle: is.na(NaN) is TRUE, so NaN follows the NA rules + # (one-sided = difference, two-sided follows na_equal) + expect_identical(run_eq(cand, ref, na_equal = TRUE), + c(TRUE, FALSE, TRUE, TRUE)) + expect_identical(run_eq(cand, ref, na_equal = FALSE), + c(FALSE, FALSE, FALSE, TRUE)) +}) + +test_that("local and DuckDB verdicts agree end to end on NaN/Inf data", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + + ref <- data.frame(id = 1:4, x = c(NaN, Inf, Inf, NaN)) + cand <- data.frame(id = 1:4, x = c(0, 1, Inf, NaN)) + + res_local <- suppressMessages( + compare_datasets_from_yaml(ref, cand, key = "id") + ) + + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + duckdb::dbWriteTable(con, "ref_e2e", ref) + duckdb::dbWriteTable(con, "cand_e2e", cand) + res_lazy <- suppressMessages( + compare_datasets_from_yaml( + dplyr::tbl(con, "ref_e2e"), dplyr::tbl(con, "cand_e2e"), + key = "id" + ) + ) + + expect_identical(res_local$all_passed, res_lazy$all_passed) + # Same per-column failure counts, not just the same global verdict: + # rows 1 (NaN vs 0) and 2 (Inf vs 1) fail, row 3 (Inf vs Inf) and + # row 4 (NaN vs NaN, na_equal default TRUE) pass + tol_row_failures <- function(cov) { + as.integer(cov$n_failed[cov$column == "x" & cov$check == "tolerance"]) + } + expect_identical(tol_row_failures(res_local$coverage), 2L) + expect_identical(tol_row_failures(res_lazy$coverage), 2L) +}) From a16ac0f6c90d6a82de227bab1ea5e4ab493270ad Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 23:51:45 +0200 Subject: [PATCH 13/40] test: boucle rel-tol SQLite + limitation NaN hors DuckDB documentee Reponse a la review Copilot de la PR : - le test SQLite couvre desormais aussi la tolerance relative (une reference infinie ne doit pas laisser passer un candidat fini via un seuil infini, detection DBL_MAX) - limitation connue documentee : la detection NaN n'est cablee que pour DuckDB (backend lazy teste) ; un backend non-DuckDB qui stocke NaN (PostgreSQL) garde les regles NULL-only sur ces valeurs --- R/tolerance.R | 5 ++++- tests/testthat/test-nan-inf-sql-equivalence.R | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/R/tolerance.R b/R/tolerance.R index 671ead3..af9a2d9 100644 --- a/R/tolerance.R +++ b/R/tolerance.R @@ -115,7 +115,10 @@ add_ok_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { #' (ordered above everything, `NaN = NaN` true), so NaN detection uses #' `isnan()` there. Backends without NaN storage (SQLite turns NaN into NULL #' on insert) fall back to the NULL rules; infinity detection uses a -#' `> DBL_MAX` comparison where `isinf()` is unavailable. +#' `> DBL_MAX` comparison where `isinf()` is unavailable. Known limitation: +#' NaN detection is only wired for DuckDB (the tested lazy backend). A +#' non-DuckDB backend that does store NaN (e.g. PostgreSQL) keeps the +#' NULL-only rules for NaN, i.e. the pre-fix semantics on those values. #' #' @param cmp A lazy table (the join of candidate and reference). #' @param tol_cols,eq_cols Tolerance / equality column names. diff --git a/tests/testthat/test-nan-inf-sql-equivalence.R b/tests/testthat/test-nan-inf-sql-equivalence.R index 2e885b5..0e7eb8e 100644 --- a/tests/testthat/test-nan-inf-sql-equivalence.R +++ b/tests/testthat/test-nan-inf-sql-equivalence.R @@ -69,6 +69,18 @@ test_that("SQLite tolerance booleans match the R kernel on Inf/NA", { abs_tol = 0.1, rel_tol = 0, na_equal = na_equal) expect_identical(got, expected) } + + # Relative tolerance: an infinite reference must not let a finite + # candidate pass through an infinite threshold (DBL_MAX detection) + for (na_equal in c(TRUE, FALSE)) { + expected_rel <- compute_tolerance_col( + grid$cand, grid$ref, + abs_tol = 0, rel_tol = 0.5, na_equal = na_equal + )$ok + got_rel <- lazy_tol_ok(con, grid$cand, grid$ref, + abs_tol = 0, rel_tol = 0.5, na_equal = na_equal) + expect_identical(got_rel, expected_rel) + } }) test_that("numeric equality columns treat NaN as NA-like on DuckDB", { From be6f47414f45c373da5cc443961545f9df3a0c74 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 19:50:19 +0200 Subject: [PATCH 14/40] fix: colonnes factor comparees comme leurs valeurs character (issue #28) detect_column_types() classe les factors en character (ils recoivent les regles case_insensitive/trim) mais normalize_text() ne transformait que les vecteurs is.character() : une colonne factor n'etait jamais normalisee, sans avertissement - verdict different selon stringsAsFactors ou un import haven. Par ailleurs == entre deux factors a niveaux differents errait. preprocess_dataframe() convertit les factors locaux en character avant normalisation (les tables lazy ne portent pas de factors) et le predicat is_char reconnait un factor dans le schema. Doc de detect_column_types clarifiee (fourre-tout character intentionnel, list-columns non supportees). --- NEWS.md | 8 +++ R/compare_datasets_from_yaml.R | 6 +- R/data_types.R | 6 ++ R/preprocessing.R | 27 +++++++- man/detect_column_types.Rd | 7 ++ man/preprocess_dataframe.Rd | 9 ++- tests/testthat/test-factor-columns.R | 99 ++++++++++++++++++++++++++++ 7 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 tests/testthat/test-factor-columns.R diff --git a/NEWS.md b/NEWS.md index 996a3b3..7ad05af 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,14 @@ ## Bug fixes +* Factor columns are now compared as the character values they display: + preprocessing converts them, so the text normalization rules + (`case_insensitive`, `trim`) apply to them and the verdict no longer depends + on `stringsAsFactors` or haven-style imports. This also fixes an opaque + error when reference and candidate factors carried different level sets + ("level sets of factors are different"). Factor key columns join correctly + (issue #28). + * The lazy SQL booleans now reproduce the R tolerance kernel's NaN/Inf semantics: same-sign infinities pass, a one-sided NA/NaN/Inf fails, NaN on both sides follows `na_equal`. The previous CASE WHEN only handled SQL NULL, diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 1646a02..c6614cd 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -601,9 +601,9 @@ compare_datasets_from_yaml <- function(data_reference, eq_num_cols <- eq_cols[vapply(X = eq_cols, FUN = function(nm) { is.numeric(schema_ref[[nm]]) }, FUN.VALUE = logical(1))] - add_bool_cols_sql(cmp, tol_cols, eq_cols, - col_rules, ref_suffix, na_equal, - eq_num_cols = eq_num_cols) + add_bool_cols_sql(cmp, tol_cols = tol_cols, eq_cols = eq_cols, + col_rules = col_rules, ref_suffix = ref_suffix, + na_equal = na_equal, eq_num_cols = eq_num_cols) } else { add_ok_columns(cmp, tol_cols, col_rules, ref_suffix, na_equal) } diff --git a/R/data_types.R b/R/data_types.R index 4561d6a..58e6836 100644 --- a/R/data_types.R +++ b/R/data_types.R @@ -3,6 +3,12 @@ #' Analyzes each column of a dataframe to determine its data type #' (integer, numeric, date, datetime, logical, or character) #' +#' Everything that is none of the first five types falls in the "character" +#' bucket and receives the character rules. That is intentional for factors +#' (compared as the character values they display, see +#' [preprocess_dataframe()]) and works for types whose `==` behaves like +#' character (e.g. `bit64::integer64`). List-columns are not supported. +#' #' @param data A dataframe or tibble #' @return A named character vector with column names as names and types as values #' @examples diff --git a/R/preprocessing.R b/R/preprocessing.R index 3100a4f..678068d 100644 --- a/R/preprocessing.R +++ b/R/preprocessing.R @@ -30,12 +30,18 @@ normalize_text <- function(x, case_insensitive = FALSE, trim = FALSE) { #' such as text normalization for character columns. Supports both local data.frames #' and lazy tables (tbl_lazy) via dplyr::mutate(). #' +#' On a local data.frame, every factor column (including key and non-compared +#' columns) is first converted to character: factors are compared as the +#' character values they display, so the text normalization rules apply to +#' them and factors with differing level sets compare cleanly. +#' #' @param df A dataframe or lazy table to preprocess #' @param col_rules A list of column-specific rules from derive_column_rules() #' @param schema Optional local data.frame with 0 rows used to determine column #' types when \code{df} is a lazy table. Obtained via #' \code{dplyr::collect(utils::head(df, 0L))}. -#' @return Preprocessed dataframe (or lazy table) with transformations applied +#' @return Preprocessed dataframe (or lazy table) with transformations applied. +#' Factor columns of a local data.frame come back as character vectors. #' @examples #' df <- data.frame(text_col = c(" HELLO ", "world")) #' rules <- list(text_col = list(equal_mode = "normalized", case_insensitive = TRUE, trim = TRUE)) @@ -44,6 +50,19 @@ normalize_text <- function(x, case_insensitive = FALSE, trim = FALSE) { #' @export preprocess_dataframe <- function(df, col_rules, schema = NULL) { out <- df + # Factors are classed "character" by detect_column_types() and receive the + # character rules, but == on factors with differing level sets errors and + # normalize_text() skips non-character vectors. Compare them as the + # character values they display. SQL lazy tables convert factors to varchar + # upstream; Arrow dictionary columns stay lazy and are handled by the + # is.factor(schema[[nm]]) branch of the is_char predicate below. + if (!is_non_local(out)) { + for (nm in names(out)) { + if (is.factor(out[[nm]])) { + out[[nm]] <- as.character(out[[nm]]) + } + } + } for (nm in names(col_rules)) { cr <- col_rules[[nm]] eq_mode <- cr$equal_mode %||% "exact" @@ -53,9 +72,11 @@ preprocess_dataframe <- function(df, col_rules, schema = NULL) { # Apply normalization if equal_mode is "normalized" OR if case_insensitive/trim is TRUE should_normalize <- identical(eq_mode, "normalized") || case_insensitive || trim - # Determine column type: use schema for lazy tables, otherwise inspect df directly + # Determine column type: use schema for lazy tables, otherwise inspect df + # directly. A factor in the schema counts as character: the local values + # were converted above and the character rules apply to it. is_char <- if (!is.null(schema)) { - is.character(schema[[nm]]) + is.character(schema[[nm]]) || is.factor(schema[[nm]]) } else { is.character(out[[nm]]) } diff --git a/man/detect_column_types.Rd b/man/detect_column_types.Rd index 60f96d0..52ccf53 100644 --- a/man/detect_column_types.Rd +++ b/man/detect_column_types.Rd @@ -16,6 +16,13 @@ A named character vector with column names as names and types as values Analyzes each column of a dataframe to determine its data type (integer, numeric, date, datetime, logical, or character) } +\details{ +Everything that is none of the first five types falls in the "character" +bucket and receives the character rules. That is intentional for factors +(compared as the character values they display, see +\code{\link[=preprocess_dataframe]{preprocess_dataframe()}}) and works for types whose \code{==} behaves like +character (e.g. \code{bit64::integer64}). List-columns are not supported. +} \examples{ df <- data.frame(a = 1:3, b = c(1.1, 2.2, 3.3), c = c("x", "y", "z")) detect_column_types(df) diff --git a/man/preprocess_dataframe.Rd b/man/preprocess_dataframe.Rd index eef4dc8..91057ab 100644 --- a/man/preprocess_dataframe.Rd +++ b/man/preprocess_dataframe.Rd @@ -16,13 +16,20 @@ types when \code{df} is a lazy table. Obtained via \code{dplyr::collect(utils::head(df, 0L))}.} } \value{ -Preprocessed dataframe (or lazy table) with transformations applied +Preprocessed dataframe (or lazy table) with transformations applied. +Factor columns of a local data.frame come back as character vectors. } \description{ Applies preprocessing transformations to dataframe columns based on validation rules, such as text normalization for character columns. Supports both local data.frames and lazy tables (tbl_lazy) via dplyr::mutate(). } +\details{ +On a local data.frame, every factor column (including key and non-compared +columns) is first converted to character: factors are compared as the +character values they display, so the text normalization rules apply to +them and factors with differing level sets compare cleanly. +} \examples{ df <- data.frame(text_col = c(" HELLO ", "world")) rules <- list(text_col = list(equal_mode = "normalized", case_insensitive = TRUE, trim = TRUE)) diff --git a/tests/testthat/test-factor-columns.R b/tests/testthat/test-factor-columns.R new file mode 100644 index 0000000..6d14f1c --- /dev/null +++ b/tests/testthat/test-factor-columns.R @@ -0,0 +1,99 @@ +# Factor columns are classed "character" by detect_column_types() and receive +# the character rules, so they must be compared as the character values they +# display: preprocessing converts them, making the verdict independent of +# stringsAsFactors / haven-style imports. + +test_that("factor and character candidates get the same verdict under normalization", { + ref <- data.frame(id = 1:2, s = c("alpha", "beta"), stringsAsFactors = FALSE) + cand_chr <- data.frame(id = 1:2, s = c("ALPHA", " beta"), stringsAsFactors = FALSE) + cand_fct <- data.frame(id = 1:2, s = factor(c("ALPHA", " beta"))) + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + write_rules_template( + ref, + key = "id", path = yaml_path, + character_case_insensitive = TRUE, character_trim = TRUE + ) + + res_chr <- compare_datasets_from_yaml(ref, cand_chr, key = "id", path = yaml_path) + res_fct <- compare_datasets_from_yaml(ref, cand_fct, key = "id", path = yaml_path) + expect_true(res_chr$all_passed) + expect_true(res_fct$all_passed) +}) + +test_that("factor columns are normalized on the reference side and both sides", { + ref_fct <- data.frame(id = 1:2, s = factor(c("ALPHA", " beta"))) + cand_chr <- data.frame(id = 1:2, s = c("alpha", "beta"), stringsAsFactors = FALSE) + cand_fct <- data.frame(id = 1:2, s = factor(c("alpha", "beta"))) + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + write_rules_template( + data.frame(id = 1:2, s = c("x", "y"), stringsAsFactors = FALSE), + key = "id", path = yaml_path, + character_case_insensitive = TRUE, character_trim = TRUE + ) + + res_ref_side <- compare_datasets_from_yaml(ref_fct, cand_chr, key = "id", path = yaml_path) + expect_true(res_ref_side$all_passed) + + res_both <- compare_datasets_from_yaml(ref_fct, cand_fct, key = "id", path = yaml_path) + expect_true(res_both$all_passed) +}) + +test_that("factors with differing level sets compare cleanly", { + # == on two factors with different level sets errors in base R; after the + # character conversion the comparison must work and give the right verdict + ref <- data.frame(id = 1:3, s = factor(c("a", "b", "c"), levels = c("a", "b", "c"))) + cand <- data.frame(id = 1:3, s = factor(c("a", "b", "d"), levels = c("a", "b", "d"))) + + res <- compare_datasets_from_yaml(ref, cand, key = "id") + expect_false(res$all_passed) + cov <- res$coverage + expect_identical( + as.integer(cov$n_failed[cov$column == "s" & cov$check == "equality"]), + 1L + ) + + # Identical displayed values with different (unused) levels pass + cand_same <- data.frame(id = 1:3, s = factor(c("a", "b", "c"), levels = c("c", "b", "a", "z"))) + res_same <- compare_datasets_from_yaml(ref, cand_same, key = "id") + expect_true(res_same$all_passed) +}) + +test_that("a factor key column joins correctly", { + ref <- data.frame(id = factor(c("k1", "k2", "k3")), x = c(1.0, 2.0, 3.0)) + cand <- data.frame(id = c("k3", "k1", "k2"), x = c(3.0, 1.0, 2.0), + stringsAsFactors = FALSE) + + res <- compare_datasets_from_yaml(ref, cand, key = "id") + expect_true(res$all_passed) +}) + +test_that("Arrow dictionary (factor) columns are normalized on the lazy path", { + skip_if_not_installed("arrow") + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + skip_if_not_installed("stringr") + + # Arrow carries factors as dictionary columns: the raw 0-row schema still + # answers is.factor() TRUE, which must route them through the lazy + # normalization path + ref <- data.frame(id = 1:2, s = c("alpha", "beta"), stringsAsFactors = FALSE) + cand <- data.frame(id = 1:2, s = factor(c("ALPHA", " beta"))) + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + write_rules_template( + ref, + key = "id", path = yaml_path, + character_case_insensitive = TRUE, character_trim = TRUE + ) + + res <- suppressMessages(compare_datasets_from_yaml( + arrow::arrow_table(ref), arrow::arrow_table(cand), + key = "id", path = yaml_path + )) + expect_true(res$all_passed) +}) From 60ad884aca6433c697074ff2be3aef52ec74d2c1 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 00:10:11 +0200 Subject: [PATCH 15/40] docs: reformulation du paragraphe fourre-tout character de detect_column_types Reponse a la re-review Copilot de la PR (tournure agrammaticale dans la doc generee). --- R/data_types.R | 2 +- man/detect_column_types.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/data_types.R b/R/data_types.R index 58e6836..2c97a8f 100644 --- a/R/data_types.R +++ b/R/data_types.R @@ -3,7 +3,7 @@ #' Analyzes each column of a dataframe to determine its data type #' (integer, numeric, date, datetime, logical, or character) #' -#' Everything that is none of the first five types falls in the "character" +#' Any column matching none of the first five types falls in the "character" #' bucket and receives the character rules. That is intentional for factors #' (compared as the character values they display, see #' [preprocess_dataframe()]) and works for types whose `==` behaves like diff --git a/man/detect_column_types.Rd b/man/detect_column_types.Rd index 52ccf53..6f2487c 100644 --- a/man/detect_column_types.Rd +++ b/man/detect_column_types.Rd @@ -17,7 +17,7 @@ Analyzes each column of a dataframe to determine its data type (integer, numeric, date, datetime, logical, or character) } \details{ -Everything that is none of the first five types falls in the "character" +Any column matching none of the first five types falls in the "character" bucket and receives the character rules. That is intentional for factors (compared as the character values they display, see \code{\link[=preprocess_dataframe]{preprocess_dataframe()}}) and works for types whose \code{==} behaves like From 9cfc64fff1b0f054155e4c9f530d983df6acbdcc Mon Sep 17 00:00:00 2001 From: infra-bot Date: Thu, 2 Jul 2026 20:07:15 +0200 Subject: [PATCH 16/40] fix: colonne booleenne __ok/__eq absente = erreur interne franche (issue #17) all(NULL) vaut TRUE : une colonne de validation silencieusement droppee devenait un faux all-pass avec n = 0 dans la coverage - le pire mode de casse pour un outil de non-regression. - tol_col_bool / eq_col_bool : stop() explicite nommant la colonne manquante ; les reducteurs (passes/counts) heritent du garde - projection slim lazy : all_of() (bruyant) au lieu de any_of() (drop silencieux) - la pose du garde a immediatement demasque un bug latent : paste0(character(0), "__ok") produit "__ok" (recycle0 FALSE par defaut), un nom fantome present dans val_cols depuis la 0.4.8 et avale par any_of() ; gardes de longueur ajoutes --- NEWS.md | 11 ++++++ R/compare_datasets_from_yaml.R | 22 +++++++++-- R/fast_path.R | 27 ++++++++++++- tests/testthat/test-boolean-column-guards.R | 43 +++++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 tests/testthat/test-boolean-column-guards.R diff --git a/NEWS.md b/NEWS.md index 7ad05af..d2e1d5c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,16 @@ # datadiff (development version) +## Robustness + +* A missing `__ok`/`__eq` boolean validation column now raises an explicit + internal error instead of silently reading as an all-pass (`all(NULL)` is + `TRUE`, and a dropped column previously produced a PASS row with `n = 0` in + the coverage: the worst failure mode for a non-regression tool). The lazy + slim-table projection uses `all_of()` (loud) instead of `any_of()` (silent + drop); posing that guard immediately surfaced a latent phantom name in the + projection (`paste0(character(0), "__ok")` yields `"__ok"`) that `any_of()` + had been eating since 0.4.8 (issue #17). + ## Bug fixes * Factor columns are now compared as the character values they display: diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index c6614cd..9f2b2bb 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -632,12 +632,26 @@ compare_datasets_from_yaml <- function(data_reference, is_lazy <- is_non_local(cmp) cmp_for_agent <- cmp if (is_lazy) { + # Length guards: paste0(character(0), "__ok") yields "__ok" (recycle0 is + # FALSE by default), a phantom name the previous any_of() silently ate + suffix_all <- function(cols, suffix) { + if (length(cols) == 0) { + return(character(0)) + } + paste0(cols, suffix) + } val_cols <- c( - paste0(tol_cols, "__ok"), - paste0(eq_cols, "__eq"), - if (isTRUE(row_validation_info$check_count)) "row_count_ok" else character(0) + suffix_all(tol_cols, suffix = "__ok"), + suffix_all(eq_cols, suffix = "__eq"), + if (isTRUE(row_validation_info$check_count)) { + "row_count_ok" + } else { + character(0) + } ) - cmp_slim <- dplyr::select(cmp, dplyr::any_of(val_cols)) + # all_of(): a validation column silently dropped here would later read as + # NULL by the boolean accessors, i.e. a false all-pass; fail loudly instead + cmp_slim <- dplyr::select(cmp, dplyr::all_of(val_cols)) tmp_tbl_name <- datadiff_tmp_table_name() # compute() sends CREATE TEMP TABLE AS SELECT ... to DuckDB: all computation # (join, boolean expressions) happens inside DuckDB's process, with disk diff --git a/R/fast_path.R b/R/fast_path.R index 956c279..e8af144 100644 --- a/R/fast_path.R +++ b/R/fast_path.R @@ -10,9 +10,16 @@ # Per-row boolean outcome of a tolerance column: the pre-computed __ok # vector. NA (never produced by construction) would count as a failure, matching -# col_vals_equal(..., na_pass = FALSE). +# col_vals_equal(..., na_pass = FALSE). A missing __ok column is a hard internal +# error: all(NULL) is TRUE, so returning NULL would silently turn a dropped +# column into a false all-pass. tol_col_bool <- function(tbl, col) { - tbl[[paste0(col, "__ok")]] + b <- tbl[[paste0(col, "__ok")]] + if (is.null(b)) { + stop(sprintf("internal error: boolean column '%s__ok' missing", col), + call. = FALSE) + } + b } # Per-row boolean outcome of an equality column, resolved to TRUE/FALSE (no NA): @@ -25,6 +32,22 @@ eq_col_bool <- function(tbl, col, ref_suffix, na_equal) { } cand_vals <- tbl[[col]] ref_vals <- tbl[[paste0(col, ref_suffix)]] + # Same rationale as tol_col_bool: a missing pair must fail loudly, not + # yield NULL and a silent all-pass downstream + if (is.null(cand_vals) || is.null(ref_vals)) { + missing_cols <- c( + if (is.null(cand_vals)) { + sprintf("'%s'", col) + }, + if (is.null(ref_vals)) { + sprintf("'%s%s'", col, ref_suffix) + } + ) + stop(sprintf( + "internal error: equality column(s) %s missing (no precomputed '%s__eq' either)", + paste(missing_cols, collapse = " and "), col + ), call. = FALSE) + } cand_na <- is.na(cand_vals) ref_na <- is.na(ref_vals) cmp_res <- cand_vals == ref_vals diff --git a/tests/testthat/test-boolean-column-guards.R b/tests/testthat/test-boolean-column-guards.R new file mode 100644 index 0000000..f3dd122 --- /dev/null +++ b/tests/testthat/test-boolean-column-guards.R @@ -0,0 +1,43 @@ +# A missing __ok/__eq boolean column must fail loudly: all(NULL) is TRUE, so a +# silently dropped column would turn into a false "all pass" with n = 0 in the +# coverage - the worst failure mode for a non-regression tool. + +test_that("tol_col_bool errors on a missing __ok column", { + tbl <- data.frame(a = 1:3) + expect_error( + tol_col_bool(tbl, col = "a"), + regexp = "internal error.*a__ok" + ) +}) + +test_that("eq_col_bool errors when both __eq and the raw pair are missing", { + tbl_no_ref <- data.frame(a = 1:3) + expect_error( + eq_col_bool(tbl_no_ref, col = "a", ref_suffix = "__reference", na_equal = TRUE), + regexp = "internal error.*a__reference" + ) + + tbl_no_cand <- data.frame(a__reference = 1:3) + expect_error( + eq_col_bool(tbl_no_cand, col = "a", ref_suffix = "__reference", na_equal = TRUE), + regexp = "internal error.*'a'" + ) +}) + +test_that("the count reducers inherit the guard instead of PASS with n = 0", { + tbl <- data.frame(a = 1:3) + expect_error(tol_col_counts(tbl, col = "a"), regexp = "internal error") + expect_error( + eq_col_counts(tbl, col = "a", ref_suffix = "__reference", na_equal = TRUE), + regexp = "internal error" + ) +}) + +test_that("the pass predicates inherit the guard instead of TRUE", { + tbl <- data.frame(a = 1:3) + expect_error(tol_col_passes(tbl, col = "a"), regexp = "internal error") + expect_error( + eq_col_passes(tbl, col = "a", ref_suffix = "__reference", na_equal = TRUE), + regexp = "internal error" + ) +}) From 48d9aaef56161440762b3afe762d2e4cd5546c8f Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 00:04:04 +0200 Subject: [PATCH 17/40] fix: SQL arrow_dataset_to_duckdb echappe et schema unifie par nom (issue #30) - chemins Parquet quotes via DBI::dbQuoteString : un chemin contenant une apostrophe (l'export/, courant en francais) cassait la requete avec une erreur SQL brute ; nom de table via dbQuoteIdentifier - read_parquet(union_by_name = true) : binding des fichiers par NOM de colonne, aligne sur la garantie de schema unifie du fallback arrow::to_duckdb() quand l'ordre physique differe entre fichiers - duckdb_memory_limit valide comme litteral de taille avant interpolation dans SET (rejet des valeurs injectables ou invalides) ; SET temp_directory quote aussi - tests dedies : chemin avec apostrophe bout-en-bout, schemas decales multi-fichiers, fallback to_duckdb, memory_limit invalide --- NEWS.md | 9 +++ R/compare_datasets_from_yaml.R | 38 ++++++++- R/utils.R | 15 +++- man/compare_datasets_from_yaml.Rd | 6 +- tests/testthat/test-arrow-dataset-to-duckdb.R | 80 +++++++++++++++++++ 5 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 tests/testthat/test-arrow-dataset-to-duckdb.R diff --git a/NEWS.md b/NEWS.md index d2e1d5c..f99cb36 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,15 @@ ## Robustness +* `arrow_dataset_to_duckdb()` builds its SQL safely: Parquet file paths are + quoted with `DBI::dbQuoteString()` (a path containing a quote, common in + French like `l'export/`, broke the query with a raw SQL syntax error), the + temp table name goes through `dbQuoteIdentifier()`, and `read_parquet()` + gets `union_by_name = true` so multi-file datasets bind columns by NAME + like the `arrow::to_duckdb()` fallback always did. `duckdb_memory_limit` + is validated as a size literal before being interpolated into `SET`, and + the `SET temp_directory` path is quoted too (issue #30). + * A missing `__ok`/`__eq` boolean validation column now raises an explicit internal error instead of silently reading as an all-pass (`all(NULL)` is `TRUE`, and a dropped column previously produced a PASS row with `n = 0` in diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 9f2b2bb..24d2a56 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -133,6 +133,29 @@ validate_label <- function(label) { invisible(NULL) } +#' Validate the duckdb_memory_limit argument +#' +#' The value is interpolated into a DuckDB SET statement: only a plain size +#' literal (number plus optional unit) or a percentage is accepted. +#' +#' @param duckdb_memory_limit The value to validate. +#' @return \code{NULL}, invisibly. Called for its side effect (error). +#' @noRd +validate_duckdb_memory_limit <- function(duckdb_memory_limit) { + ok <- is.character(duckdb_memory_limit) && + length(duckdb_memory_limit) == 1 && + !is.na(duckdb_memory_limit) && + grepl("^\\s*[0-9]+(\\.[0-9]+)?\\s*(B|KB|MB|GB|TB|KiB|MiB|GiB|TiB|%)?\\s*$", + x = duckdb_memory_limit, ignore.case = TRUE) + if (!ok) { + stop( + "Parameter 'duckdb_memory_limit' must be a single size literal such as \"8GB\" (number plus optional unit or %).", + call. = FALSE + ) + } + invisible(NULL) +} + #' Validate a comparison key against both datasets #' #' Shared guard for every code path that consumes a key: checks the type and @@ -246,8 +269,10 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' may use before spilling intermediate results to `tempdir()`. The default leaves #' headroom for R, Arrow, and the OS alongside DuckDB. Raise it (e.g. `"16GB"`) #' on machines with ample free RAM to reduce disk I/O; lower it (e.g. `"4GB"`) -#' when memory is very constrained. Has no effect when both inputs are plain -#' `data.frame`s or `tbl_lazy` objects. +#' when memory is very constrained. The value is always validated as a size +#' literal (an invalid one is an error on every path), but it only takes +#' effect when Arrow datasets are used: plain `data.frame`s or `tbl_lazy` +#' inputs ignore a valid value. #' @return A list containing: #' \item{agent}{Configured pointblank agent with validation results} #' \item{reponse}{Interrogated pointblank agent (class \code{datadiff_report}): @@ -309,6 +334,7 @@ compare_datasets_from_yaml <- function(data_reference, stop("data_candidate must be a data.frame, tibble, lazy table, or Arrow object") } validate_label(label) + validate_duckdb_memory_limit(duckdb_memory_limit) # Guard: ensure dbplyr is available when lazy tables are used if (inherits(data_reference, "tbl_lazy") || inherits(data_candidate, "tbl_lazy")) { @@ -347,12 +373,16 @@ compare_datasets_from_yaml <- function(data_reference, on.exit(duckdb::dbDisconnect(fresh_con, shutdown = TRUE), add = TRUE) # Enable disk-spilling. DBI::dbExecute(fresh_con, paste0( - "SET temp_directory='", gsub("\\\\", "/", tempdir()), "'" + "SET temp_directory = ", + as.character(DBI::dbQuoteString(fresh_con, x = gsub("\\\\", "/", tempdir()))) )) # Cap DuckDB's buffer pool so it starts spilling well before exhausting # system RAM. The default (80 % of total RAM) leaves no headroom for # R, Arrow, and OS memory. Configurable via duckdb_memory_limit. - DBI::dbExecute(fresh_con, paste0("SET memory_limit = '", duckdb_memory_limit, "'")) + DBI::dbExecute(fresh_con, paste0( + "SET memory_limit = ", + as.character(DBI::dbQuoteString(fresh_con, x = duckdb_memory_limit)) + )) if (is_arrow(data_reference)) data_reference <- arrow_dataset_to_duckdb(data_reference, fresh_con, "datadiff_ref") if (is_arrow(data_candidate)) diff --git a/R/utils.R b/R/utils.R index 43a8953..a8dba33 100644 --- a/R/utils.R +++ b/R/utils.R @@ -45,10 +45,19 @@ arrow_dataset_to_duckdb <- function(ds, con, tbl_name) { }) if (!is.null(files) && length(files) > 0 && all(grepl("\\.parquet$", files, ignore.case = TRUE))) { - paths_sql <- paste0("'", gsub("\\\\", "/", files), "'", collapse = ", ") + # dbQuoteString: a path containing a quote (l'export.parquet) must not + # break the SQL. union_by_name: bind the files by column NAME, matching + # the unified-schema guarantee of the arrow::to_duckdb() fallback even + # when the physical column order differs across files. + quoted_paths <- vapply(X = gsub("\\\\", "/", files), FUN = function(p) { + as.character(DBI::dbQuoteString(con, x = p)) + }, FUN.VALUE = character(1), USE.NAMES = FALSE) DBI::dbExecute(con, paste0( - "CREATE OR REPLACE TEMP TABLE \"", tbl_name, "\" AS ", - "SELECT * FROM read_parquet([", paths_sql, "])" + "CREATE OR REPLACE TEMP TABLE ", + as.character(DBI::dbQuoteIdentifier(con, x = tbl_name)), " AS ", + "SELECT * FROM read_parquet([", + paste(quoted_paths, collapse = ", "), + "], union_by_name = true)" )) return(dplyr::tbl(con, tbl_name)) } diff --git a/man/compare_datasets_from_yaml.Rd b/man/compare_datasets_from_yaml.Rd index 07ba477..91bf198 100644 --- a/man/compare_datasets_from_yaml.Rd +++ b/man/compare_datasets_from_yaml.Rd @@ -82,8 +82,10 @@ when Arrow datasets are used (default: \code{"8GB"}). Controls how much RAM Duck may use before spilling intermediate results to \code{tempdir()}. The default leaves headroom for R, Arrow, and the OS alongside DuckDB. Raise it (e.g. \code{"16GB"}) on machines with ample free RAM to reduce disk I/O; lower it (e.g. \code{"4GB"}) -when memory is very constrained. Has no effect when both inputs are plain -\code{data.frame}s or \code{tbl_lazy} objects.} +when memory is very constrained. The value is always validated as a size +literal (an invalid one is an error on every path), but it only takes +effect when Arrow datasets are used: plain \code{data.frame}s or \code{tbl_lazy} +inputs ignore a valid value.} } \value{ A list containing: diff --git a/tests/testthat/test-arrow-dataset-to-duckdb.R b/tests/testthat/test-arrow-dataset-to-duckdb.R new file mode 100644 index 0000000..eb47722 --- /dev/null +++ b/tests/testthat/test-arrow-dataset-to-duckdb.R @@ -0,0 +1,80 @@ +# arrow_dataset_to_duckdb(): SQL built from file paths must survive special +# characters, and multi-file datasets must keep a unified schema (as the +# arrow::to_duckdb() fallback always did). + +test_that("a Parquet path containing a quote works end to end", { + skip_if_not_installed("arrow") + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + + d <- file.path(tempdir(), "l'export datadiff") + dir.create(d, showWarnings = FALSE) + on.exit(unlink(d, recursive = TRUE), add = TRUE) + arrow::write_parquet( + data.frame(id = 1:3, x = c(1.0, 2.0, 3.0)), + file.path(d, "part-0.parquet") + ) + ds <- arrow::open_dataset(d) + + res <- suppressMessages(compare_datasets_from_yaml(ds, ds, key = "id")) + expect_true(res$all_passed) +}) + +test_that("multi-file datasets with shifted schemas are unified by name", { + skip_if_not_installed("arrow") + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + + d <- file.path(tempdir(), "datadiff_union_by_name") + dir.create(d, showWarnings = FALSE) + on.exit(unlink(d, recursive = TRUE), add = TRUE) + # Same columns, different physical order in the second file: a positional + # binding would silently misalign x and y + arrow::write_parquet( + data.frame(id = 1:2, x = c(1.0, 2.0), y = c(10.0, 20.0)), + file.path(d, "part-0.parquet") + ) + arrow::write_parquet( + data.frame(y = c(30.0, 40.0), x = c(3.0, 4.0), id = 3:4), + file.path(d, "part-1.parquet") + ) + ds <- arrow::open_dataset(d) + + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + tbl_lazy <- arrow_dataset_to_duckdb(ds, con, "datadiff_union_test") + collected <- dplyr::collect(tbl_lazy) + collected <- collected[order(collected$id), , drop = FALSE] + + expect_setequal(names(collected), c("id", "x", "y")) + expect_identical(collected$x, c(1.0, 2.0, 3.0, 4.0)) + expect_identical(collected$y, c(10.0, 20.0, 30.0, 40.0)) +}) + +test_that("non-file-backed Arrow objects still use the to_duckdb fallback", { + skip_if_not_installed("arrow") + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + + tbl_arrow <- arrow::arrow_table(data.frame(id = 1:3, x = c(1.0, 2.0, 3.0))) + res <- suppressMessages(compare_datasets_from_yaml(tbl_arrow, tbl_arrow, key = "id")) + expect_true(res$all_passed) +}) + +test_that("an invalid duckdb_memory_limit is rejected before reaching SQL", { + skip_if_not_installed("arrow") + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + + tbl_arrow <- arrow::arrow_table(data.frame(id = 1:3, x = c(1.0, 2.0, 3.0))) + expect_error( + compare_datasets_from_yaml(tbl_arrow, tbl_arrow, key = "id", + duckdb_memory_limit = "8GB'; DROP TABLE x; --"), + regexp = "duckdb_memory_limit" + ) + expect_error( + compare_datasets_from_yaml(tbl_arrow, tbl_arrow, key = "id", + duckdb_memory_limit = "beaucoup"), + regexp = "duckdb_memory_limit" + ) +}) From 3be6ffcae18bdb55a9feccea8ad8a288ccb9e776 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 00:41:01 +0200 Subject: [PATCH 18/40] test: repertoires uniques par run dans les tests arrow (tempfile) Reponse a la review Copilot de la PR : un nom de repertoire fixe sous tempdir() peut entrer en collision entre runs paralleles ou avec les restes d'un run precedent. Base unique via tempfile(), l'apostrophe reste dans le nom feuille (la partie porteuse du test). --- tests/testthat/test-arrow-dataset-to-duckdb.R | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/testthat/test-arrow-dataset-to-duckdb.R b/tests/testthat/test-arrow-dataset-to-duckdb.R index eb47722..dc4bd4a 100644 --- a/tests/testthat/test-arrow-dataset-to-duckdb.R +++ b/tests/testthat/test-arrow-dataset-to-duckdb.R @@ -7,9 +7,11 @@ test_that("a Parquet path containing a quote works end to end", { skip_if_not_installed("duckdb") skip_if_not_installed("dbplyr") - d <- file.path(tempdir(), "l'export datadiff") - dir.create(d, showWarnings = FALSE) - on.exit(unlink(d, recursive = TRUE), add = TRUE) + # Unique base dir per run (parallel/leftover safety), apostrophe kept in + # the leaf name since that is the load-bearing part of the test + d <- file.path(tempfile(pattern = "datadiff_quote_"), "l'export datadiff") + dir.create(d, recursive = TRUE, showWarnings = FALSE) + on.exit(unlink(dirname(d), recursive = TRUE), add = TRUE) arrow::write_parquet( data.frame(id = 1:3, x = c(1.0, 2.0, 3.0)), file.path(d, "part-0.parquet") @@ -25,7 +27,7 @@ test_that("multi-file datasets with shifted schemas are unified by name", { skip_if_not_installed("duckdb") skip_if_not_installed("dbplyr") - d <- file.path(tempdir(), "datadiff_union_by_name") + d <- tempfile(pattern = "datadiff_union_by_name_") dir.create(d, showWarnings = FALSE) on.exit(unlink(d, recursive = TRUE), add = TRUE) # Same columns, different physical order in the second file: a positional From 94cb73370b654c62dac391959ffb33e96b884966 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 00:20:33 +0200 Subject: [PATCH 19/40] fix: report.R durci - memoisation partagee, eval_error propage, constantes centralisees (issue #29) - datadiff_report_html() passe par datadiff_render_report() : lit le cache alimente par print() et l'alimente en retour (plus de double construction agent + rapport) - un eval_error sur l'agent reel (n_failed NA) prend la branche reelle du build (le garde any(n_failed > 0) seul routait silencieusement vers la branche synthetique all-pass) et survit au bloc vectorise (qui forcait eval_error/eval_warning a FALSE sur toutes les lignes) - bloc mort supprime dans build_report_agent : les injections de compteurs par ligne du else etaient integralement reecrites par le bloc vectorise authoritative-from-coverage - R/constants.R : suffixes __ok/__eq, prefixes __missing_col_/ __type_mismatch_ et defauts warn_at/stop_at definis une fois et partages par producteurs, consommateurs du verdict et mapping du rapport ; helpers datadiff_ok_col()/datadiff_eq_col() --- NEWS.md | 13 +++++ R/compare_datasets_from_yaml.R | 8 +-- R/constants.R | 31 ++++++++++ R/fast_path.R | 11 ++-- R/pointblank_setup.R | 12 ++-- R/report.R | 77 +++++++++++++------------ R/tolerance.R | 10 ++-- tests/testthat/test-report-robustness.R | 71 +++++++++++++++++++++++ 8 files changed, 176 insertions(+), 57 deletions(-) create mode 100644 R/constants.R create mode 100644 tests/testthat/test-report-robustness.R diff --git a/NEWS.md b/NEWS.md index f99cb36..af43fca 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,19 @@ ## Robustness +* Report construction hardening (all internal to `R/report.R`): + `datadiff_report_html()` now shares the `print()` memoization (it reads the + cached report and feeds the cache, instead of rebuilding agent + report on + every call); a genuine evaluation error on the real agent (`eval_error`, + where `n_failed` is NA) is propagated to the report instead of being + silently replaced by the synthetic all-pass branch and forced to FALSE; a + dead per-row count-injection block was removed (its fields were entirely + overwritten by the vectorised coverage block); and the `__ok`/`__eq`/ + `__missing_col_`/`__type_mismatch_` naming conventions plus the default + warn/stop levels are now defined once in `R/constants.R` and shared by the + producers, the verdict consumers and the report mapping, so a one-sided + rename can no longer silently break the step mapping (issue #29). + * `arrow_dataset_to_duckdb()` builds its SQL safely: Parquet file paths are quoted with `DBI::dbQuoteString()` (a path containing a quote, common in French like `l'export/`, broke the query with a raw SQL syntax error), the diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 24d2a56..e551c13 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -312,7 +312,7 @@ compare_datasets_from_yaml <- function(data_reference, data_candidate, key = NULL, path = NULL, - warn_at = 0.00000000000001, stop_at = 0.00000000000001, + warn_at = 1e-14, stop_at = 1e-14, ref_suffix = "__reference", label = NULL, error_msg_no_key = "Without keys, both tables must have the same number of rows.", @@ -671,8 +671,8 @@ compare_datasets_from_yaml <- function(data_reference, paste0(cols, suffix) } val_cols <- c( - suffix_all(tol_cols, suffix = "__ok"), - suffix_all(eq_cols, suffix = "__eq"), + suffix_all(tol_cols, suffix = datadiff_suffix_ok), + suffix_all(eq_cols, suffix = datadiff_suffix_eq), if (isTRUE(row_validation_info$check_count)) { "row_count_ok" } else { @@ -758,7 +758,7 @@ compare_datasets_from_yaml <- function(data_reference, # express these semantics. if (!is_lazy && length(fail$eq) > 0) { for (c in fail$eq) { - cmp_for_agent[[paste0(c, "__eq")]] <- eq_col_bool( + cmp_for_agent[[datadiff_eq_col(c)]] <- eq_col_bool( cmp_for_agent, col = c, ref_suffix = ref_suffix, na_equal = na_equal ) } diff --git a/R/constants.R b/R/constants.R new file mode 100644 index 0000000..97d7dd9 --- /dev/null +++ b/R/constants.R @@ -0,0 +1,31 @@ +# Internal naming conventions shared by the boolean producers (tolerance.R, +# compare_datasets_from_yaml.R), the verdict consumers (fast_path.R, +# coverage.R) and the report step mapping (report.R, pointblank_setup.R). +# They live in one place because a drift on one side breaks the mapping +# SILENTLY: the report step is simply not found and its extract is lost. + +# Suffix of the per-column boolean produced for a tolerance column. +datadiff_suffix_ok <- "__ok" + +# Suffix of the per-column boolean produced for an equality column. +datadiff_suffix_eq <- "__eq" + +# Prefix of the dummy column carrying a missing-column failure step. +datadiff_prefix_missing_col <- "__missing_col_" + +# Prefix of the dummy column carrying a type-mismatch failure step. +datadiff_prefix_type_mismatch <- "__type_mismatch_" + +# Column name helpers: the only supported way to build these names. +datadiff_ok_col <- function(col) { + paste0(col, datadiff_suffix_ok) +} +datadiff_eq_col <- function(col) { + paste0(col, datadiff_suffix_eq) +} + +# Default pointblank action levels (fractions of failing rows). Used as the +# internal fallback everywhere a threshold is reconstructed from attributes; +# the public compare_datasets_from_yaml() signature documents the same value. +datadiff_default_warn_at <- 1e-14 +datadiff_default_stop_at <- 1e-14 diff --git a/R/fast_path.R b/R/fast_path.R index e8af144..9216238 100644 --- a/R/fast_path.R +++ b/R/fast_path.R @@ -14,9 +14,10 @@ # error: all(NULL) is TRUE, so returning NULL would silently turn a dropped # column into a false all-pass. tol_col_bool <- function(tbl, col) { - b <- tbl[[paste0(col, "__ok")]] + b <- tbl[[datadiff_ok_col(col)]] if (is.null(b)) { - stop(sprintf("internal error: boolean column '%s__ok' missing", col), + stop(sprintf("internal error: boolean column '%s' missing", + datadiff_ok_col(col)), call. = FALSE) } b @@ -26,7 +27,7 @@ tol_col_bool <- function(tbl, col) { # the pre-computed __eq vector (lazy path) or, when absent (local path), # the raw comparison with one-sided NA = FALSE and two-sided NA = na_equal. eq_col_bool <- function(tbl, col, ref_suffix, na_equal) { - eq_precomputed <- tbl[[paste0(col, "__eq")]] + eq_precomputed <- tbl[[datadiff_eq_col(col)]] if (!is.null(eq_precomputed)) { return(eq_precomputed) } @@ -44,8 +45,8 @@ eq_col_bool <- function(tbl, col, ref_suffix, na_equal) { } ) stop(sprintf( - "internal error: equality column(s) %s missing (no precomputed '%s__eq' either)", - paste(missing_cols, collapse = " and "), col + "internal error: equality column(s) %s missing (no precomputed '%s' either)", + paste(missing_cols, collapse = " and "), datadiff_eq_col(col) ), call. = FALSE) } cand_na <- is.na(cand_vals) diff --git a/R/pointblank_setup.R b/R/pointblank_setup.R index 29d156f..b3dc16c 100644 --- a/R/pointblank_setup.R +++ b/R/pointblank_setup.R @@ -47,7 +47,7 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, # Add dummy columns for missing columns BEFORE creating the agent # These columns are set to FALSE and we'll check they equal TRUE (will fail) for (c in missing_in_candidate) { - dummy_col <- paste0("__missing_col_", c) + dummy_col <- paste0(datadiff_prefix_missing_col, c) if (is_non_local(cmp)) { cmp <- dplyr::mutate(cmp, !!dummy_col := FALSE) } else { @@ -58,7 +58,7 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, # Add dummy FALSE columns for type-mismatched columns. # These will generate a dedicated failing validation step per column. for (c in type_mismatch_cols) { - dummy_col <- paste0("__type_mismatch_", c) + dummy_col <- paste0(datadiff_prefix_type_mismatch, c) if (is_non_local(cmp)) { cmp <- dplyr::mutate(cmp, !!dummy_col := FALSE) } else { @@ -83,7 +83,7 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, # For missing columns, add a validation that will fail for (c in missing_in_candidate) { - dummy_col <- paste0("__missing_col_", c) + dummy_col <- paste0(datadiff_prefix_missing_col, c) agent <- agent %>% col_vals_equal( columns = all_of(dummy_col), @@ -95,7 +95,7 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, # For type-mismatched columns, add a validation that will always fail. for (c in type_mismatch_cols) { - dummy_col <- paste0("__type_mismatch_", c) + dummy_col <- paste0(datadiff_prefix_type_mismatch, c) agent <- agent %>% col_vals_equal( columns = all_of(dummy_col), @@ -107,7 +107,7 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, eq_cols <- setdiff(x = common_cols, y = tol_cols) for (c in eq_cols) { - eq_precomputed <- paste0(c, "__eq") + eq_precomputed <- datadiff_eq_col(c) if (eq_precomputed %in% get_col_names(cmp)) { # Lazy path: use pre-computed boolean equality column agent <- agent %>% @@ -124,7 +124,7 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, } for (c in tol_cols) { - ok_col <- paste0(c, "__ok") + ok_col <- datadiff_ok_col(c) agent <- agent %>% col_vals_equal(columns = all_of(ok_col), value = TRUE, na_pass = FALSE) } diff --git a/R/report.R b/R/report.R index 56c70f8..e07f5b5 100644 --- a/R/report.R +++ b/R/report.R @@ -14,13 +14,13 @@ report_underlying_col <- function(col) { if (identical(col, "row_count_ok")) { return("") } - if (startsWith(col, "__missing_col_")) { - return(sub("^__missing_col_", "", col)) + if (startsWith(col, datadiff_prefix_missing_col)) { + return(sub(paste0("^", datadiff_prefix_missing_col), "", x = col)) } - if (startsWith(col, "__type_mismatch_")) { - return(sub("^__type_mismatch_", "", col)) + if (startsWith(col, datadiff_prefix_type_mismatch)) { + return(sub(paste0("^", datadiff_prefix_type_mismatch), "", x = col)) } - sub("__(ok|eq)$", "", col) + sub(sprintf("(%s|%s)$", datadiff_suffix_ok, datadiff_suffix_eq), "", x = col) } # Build a pointblank agent whose report mirrors the coverage table. @@ -33,17 +33,20 @@ report_underlying_col <- function(col) { # synthetic passing row is added for the rest. Without `real_agent`, a purely # synthetic count-only agent is produced (no extracts). build_report_agent <- function(coverage, label, lang = "fr", locale = "fr_FR", - warn_at = 1e-14, stop_at = 1e-14, + warn_at = datadiff_default_warn_at, stop_at = datadiff_default_stop_at, real_agent = NULL) { n <- nrow(coverage) # Augment the real interrogated agent only when it has genuine failures - # (real extracts to preserve). On an all-pass comparison the real agent is the + # (real extracts to preserve) or evaluation errors (n_failed is NA there, so + # the count test alone would silently route a broken interrogation to the + # synthetic all-pass branch). On an all-pass comparison the real agent is the # minimal placeholder (a single col_exists step); cloning its template would # mislabel every value check as col_exists, so fall through to the synthetic # col_vals_equal build instead. if (!is.null(real_agent) && - any(real_agent$validation_set$n_failed > 0, na.rm = TRUE)) { + (any(real_agent$validation_set$n_failed > 0, na.rm = TRUE) || + any(real_agent$validation_set$eval_error, na.rm = TRUE))) { rvs <- real_agent$validation_set real_cols <- vapply(seq_len(nrow(rvs)), function(j) { report_underlying_col(rvs$column[[j]][1]) @@ -55,6 +58,8 @@ build_report_agent <- function(coverage, label, lang = "fr", locale = "fr_FR", template <- rvs[1, , drop = FALSE] rows <- vector("list", n) new_extracts <- list() + real_eval_error <- logical(n) + real_eval_warning <- logical(n) for (i in seq_len(n)) { col <- coverage$column[i] # Only value checks (tolerance / equality) map to a real step: that is @@ -62,34 +67,30 @@ build_report_agent <- function(coverage, label, lang = "fr", locale = "fr_FR", # structural checks (missing_column, type_mismatch, row_count) are # synthesized from coverage - mapping them to a real step would pull in # per-row dummy results (e.g. n_failed = nrow) that contradict coverage. + # Consequence: an eval_error on a structural step is intentionally not + # surfaced per-column (only value-step evaluation errors are). j <- if (coverage$check[i] %in% c("tolerance", "equality")) { match(col, real_cols) } else { NA_integer_ } if (!is.na(j)) { - # Genuine interrogated step: keep it, and carry its real extract over to - # the new step index. + # Genuine interrogated step: keep it, carry its real extract over to + # the new step index, and remember its evaluation outcome (the + # vectorised coverage overwrite below must not mask a real + # eval_error/eval_warning behind FALSE). row <- rvs[j, , drop = FALSE] + real_eval_error[i] <- isTRUE(rvs$eval_error[j]) + real_eval_warning[i] <- isTRUE(rvs$eval_warning[j]) old_key <- as.character(rvs$i[j]) if (!is.null(old_extracts[[old_key]])) { new_extracts[[as.character(i)]] <- old_extracts[[old_key]] } } else { # Column the targeted agent did not validate (it passed): synthesise a - # passing row from the real-row template. + # passing row from the real-row template. Every count/verdict field is + # set authoritatively from coverage in the vectorised block below. row <- template - row$eval_error <- FALSE - row$eval_warning <- FALSE - row$n <- as.numeric(coverage$n[i]) - row$n_passed <- as.numeric(coverage$n[i] - coverage$n_failed[i]) - row$n_failed <- as.numeric(coverage$n_failed[i]) - row$f_passed <- if (coverage$n[i] > 0) row$n_passed / coverage$n[i] else 1 - row$f_failed <- if (coverage$n[i] > 0) coverage$n_failed[i] / coverage$n[i] else 0 - row$all_passed <- coverage$n_failed[i] == 0L - row$warn <- FALSE - row$stop <- FALSE - row$notify <- FALSE } row$column <- list(coverage$column[i]) row$label <- coverage$check[i] @@ -113,8 +114,8 @@ build_report_agent <- function(coverage, label, lang = "fr", locale = "fr_FR", new_vs$warn <- coverage$n_failed > 0L & new_vs$f_failed >= warn_at new_vs$stop <- coverage$n_failed > 0L & new_vs$f_failed >= stop_at new_vs$notify <- rep(FALSE, nrow(new_vs)) - new_vs$eval_error <- rep(FALSE, nrow(new_vs)) - new_vs$eval_warning <- rep(FALSE, nrow(new_vs)) + new_vs$eval_error <- real_eval_error + new_vs$eval_warning <- real_eval_warning real_agent$validation_set <- new_vs real_agent$extracts <- new_extracts return(real_agent) @@ -180,7 +181,7 @@ mark_agent_interrogated <- function(agent) { # class (print dispatches to print.datadiff_report) and stash what is needed to # render the full report, plus a (reference-semantics) environment for caching. as_datadiff_report <- function(reponse, coverage, label, lang, locale, - warn_at = 1e-14, stop_at = 1e-14) { + warn_at = datadiff_default_warn_at, stop_at = datadiff_default_stop_at) { attr(reponse, "datadiff_coverage") <- coverage attr(reponse, "datadiff_label") <- label attr(reponse, "datadiff_lang") <- lang @@ -205,8 +206,8 @@ datadiff_render_report <- function(x) { label = attr(x, "datadiff_label") %||% "datadiff report", lang = attr(x, "datadiff_lang") %||% "fr", locale = attr(x, "datadiff_locale") %||% "fr_FR", - warn_at = attr(x, "datadiff_warn_at") %||% 1e-14, - stop_at = attr(x, "datadiff_stop_at") %||% 1e-14, + warn_at = attr(x, "datadiff_warn_at") %||% datadiff_default_warn_at, + stop_at = attr(x, "datadiff_stop_at") %||% datadiff_default_stop_at, real_agent = x ) ) @@ -251,16 +252,18 @@ datadiff_report_html <- function(res, file = NULL) { stop("`res` has no `coverage`; was it produced by compare_datasets_from_yaml()?") } reponse <- res$reponse - agent <- build_report_agent( - coverage = coverage, - label = attr(reponse, "datadiff_label") %||% "datadiff report", - lang = attr(reponse, "datadiff_lang") %||% "fr", - locale = attr(reponse, "datadiff_locale") %||% "fr_FR", - warn_at = attr(reponse, "datadiff_warn_at") %||% 1e-14, - stop_at = attr(reponse, "datadiff_stop_at") %||% 1e-14, - real_agent = reponse - ) - report <- pointblank::get_agent_report(agent) + report <- if (inherits(reponse, "datadiff_report")) { + # Share the print() memoization: read the cached report when a print + # already built it, and feed the cache otherwise + datadiff_render_report(reponse) + } else { + # No lazy-report wrapper (hand-assembled res): one-shot synthetic build + pointblank::get_agent_report(build_report_agent( + coverage = coverage, + label = "datadiff report", + real_agent = reponse + )) + } if (!is.null(file)) { pointblank::export_report(report, filename = file, quiet = TRUE) } diff --git a/R/tolerance.R b/R/tolerance.R index af9a2d9..2b7e604 100644 --- a/R/tolerance.R +++ b/R/tolerance.R @@ -96,7 +96,7 @@ add_ok_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { na_equal = na_equal ) } - names(ok_cols) <- paste0(tol_cols, "__ok") + names(ok_cols) <- datadiff_ok_col(tol_cols) cbind(cmp, list2DF(ok_cols)) } @@ -204,7 +204,7 @@ add_bool_cols_sql <- function(cmp, tol_cols, eq_cols, col_rules, ref_suffix, rt <- col_rules[[c]][["rel"]] %||% 0 within <- sprintf("ABS(%s - %s) <= (%s + %s * ABS(%s)) + %s * ABS(%s)", cc, rc, num(at), num(rt), rc, num(fp_eps), rc) - sprintf("%s AS %s", case_tol(cc, rc, cond = within), q(paste0(c, "__ok"))) + sprintf("%s AS %s", case_tol(cc, rc, cond = within), q(datadiff_ok_col(c))) }, FUN.VALUE = character(1), USE.NAMES = FALSE) eq_exprs <- vapply(X = eq_cols, FUN = function(c) { @@ -212,7 +212,7 @@ add_bool_cols_sql <- function(cmp, tol_cols, eq_cols, col_rules, ref_suffix, rc <- q(paste0(c, ref_suffix)) sprintf("%s AS %s", case_eq(cc, rc, nan_aware = c %in% eq_num_cols), - q(paste0(c, "__eq"))) + q(datadiff_eq_col(c))) }, FUN.VALUE = character(1), USE.NAMES = FALSE) exprs <- c(tol_exprs, eq_exprs) @@ -258,7 +258,7 @@ add_tolerance_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal rc_sym <- dplyr::sym(reference_c) absdiff_col <- paste0(c, "__absdiff") thresh_col <- paste0(c, "__thresh") - ok_col <- paste0(c, "__ok") + ok_col <- datadiff_ok_col(c) absdiff_sym <- dplyr::sym(absdiff_col) thresh_sym <- dplyr::sym(thresh_col) @@ -323,7 +323,7 @@ add_tolerance_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal names(absdiff_cols) <- paste0(tol_cols, "__absdiff") names(thresh_cols) <- paste0(tol_cols, "__thresh") - names(ok_cols) <- paste0(tol_cols, "__ok") + names(ok_cols) <- datadiff_ok_col(tol_cols) cbind(cmp, list2DF(c(absdiff_cols, thresh_cols, ok_cols))) } diff --git a/tests/testthat/test-report-robustness.R b/tests/testthat/test-report-robustness.R new file mode 100644 index 0000000..cc690a7 --- /dev/null +++ b/tests/testthat/test-report-robustness.R @@ -0,0 +1,71 @@ +# Report construction robustness: the HTML export shares the print() +# memoization, and a genuine evaluation error on the real agent must reach the +# report instead of being silently replaced by synthetic passing rows. + +test_that("datadiff_report_html reads and feeds the print() memoization", { + ref <- data.frame(id = 1:2, x = c(1, 2)) + cand <- data.frame(id = 1:2, x = c(1, 3)) + res <- suppressMessages(compare_datasets_from_yaml(ref, cand, key = "id")) + + cache <- attr(res$reponse, "datadiff_render") + expect_true(is.environment(cache)) + expect_null(cache$report) + + # Rendering through the HTML entry point must populate the shared cache... + r1 <- datadiff_report_html(res, file = NULL) + expect_false(is.null(cache$report)) + + # ...and a second render must return the memoized object itself + r2 <- datadiff_report_html(res, file = NULL) + expect_identical(r2, cache$report) +}) + +test_that("an eval_error on the real agent reaches the report", { + ref <- data.frame(id = 1:2, x = c(1, 2)) + cand <- data.frame(id = 1:2, x = c(1, 3)) + res <- suppressMessages(compare_datasets_from_yaml(ref, cand, key = "id")) + + # Simulate a step whose interrogation failed to evaluate: pointblank stores + # eval_error = TRUE and n_failed = NA for such steps. Target the x__ok value + # step (the one the report maps back to the "x" coverage row). + vs <- res$reponse$validation_set + err_idx <- which(vapply(vs$column, function(cc) { + identical(cc[1], "x__ok") + }, logical(1)))[1] + vs$eval_error[err_idx] <- TRUE + vs$n_failed[err_idx] <- NA_real_ + res$reponse$validation_set <- vs + + agent <- build_report_agent( + coverage = res$coverage, + label = "eval error propagation", + real_agent = res$reponse + ) + + # The synthetic branch must not swallow the failure: the rebuilt validation + # set still carries the eval_error of the mapped real step + expect_true(any(agent$validation_set$eval_error, na.rm = TRUE)) +}) + +test_that("an all-NA n_failed agent (pure eval_error) is not treated as all-pass", { + ref <- data.frame(id = 1:2, x = c(1, 2)) + cand <- data.frame(id = 1:2, x = c(1, 3)) + res <- suppressMessages(compare_datasets_from_yaml(ref, cand, key = "id")) + + # Every real step errored: n_failed is NA everywhere, eval_error TRUE + vs <- res$reponse$validation_set + vs$eval_error <- rep(TRUE, nrow(vs)) + vs$n_failed <- rep(NA_real_, nrow(vs)) + res$reponse$validation_set <- vs + + agent <- build_report_agent( + coverage = res$coverage, + label = "pure eval error", + real_agent = res$reponse + ) + + # Pre-fix, the any(n_failed > 0, na.rm = TRUE) guard was FALSE and the + # synthetic count-only branch silently dropped the real agent (and its + # extracts); the eval_error must survive instead + expect_true(any(agent$validation_set$eval_error, na.rm = TRUE)) +}) From e7be28753f500b325a61f28727adceed3c985006 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 00:33:58 +0200 Subject: [PATCH 20/40] perf: fast-path intermediaire NA-sans-Inf dans compute_tolerance_ok (issue #23) Une colonne avec NA mais sans infini ne bascule plus sur le kernel complet (~15 passes) : la comparaison brute donne NA exactement ou un NA/NaN est implique, donc ok[is.na(ok)] <- FALSE + correction du NA bilateral reproduit le kernel. NaN n'a pas besoin de garde dediee (is.na couvre NaN, memes regles des deux cotes) ; seul Inf reroute vers le kernel (abs(Inf - Inf) vaut NaN mais les infinis de meme signe doivent passer). Resultat bit-identique, verrouille par tests etendus (NA epars, NaN, Inf, na_equal TRUE/FALSE), ~2,6x plus rapide par colonne porteuse de NA (200k lignes). Le kernel complet perd ses masques NaN redondants (both_nan/one_nan sous-ensembles des masques NA), resultat inchange. Refactor de perf pur : le verrou TDD est la suite d'equivalence bit-a-bit (verte avant et apres), pas un rouge. --- NEWS.md | 11 ++++++++ R/tolerance.R | 43 ++++++++++++++++++++++-------- tests/testthat/test-tolerance-ok.R | 35 ++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index af43fca..5027d97 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,16 @@ # datadiff (development version) +## Performance + +* `compute_tolerance_ok()` gains an intermediate fast path for the most common + real-data case: a column with NA but no infinity no longer falls back to the + full special-value kernel. NaN needs no dedicated handling there (`is.na()` + covers NaN, so NaN follows the NA rules identically on both paths); only + infinities reroute to the kernel. Bit-identical results, locked by tests; + measured ~2.6x faster per NA-bearing column (200k rows, 0.008 s vs 0.020 s). + The full kernel itself drops its redundant NaN masks (`both_nan`/`one_nan` + are subsets of the NA masks), with unchanged results (issue #23). + ## Robustness * Report construction hardening (all internal to `R/report.R`): diff --git a/R/tolerance.R b/R/tolerance.R index 2b7e604..0dae6c0 100644 --- a/R/tolerance.R +++ b/R/tolerance.R @@ -23,12 +23,13 @@ #' logical vector \code{ok}. #' @noRd compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal) { + # is.na() covers NaN in R, so the NA masks subsume the NaN ones: NaN-specific + # masks would be redundant (both_nan is a subset of both_na, one_nan of + # one_na) and are not computed. both_na <- is.na(cand_vals) & is.na(ref_vals) one_na <- is.na(cand_vals) | is.na(ref_vals) both_inf <- is.infinite(cand_vals) & is.infinite(ref_vals) same_inf <- both_inf & (sign(cand_vals) == sign(ref_vals)) - both_nan <- is.nan(cand_vals) & is.nan(ref_vals) - one_nan <- is.nan(cand_vals) | is.nan(ref_vals) one_inf <- is.infinite(cand_vals) | is.infinite(ref_vals) absdiff <- abs(cand_vals - ref_vals) @@ -41,10 +42,9 @@ compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equa within_tol <- absdiff <= thresh + fp_correction if (na_equal) { - ok <- both_na | both_nan | same_inf | - (!one_na & !one_nan & !one_inf & within_tol) + ok <- both_na | same_inf | (!one_na & !one_inf & within_tol) } else { - ok <- same_inf | (!one_na & !one_nan & !one_inf & within_tol) + ok <- same_inf | (!one_na & !one_inf & within_tol) } list(absdiff = absdiff, thresh = thresh, ok = ok) @@ -52,12 +52,17 @@ compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equa #' Within-tolerance boolean for one column (hot path, ok only) #' -#' Returns ONLY the boolean within-tolerance vector (not absdiff/thresh), with a -#' fast path for the common case of a column with no NA/NaN/Inf: the dozen -#' special-value passes of `compute_tolerance_col()` are skipped and the result -#' reduces to `abs(cand - ref) <= thresh + fp`. Falls back to the full kernel -#' (taking only its `$ok`) when special values are present, so the result is -#' identical to `compute_tolerance_col(...)$ok` in every case. +#' Returns ONLY the boolean within-tolerance vector (not absdiff/thresh), with +#' two fast paths: +#' - no NA/NaN/Inf at all: the result reduces to +#' `abs(cand - ref) <= thresh + fp` (3 vector passes); +#' - NA present but no NaN/Inf (the most common real-data case): the raw +#' comparison yields NA exactly where an NA is involved, so one +#' `ok[is.na(ok)] <- FALSE` pass plus the two-sided-NA correction reproduces +#' the kernel at a fraction of its passes. +#' +#' Anything with NaN/Inf falls back to the full kernel (taking only its `$ok`), +#' so the result is identical to `compute_tolerance_col(...)$ok` in every case. #' #' @inheritParams compute_tolerance_col #' @return Logical vector, the same as `compute_tolerance_col(...)$ok`. @@ -69,6 +74,22 @@ compute_tolerance_ok <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal fp <- 8 * .Machine$double.eps * abs(ref_vals) return(abs(cand_vals - ref_vals) <= thresh + fp) } + if (!any(is.infinite(cand_vals)) && !any(is.infinite(ref_vals))) { + # NA/NaN present but no Inf. NaN needs no dedicated guard: is.na() covers + # NaN in R, so NaN follows the NA rules identically here and in the + # kernel. Only Inf diverges (abs(Inf - Inf) is NaN, but same-sign + # infinities must PASS), hence the infinity-only fallback test. + thresh <- abs_tol + rel_tol * abs(ref_vals) + fp <- 8 * .Machine$double.eps * abs(ref_vals) + ok <- abs(cand_vals - ref_vals) <= thresh + fp + # NA wherever an NA/NaN was involved: a one-sided one is a difference... + ok[is.na(ok)] <- FALSE + # ...and a two-sided one follows na_equal + if (na_equal) { + ok[is.na(cand_vals) & is.na(ref_vals)] <- TRUE + } + return(ok) + } compute_tolerance_col(cand_vals, ref_vals, abs_tol, rel_tol, na_equal)$ok } diff --git a/tests/testthat/test-tolerance-ok.R b/tests/testthat/test-tolerance-ok.R index b3f9677..cc6fa4b 100644 --- a/tests/testthat/test-tolerance-ok.R +++ b/tests/testthat/test-tolerance-ok.R @@ -30,6 +30,41 @@ test_that("special values fall back and still match the full kernel", { expect_ok_equiv(Inf, Inf, 0.1, 0, FALSE) }) +test_that("intermediate path (NA but no NaN/Inf) is bit-identical to the kernel", { + set.seed(7) + n <- 500 + ref <- rnorm(n) + cand <- ref + 1e-12 + cand[sample(n, 25)] <- NA_real_ + ref[sample(n, 25)] <- NA_real_ # one-sided NAs plus a few two-sided ones + ref[1] <- NA_real_; cand[1] <- NA_real_ # guarantee at least one two-sided NA + + for (na_equal in c(TRUE, FALSE)) { + expect_identical( + compute_tolerance_ok(cand, ref, 1e-9, 0, na_equal), + compute_tolerance_col(cand, ref, 1e-9, 0, na_equal)$ok + ) + expect_identical( + compute_tolerance_ok(cand, ref, 0, 0.01, na_equal), + compute_tolerance_col(cand, ref, 0, 0.01, na_equal)$ok + ) + } + + # A single NaN or Inf must reroute to the full kernel, identically + cand_nan <- cand; cand_nan[2] <- NaN + cand_inf <- cand; cand_inf[3] <- Inf + for (na_equal in c(TRUE, FALSE)) { + expect_identical( + compute_tolerance_ok(cand_nan, ref, 1e-9, 0, na_equal), + compute_tolerance_col(cand_nan, ref, 1e-9, 0, na_equal)$ok + ) + expect_identical( + compute_tolerance_ok(cand_inf, ref, 1e-9, 0, na_equal), + compute_tolerance_col(cand_inf, ref, 1e-9, 0, na_equal)$ok + ) + } +}) + test_that("randomised property check: ok-only equals full kernel", { set.seed(42) pool <- c(rnorm(40), NA, NaN, Inf, -Inf) From 211f0facb89a1f8c25bbdb7a722df46dc58ad445 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 00:56:59 +0200 Subject: [PATCH 21/40] docs: la doc du fast-path intermediaire reflete la garde Inf-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reponse a la review Copilot de la PR : la roxygen et les commentaires de test disaient encore « NA mais pas NaN/Inf » alors que l'implementation finale laisse passer NaN par le chemin intermediaire (is.na couvre NaN, memes regles des deux cotes) et ne reroute vers le kernel que les infinis. --- R/tolerance.R | 14 ++++++++------ tests/testthat/test-tolerance-ok.R | 5 +++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/R/tolerance.R b/R/tolerance.R index 0dae6c0..e201323 100644 --- a/R/tolerance.R +++ b/R/tolerance.R @@ -56,13 +56,15 @@ compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equa #' two fast paths: #' - no NA/NaN/Inf at all: the result reduces to #' `abs(cand - ref) <= thresh + fp` (3 vector passes); -#' - NA present but no NaN/Inf (the most common real-data case): the raw -#' comparison yields NA exactly where an NA is involved, so one -#' `ok[is.na(ok)] <- FALSE` pass plus the two-sided-NA correction reproduces -#' the kernel at a fraction of its passes. +#' - NA and/or NaN present but no infinity (the most common real-data case): +#' the raw comparison yields NA exactly where an NA or NaN is involved, so +#' one `ok[is.na(ok)] <- FALSE` pass plus the two-sided correction +#' reproduces the kernel at a fraction of its passes (`is.na()` covers NaN, +#' so NaN follows the NA rules identically on both paths). #' -#' Anything with NaN/Inf falls back to the full kernel (taking only its `$ok`), -#' so the result is identical to `compute_tolerance_col(...)$ok` in every case. +#' Only infinities fall back to the full kernel (taking only its `$ok`; +#' `abs(Inf - Inf)` is NaN but same-sign infinities must PASS), so the result +#' is identical to `compute_tolerance_col(...)$ok` in every case. #' #' @inheritParams compute_tolerance_col #' @return Logical vector, the same as `compute_tolerance_col(...)$ok`. diff --git a/tests/testthat/test-tolerance-ok.R b/tests/testthat/test-tolerance-ok.R index cc6fa4b..788a2fd 100644 --- a/tests/testthat/test-tolerance-ok.R +++ b/tests/testthat/test-tolerance-ok.R @@ -30,7 +30,7 @@ test_that("special values fall back and still match the full kernel", { expect_ok_equiv(Inf, Inf, 0.1, 0, FALSE) }) -test_that("intermediate path (NA but no NaN/Inf) is bit-identical to the kernel", { +test_that("intermediate path (NA/NaN, no infinity) is bit-identical to the kernel", { set.seed(7) n <- 500 ref <- rnorm(n) @@ -50,7 +50,8 @@ test_that("intermediate path (NA but no NaN/Inf) is bit-identical to the kernel" ) } - # A single NaN or Inf must reroute to the full kernel, identically + # NaN stays on the intermediate path (is.na covers it); an infinity + # reroutes to the full kernel. Both must match the kernel bit for bit. cand_nan <- cand; cand_nan[2] <- NaN cand_inf <- cand; cand_inf[3] <- Inf for (na_equal in c(TRUE, FALSE)) { From 7e21d57608f4852093904702f99242d0e0f5f35a Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 00:40:19 +0200 Subject: [PATCH 22/40] perf: une seule passe sur les booleens de validation (issue #21) build_coverage() passe en premier et devient l'unique balayage des booleens : le verdict (all(coverage$n_failed == 0L), les lignes structurelles y figurent deja) et les ensembles de colonnes en echec (filtre sur coverage) en derivent. Avant : 2 balayages complets en cas vert (all_validations_pass + build_coverage), ~3 en cas rouge (+ failing_columns), chaque passe recalculant les booleens d'egalite locaux. Les scanners devenus morts sont supprimes (all_validations_pass, failing_columns, tol_col_passes, eq_col_passes) ainsi que leur bloc de test dedie. Verdicts byte-identiques (garde d'equivalence vert, suite complete 919 PASS). ~6% bout-en-bout sur cas vert 200 colonnes x 200k (4,35 s -> 4,07 s), le join et le preprocessing dominant le reste. --- NEWS.md | 12 ++++ R/compare_datasets_from_yaml.R | 48 ++++++++-------- R/fast_path.R | 61 --------------------- tests/testthat/test-boolean-column-guards.R | 9 --- tests/testthat/test-coverage.R | 2 +- 5 files changed, 37 insertions(+), 95 deletions(-) diff --git a/NEWS.md b/NEWS.md index 5027d97..58021e0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,18 @@ ## Performance +* The verdict, the coverage and the failing-column sets now come from a + SINGLE pass over the boolean validation columns: `build_coverage()` runs + first and everything else derives from it (its rows already carry the + structural checks). Previously the same booleans were scanned twice on a + green comparison and three times on a red one, and each scan recomputed the + local equality booleans from scratch. The now-unused scanners + (`all_validations_pass()`, `failing_columns()`, the `*_col_passes()` + predicates) are removed. Verdicts are byte-identical (equivalence guard + green); measured ~6% end to end on a green 200-column x 200k comparison + (4.35 s to 4.07 s), the join and preprocessing dominating the rest + (issue #21). + * `compute_tolerance_ok()` gains an intermediate fast path for the most common real-data case: a column with NA but no infinity no longer falls back to the full special-value kernel. NaN needs no dedicated handling there (`is.na()` diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index e551c13..588d4cc 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -706,27 +706,11 @@ compare_datasets_from_yaml <- function(data_reference, cmp_for_agent <- dplyr::collect(cmp_slim_computed) } - # Fast all-pass short-circuit. - # The verdict is fully determined by the boolean validation columns already - # computed above (plus the structural checks). When everything passes there - # are no cells to extract, so the expensive per-column pointblank agent (one - # step per column, ~quadratic on wide tables) can be replaced by a constant - # cost trivially-passing agent. all_passed stays identical and - # get_data_extracts() is empty either way. Any failure falls through to the - # full per-column agent so failing cells remain extractable byte-for-byte. - all_passed_fast <- - length(missing_in_candidate) == 0 && - length(type_mismatch_cols) == 0 && - isTRUE(row_count_ok) && - all_validations_pass( - tbl = cmp_for_agent, tol_cols = tol_cols, eq_cols = eq_cols, - ref_suffix = ref_suffix, na_equal = na_equal - ) - - # Faithful, O(columns) record of every check performed, built from the same - # booleans the verdict is derived from. Always produced (green and red) so the - # caller can see what was verified even when the fast path skips the per-column - # pointblank agent. + # Faithful, O(columns) record of every check performed, built from the + # boolean validation columns computed above. It is the SINGLE pass over the + # data: the verdict and the failing-column sets below both derive from it, + # so the same booleans are no longer scanned two or three times (and the + # local equality booleans no longer recomputed once per scan). coverage <- build_coverage( tbl = cmp_for_agent, tol_cols = tol_cols, eq_cols = eq_cols, missing_in_candidate = missing_in_candidate, @@ -735,6 +719,20 @@ compare_datasets_from_yaml <- function(data_reference, ref_suffix = ref_suffix, na_equal = na_equal ) + # Fast all-pass short-circuit. + # The verdict is fully determined by the coverage, whose rows already + # include the structural checks (missing_column, type_mismatch, row_count). + # The row_count row only exists when check_count is TRUE; otherwise + # row_count_ok is TRUE by construction (unconditional init above), so the + # absent row is neutral. + # When everything passes there are no cells to extract, so the expensive + # per-column pointblank agent (one step per column, ~quadratic on wide + # tables) can be replaced by a constant cost trivially-passing agent. + # all_passed stays identical and get_data_extracts() is empty either way. + # Any failure falls through to the full per-column agent so failing cells + # remain extractable byte-for-byte. + all_passed_fast <- all(coverage$n_failed == 0L) + if (all_passed_fast) { agent <- build_pass_agent( tbl = cmp_for_agent, label = label, @@ -747,9 +745,11 @@ compare_datasets_from_yaml <- function(data_reference, # the per-column agent overhead on the passing majority. col_exists steps # (which only ever pass) are dropped for the same reason. Structural # failures (missing columns, type mismatches, row count) are always kept. - fail <- failing_columns( - tbl = cmp_for_agent, tol_cols = tol_cols, eq_cols = eq_cols, - ref_suffix = ref_suffix, na_equal = na_equal + # The failing sets come from the coverage (same booleans, already scanned). + fail_rows <- coverage[coverage$n_failed > 0L, , drop = FALSE] + fail <- list( + tol = fail_rows$column[fail_rows$check == "tolerance"], + eq = fail_rows$column[fail_rows$check == "equality"] ) # Local path: materialise the __eq boolean for the failing equality # columns so the pointblank step validates the exact boolean the verdict diff --git a/R/fast_path.R b/R/fast_path.R index 9216238..04f4756 100644 --- a/R/fast_path.R +++ b/R/fast_path.R @@ -60,67 +60,6 @@ eq_col_bool <- function(tbl, col, ref_suffix, na_equal) { out } -tol_col_passes <- function(tbl, col) { - isTRUE(all(tol_col_bool(tbl, col = col))) -} - -eq_col_passes <- function(tbl, col, ref_suffix, na_equal) { - isTRUE(all(eq_col_bool(tbl, col = col, ref_suffix = ref_suffix, na_equal = na_equal))) -} - -#' Fast global pass/fail without building the per-column pointblank agent -#' -#' The verdict produced by \code{compare_datasets_from_yaml} is entirely -#' determined by boolean columns already computed in a single vectorised pass -#' (\code{__ok} for tolerance columns, \code{__eq} for equality -#' columns on the lazy path). pointblank then re-checks those booleans through -#' one validation step per column, which dominates runtime on wide tables. When -#' every check passes there is nothing to extract, so the agent can be skipped. -#' -#' @param tbl Local data.frame holding the boolean validation columns (and, for -#' the local path, the raw and reference equality columns). -#' @param tol_cols Character vector of tolerance column names. -#' @param eq_cols Character vector of equality column names (already stripped of -#' key, type-mismatch and tolerance columns). -#' @param ref_suffix Suffix identifying reference columns. -#' @param na_equal Logical; whether NA equals NA for equality columns. -#' @return \code{TRUE} iff every tolerance and equality check passes. -#' @noRd -all_validations_pass <- function(tbl, tol_cols, eq_cols, ref_suffix, na_equal) { - for (c in tol_cols) { - if (!tol_col_passes(tbl, col = c)) { - return(FALSE) - } - } - for (c in eq_cols) { - if (!eq_col_passes(tbl, col = c, ref_suffix = ref_suffix, na_equal = na_equal)) { - return(FALSE) - } - } - TRUE -} - -#' Subset of columns whose validation fails -#' -#' Used on the failure path to build pointblank steps only for columns that -#' actually fail. Columns that pass produce empty data extracts, so omitting -#' their steps leaves \code{get_data_extracts()} unchanged while avoiding the -#' per-column agent overhead on the passing majority. -#' -#' @inheritParams all_validations_pass -#' @return A list with \code{tol} and \code{eq} character vectors of failing -#' column names. -#' @noRd -failing_columns <- function(tbl, tol_cols, eq_cols, ref_suffix, na_equal) { - fail_tol <- tol_cols[vapply(tol_cols, function(c) { - !tol_col_passes(tbl, col = c) - }, logical(1))] - fail_eq <- eq_cols[vapply(eq_cols, function(c) { - !eq_col_passes(tbl, col = c, ref_suffix = ref_suffix, na_equal = na_equal) - }, logical(1))] - list(tol = fail_tol, eq = fail_eq) -} - #' Build a minimal pointblank agent whose interrogation passes #' #' Used for the all-pass short-circuit: produces a valid interrogated diff --git a/tests/testthat/test-boolean-column-guards.R b/tests/testthat/test-boolean-column-guards.R index f3dd122..a944de9 100644 --- a/tests/testthat/test-boolean-column-guards.R +++ b/tests/testthat/test-boolean-column-guards.R @@ -32,12 +32,3 @@ test_that("the count reducers inherit the guard instead of PASS with n = 0", { regexp = "internal error" ) }) - -test_that("the pass predicates inherit the guard instead of TRUE", { - tbl <- data.frame(a = 1:3) - expect_error(tol_col_passes(tbl, col = "a"), regexp = "internal error") - expect_error( - eq_col_passes(tbl, col = "a", ref_suffix = "__reference", na_equal = TRUE), - regexp = "internal error" - ) -}) diff --git a/tests/testthat/test-coverage.R b/tests/testthat/test-coverage.R index 1b2a938..acbd82d 100644 --- a/tests/testthat/test-coverage.R +++ b/tests/testthat/test-coverage.R @@ -2,7 +2,7 @@ # already-computed boolean validation columns into one row per check actually # performed (column, check type, n rows, n_failed, status). It must list EVERY # column and stay consistent with the verdict (status == PASS iff the column -# passes in all_validations_pass / failing_columns). +# passes when deriving the verdict). # --- build_coverage: value checks (tolerance) ------------------------------- From 45772816340f8ba8d2a8407ec2608c94f6e8f5f1 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 00:55:25 +0200 Subject: [PATCH 23/40] perf: scans et transferts evitables supprimes du chemin lazy (issue #24) - lazy_nrow saute quand rien ne consomme les comptes (comparaison a cle avec check_count false) : validate_row_counts gagne count_rows (defaut inchange pour les appelants directs) - schema 0-ligne collecte une fois par table et reutilise partout, y compris par le template auto (le schema tient lieu de reference : write_rules_template ne lit que noms et types) - le join a cle ne porte que cle + colonnes comparees : les colonnes ignorees et unilaterales ne traversent plus le join ni l'agent (les extraits d'echec ne montrent plus les colonnes ignorees, documente) - find_duplicate_keys lazy agrege en SQL : 2 scalaires + 3 groupes d'exemples au plus rapatries, marqueur ... quand plus de 3 groupes --- NEWS.md | 12 +++++ R/compare_datasets_from_yaml.R | 64 ++++++++++++++++++------- R/duplicate_keys.R | 27 ++++++++--- R/validation.R | 17 +++++-- man/validate_row_counts.Rd | 14 +++++- tests/testthat/test-avoidable-scans.R | 68 +++++++++++++++++++++++++++ 6 files changed, 173 insertions(+), 29 deletions(-) create mode 100644 tests/testthat/test-avoidable-scans.R diff --git a/NEWS.md b/NEWS.md index 58021e0..b484785 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,18 @@ ## Performance +* Four avoidable scans and transfers are gone from the lazy path (issue #24): + the two systematic `COUNT(*)` full scans are skipped when nothing consumes + them (keyed comparison with `check_count: false`; `validate_row_counts()` + gains a `count_rows` parameter, default unchanged); the 0-row schema is + collected once per table and reused by every consumer including the + auto-generated template (one DB roundtrip saved when `path = NULL`); the + keyed join only carries the key and the compared columns, so ignored and + one-sided columns no longer travel through the join into the agent (the + failing-row extracts consequently no longer show ignored columns); and the + lazy duplicate-key detection aggregates in SQL, transferring 2 scalars plus + at most 3 example groups instead of every duplicated group. + * The verdict, the coverage and the failing-column sets now come from a SINGLE pass over the boolean validation columns: `build_coverage()` runs first and everything else derives from it (its rows already carry the diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 588d4cc..6c14585 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -389,6 +389,12 @@ compare_datasets_from_yaml <- function(data_reference, data_candidate <- arrow_dataset_to_duckdb(data_candidate, fresh_con, "datadiff_cand") } + # Collect 0-row schemas ONCE for type detection and every consumer below + # (key validation, auto-template generation, suffix checks): each collect is + # a database roundtrip on lazy inputs. + schema_ref <- dplyr::collect(utils::head(data_reference, 0L)) + schema_cand <- dplyr::collect(utils::head(data_candidate, 0L)) + # Validate the key argument against BOTH datasets before any other consumer: # the auto-generated-template path (path = NULL) would otherwise reach # write_rules_template() first, which validates the reference only and with @@ -396,16 +402,26 @@ compare_datasets_from_yaml <- function(data_reference, if (!is.null(key)) { validate_comparison_key( key, - ref_cols = get_col_names(data_reference), - cand_cols = get_col_names(data_candidate) + ref_cols = names(schema_ref), + cand_cols = names(schema_cand) ) } - # If no path provided, create a temporary YAML with default rules + # An empty string means "no explicit label": normalised BEFORE the + # auto-template call, so that "" falls through to the template default + # instead of triggering write_rules_template's deparse fallback (which + # would leak the internal variable name into the report label). + if (!is.null(label) && label == "") { + label <- NULL + } + + # If no path provided, create a temporary YAML with default rules. The 0-row + # schema stands in for the reference: write_rules_template() only reads + # column names and types, and this avoids a second schema roundtrip. if (is.null(path)) { path <- tempfile(fileext = ".yaml") write_rules_template( - data_reference = data_reference, + data_reference = schema_ref, key = key, label = label %||% "Comparison with default rules", path = path @@ -420,8 +436,8 @@ compare_datasets_from_yaml <- function(data_reference, } # Check for reserved suffix conflicts in column names - ref_cols_with_suffix <- grep(ref_suffix, get_col_names(data_reference), fixed = TRUE, value = TRUE) - cand_cols_with_suffix <- grep(ref_suffix, get_col_names(data_candidate), fixed = TRUE, value = TRUE) + ref_cols_with_suffix <- grep(ref_suffix, names(schema_ref), fixed = TRUE, value = TRUE) + cand_cols_with_suffix <- grep(ref_suffix, names(schema_cand), fixed = TRUE, value = TRUE) if (length(ref_cols_with_suffix) > 0 || length(cand_cols_with_suffix) > 0) { conflicting <- unique(c(ref_cols_with_suffix, cand_cols_with_suffix)) warning(sprintf( @@ -432,10 +448,6 @@ compare_datasets_from_yaml <- function(data_reference, rules <- read_rules(path) - # Collect 0-row schemas for type detection and type-dependent logic. - schema_ref <- dplyr::collect(utils::head(data_reference, 0L)) - schema_cand <- dplyr::collect(utils::head(data_candidate, 0L)) - na_equal <- isTRUE(rules$defaults$na_equal) ignore_columns <- rules$defaults$ignore_columns %||% character(0) @@ -476,18 +488,20 @@ compare_datasets_from_yaml <- function(data_reference, # Build detailed warning message warning_parts <- c() + # %.0f: the lazy counts are whole doubles from SQL aggregates, and + # sprintf %d errors on any whole double at or above 2^31 if (ref_has_dups) { warning_parts <- c(warning_parts, sprintf( - "data_reference: %d duplicate key value(s) affecting %d rows (examples: %s)", - ref_dup_info$n_dup_keys, ref_dup_info$n_dup_rows, + "data_reference: %.0f duplicate key value(s) affecting %.0f rows (examples: %s)", + as.numeric(ref_dup_info$n_dup_keys), as.numeric(ref_dup_info$n_dup_rows), paste(ref_dup_info$examples, collapse = "; ") )) } if (cand_has_dups) { warning_parts <- c(warning_parts, sprintf( - "data_candidate: %d duplicate key value(s) affecting %d rows (examples: %s)", - cand_dup_info$n_dup_keys, cand_dup_info$n_dup_rows, + "data_candidate: %.0f duplicate key value(s) affecting %.0f rows (examples: %s)", + as.numeric(cand_dup_info$n_dup_keys), as.numeric(cand_dup_info$n_dup_rows), paste(cand_dup_info$examples, collapse = "; ") )) } @@ -550,8 +564,14 @@ compare_datasets_from_yaml <- function(data_reference, data_reference_p <- preprocess_dataframe(data_reference, col_rules, schema = schema_ref) data_candidate_p <- preprocess_dataframe(data_candidate, col_rules, schema = schema_cand) - # Get row validation information - row_validation_info <- validate_row_counts(data_reference_p, data_candidate_p, rules) + # Get row validation information. The counts are only consumed by the + # row-count check and the positional path: on a keyed comparison with + # check_count off, skip the two COUNT(*) full scans entirely. + needs_counts <- isTRUE(rules$row_validation$check_count) || is.null(key) + row_validation_info <- validate_row_counts( + data_reference_p, data_candidate_p, rules, + count_rows = needs_counts + ) if (!is.null(key)) { # Re-validated here because the key may come from the YAML rules @@ -562,8 +582,16 @@ compare_datasets_from_yaml <- function(data_reference, cand_cols = get_col_names(data_candidate_p) ) - # Join candidate to reference on key to handle different row counts - cmp <- left_join(data_candidate_p, data_reference_p, by = key, suffix = c("", ref_suffix)) + # Join candidate to reference on key to handle different row counts. + # Only the key and the compared columns enter the join: ignored and + # one-sided columns are never compared, and carrying them through the + # join and into the agent costs memory on wide tables. + keep_cols <- c(key, common_cols) + cmp <- left_join( + dplyr::select(data_candidate_p, dplyr::all_of(keep_cols)), + dplyr::select(data_reference_p, dplyr::all_of(keep_cols)), + by = key, suffix = c("", ref_suffix) + ) } else { # Row-count mismatch is decidable from the counts precomputed by # validate_row_counts(): abort before the potentially expensive collect diff --git a/R/duplicate_keys.R b/R/duplicate_keys.R index ac1125d..02f9708 100644 --- a/R/duplicate_keys.R +++ b/R/duplicate_keys.R @@ -34,17 +34,32 @@ find_duplicate_keys <- function(data, key) { while (count_col %in% key) { count_col <- paste0(count_col, "_") } - dups <- data %>% + duplicated_groups <- data %>% dplyr::count(dplyr::across(dplyr::all_of(key)), name = count_col) %>% - dplyr::filter(.data[[count_col]] > 1L) %>% + dplyr::filter(.data[[count_col]] > 1L) + # Aggregate in SQL: only 2 scalars plus at most 3 example groups cross the + # wire, instead of every duplicated group (potentially millions). + agg <- duplicated_groups %>% + dplyr::summarise( + ..datadiff_dup_keys = dplyr::n(), + ..datadiff_dup_rows = sum(.data[[count_col]], na.rm = TRUE) + ) %>% dplyr::collect() - if (nrow(dups) == 0L) { + n_dup_keys <- as.numeric(agg$..datadiff_dup_keys) + if (n_dup_keys == 0) { return(NULL) } + example_groups <- duplicated_groups %>% + utils::head(3L) %>% + dplyr::collect() + examples <- format_key_examples(example_groups[, key, drop = FALSE], key) + if (n_dup_keys > 3) { + examples <- c(examples, "...") + } return(list( - n_dup_keys = nrow(dups), - n_dup_rows = sum(dups[[count_col]]), - examples = format_key_examples(dups[, key, drop = FALSE], key) + n_dup_keys = n_dup_keys, + n_dup_rows = as.numeric(agg$..datadiff_dup_rows), + examples = examples )) } diff --git a/R/validation.R b/R/validation.R index 80e4fd0..32ca2f7 100644 --- a/R/validation.R +++ b/R/validation.R @@ -17,6 +17,12 @@ lazy_nrow <- function(x) { #' @param data_reference_p Preprocessed reference dataframe #' @param data_candidate_p Preprocessed candidate dataframe #' @param rules List of validation rules containing row_validation settings +#' @param count_rows Logical; when `FALSE`, the row counts are not computed and +#' `ref_count`/`cand_count` are `NA`. On lazy tables each count is a full +#' `COUNT(*)` scan, wasted when nothing consumes it (keyed comparison with +#' `check_count: false`). Ignored when the rules activate `check_count`: +#' the counts are then always computed, so the returned structure stays +#' consistent (an active check never sees `NA` counts). #' @return A list with validation information for pointblank integration #' @examples #' ref <- data.frame(a = 1:3) @@ -24,11 +30,14 @@ lazy_nrow <- function(x) { #' rules <- list(row_validation = list(check_count = TRUE, expected_count = 3, tolerance = 0)) #' validate_row_counts(ref, cand, rules) #' @export -validate_row_counts <- function(data_reference_p, data_candidate_p, rules) { +validate_row_counts <- function(data_reference_p, data_candidate_p, rules, + count_rows = TRUE) { + check_count <- isTRUE(rules$row_validation$check_count) + do_count <- count_rows || check_count list( - check_count = isTRUE(rules$row_validation$check_count), - ref_count = lazy_nrow(data_reference_p), - cand_count = lazy_nrow(data_candidate_p), + check_count = check_count, + ref_count = if (do_count) lazy_nrow(data_reference_p) else NA_integer_, + cand_count = if (do_count) lazy_nrow(data_candidate_p) else NA_integer_, expected_count = rules$row_validation$expected_count, tolerance = rules$row_validation$tolerance %||% 0 ) diff --git a/man/validate_row_counts.Rd b/man/validate_row_counts.Rd index c977dc6..f85d6d8 100644 --- a/man/validate_row_counts.Rd +++ b/man/validate_row_counts.Rd @@ -4,7 +4,12 @@ \alias{validate_row_counts} \title{Validate row counts according to rules} \usage{ -validate_row_counts(data_reference_p, data_candidate_p, rules) +validate_row_counts( + data_reference_p, + data_candidate_p, + rules, + count_rows = TRUE +) } \arguments{ \item{data_reference_p}{Preprocessed reference dataframe} @@ -12,6 +17,13 @@ validate_row_counts(data_reference_p, data_candidate_p, rules) \item{data_candidate_p}{Preprocessed candidate dataframe} \item{rules}{List of validation rules containing row_validation settings} + +\item{count_rows}{Logical; when \code{FALSE}, the row counts are not computed and +\code{ref_count}/\code{cand_count} are \code{NA}. On lazy tables each count is a full +\verb{COUNT(*)} scan, wasted when nothing consumes it (keyed comparison with +\code{check_count: false}). Ignored when the rules activate \code{check_count}: +the counts are then always computed, so the returned structure stays +consistent (an active check never sees \code{NA} counts).} } \value{ A list with validation information for pointblank integration diff --git a/tests/testthat/test-avoidable-scans.R b/tests/testthat/test-avoidable-scans.R new file mode 100644 index 0000000..cce2882 --- /dev/null +++ b/tests/testthat/test-avoidable-scans.R @@ -0,0 +1,68 @@ +# Avoidable scans and transfers: row counts are only computed when consumed, +# the keyed join carries only the key and the compared columns, and the lazy +# duplicate-key detection aggregates in SQL instead of collecting every +# duplicated group. + +test_that("validate_row_counts can skip the COUNT(*) when nothing consumes it", { + ref <- data.frame(id = 1:3, x = 1:3) + cand <- data.frame(id = 1:3, x = 1:3) + rules <- list(row_validation = list(check_count = FALSE, expected_count = NULL, tolerance = 0)) + + info <- validate_row_counts(ref, cand, rules, count_rows = FALSE) + expect_false(info$check_count) + expect_true(is.na(info$ref_count)) + expect_true(is.na(info$cand_count)) + + # Default stays counting (backward compatible for direct callers) + info_default <- validate_row_counts(ref, cand, rules) + expect_identical(info_default$ref_count, 3L) + expect_identical(info_default$cand_count, 3L) +}) + +test_that("ignored and extra columns stay out of the joined comparison", { + ref <- data.frame(id = 1:3, x = c(1, 2, 3), noise = c("a", "b", "c")) + cand <- data.frame(id = 1:3, x = c(1, 2, 9), noise = c("zz", "zz", "zz"), + extra = c(TRUE, FALSE, TRUE)) + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + write_rules_template(ref, key = "id", path = yaml_path, + ignore_columns_default = "noise") + + res <- suppressWarnings(suppressMessages( + compare_datasets_from_yaml(ref, cand, key = "id", path = yaml_path) + )) + expect_false(res$all_passed) + + # The failing-row extract carries the key and the compared columns, not the + # ignored column nor the candidate-only extra column + ex <- pointblank::get_data_extracts(res$reponse) + ex1 <- as.data.frame(ex[[1]]) + expect_true("id" %in% names(ex1)) + expect_false("noise" %in% names(ex1)) + expect_false("extra" %in% names(ex1)) +}) + +test_that("lazy duplicate detection aggregates in SQL (few rows transferred)", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + # 1000 duplicated key values: the old code collected all 1000 groups, + # the aggregate path must return the same counts and 3 examples + "..." + df <- data.frame(id = rep(seq_len(1000), each = 2), v = 1) + duckdb::dbWriteTable(con, "t_many_dups", df) + info <- find_duplicate_keys(dplyr::tbl(con, "t_many_dups"), "id") + expect_equal(as.numeric(info$n_dup_keys), 1000) + expect_equal(as.numeric(info$n_dup_rows), 2000) + expect_length(info$examples, 4L) # 3 examples + "..." + expect_identical(info$examples[4], "...") + + # Below the truncation threshold: no "..." marker + duckdb::dbWriteTable(con, "t_two_dups", + data.frame(id = c(1, 1, 2, 2, 3), v = 1:5)) + info2 <- find_duplicate_keys(dplyr::tbl(con, "t_two_dups"), "id") + expect_equal(as.numeric(info2$n_dup_keys), 2) + expect_length(info2$examples, 2L) +}) From 4905f935682a39d71fe8e7a4eb73e010224c49f9 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 01:04:42 +0200 Subject: [PATCH 24/40] feat: datadiff_report_html(extracts_dir =) ecrit les extraits en vrais CSV (issue #10) Les boutons CSV du rapport HTML pointblank sont des liens data: URI avec attribut download ; la webview de Positron / Posit Workbench les bloque silencieusement (clic muet, constate par l'utilisateur de l'issue). extracts_dir ecrit chaque extrait d'etape en echec en fichier CSV reel (extract__.csv, noms de colonnes assainis), rien n'est cree sur une comparaison toute verte. --- NEWS.md | 8 ++++ R/report.R | 43 +++++++++++++++++- man/datadiff_report_html.Rd | 10 ++++- tests/testthat/test-report-extracts-dir.R | 53 +++++++++++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 tests/testthat/test-report-extracts-dir.R diff --git a/NEWS.md b/NEWS.md index 60ea88c..e6da9c0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,13 @@ # datadiff (development version) +## New features + +* `datadiff_report_html()` gains an `extracts_dir` argument: the failing-row + extracts are also written as plain CSV files (one per failing step). The + CSV buttons inside the HTML report are `data:` URI downloads, which some + viewers block silently (Positron / Posit Workbench webview): files on disk + are the robust alternative (issue #10). + ## Performance * Four avoidable scans and transfers are gone from the lazy path (issue #24): diff --git a/R/report.R b/R/report.R index e07f5b5..d96806a 100644 --- a/R/report.R +++ b/R/report.R @@ -244,9 +244,16 @@ print.datadiff_report <- function(x, ...) { #' @param res The list returned by [compare_datasets_from_yaml()]. #' @param file Optional path to write the report to as a self-contained HTML #' file (via [pointblank::export_report()]). When `NULL`, nothing is written. +#' @param extracts_dir Optional directory where the failing-row extracts are +#' also written as plain CSV files, one per failing validation step, named +#' `extract__.csv`. The CSV buttons inside the HTML report are +#' `data:` URI downloads, which some viewers block (Positron / Posit +#' Workbench webview): files on disk are the robust alternative. The +#' directory is created when needed; nothing is created when there is no +#' extract (all-pass comparison or `extract_failed = FALSE`). #' @return The pointblank agent report object, invisibly. #' @export -datadiff_report_html <- function(res, file = NULL) { +datadiff_report_html <- function(res, file = NULL, extracts_dir = NULL) { coverage <- res$coverage if (is.null(coverage)) { stop("`res` has no `coverage`; was it produced by compare_datasets_from_yaml()?") @@ -267,5 +274,39 @@ datadiff_report_html <- function(res, file = NULL) { if (!is.null(file)) { pointblank::export_report(report, filename = file, quiet = TRUE) } + if (!is.null(extracts_dir)) { + write_extract_csvs(reponse, extracts_dir = extracts_dir) + } invisible(report) } + +# Write each failing-step extract of an interrogated agent as a CSV file. +# Returns the written paths, invisibly. +write_extract_csvs <- function(reponse, extracts_dir) { + extracts <- tryCatch( + pointblank::get_data_extracts(reponse), + error = function(e) { + list() + } + ) + if (length(extracts) == 0) { + return(invisible(character(0))) + } + if (!dir.exists(extracts_dir)) { + dir.create(extracts_dir, recursive = TRUE) + } + vs <- reponse$validation_set + paths <- vapply(X = names(extracts), FUN = function(nm) { + i <- as.integer(nm) + col <- report_underlying_col(vs$column[[i]][1]) + safe_col <- gsub("[^A-Za-z0-9_.-]", "_", x = col) + path <- file.path( + extracts_dir, + sprintf("extract_%04d_%s.csv", i, safe_col) + ) + utils::write.csv(as.data.frame(extracts[[nm]]), file = path, + row.names = FALSE) + path + }, FUN.VALUE = character(1), USE.NAMES = FALSE) + invisible(paths) +} diff --git a/man/datadiff_report_html.Rd b/man/datadiff_report_html.Rd index bf6dae5..f3c3001 100644 --- a/man/datadiff_report_html.Rd +++ b/man/datadiff_report_html.Rd @@ -4,13 +4,21 @@ \alias{datadiff_report_html} \title{Render a comparison result as a pointblank HTML report} \usage{ -datadiff_report_html(res, file = NULL) +datadiff_report_html(res, file = NULL, extracts_dir = NULL) } \arguments{ \item{res}{The list returned by \code{\link[=compare_datasets_from_yaml]{compare_datasets_from_yaml()}}.} \item{file}{Optional path to write the report to as a self-contained HTML file (via \code{\link[pointblank:export_report]{pointblank::export_report()}}). When \code{NULL}, nothing is written.} + +\item{extracts_dir}{Optional directory where the failing-row extracts are +also written as plain CSV files, one per failing validation step, named +\verb{extract__.csv}. The CSV buttons inside the HTML report are +\verb{data:} URI downloads, which some viewers block (Positron / Posit +Workbench webview): files on disk are the robust alternative. The +directory is created when needed; nothing is created when there is no +extract (all-pass comparison or \code{extract_failed = FALSE}).} } \value{ The pointblank agent report object, invisibly. diff --git a/tests/testthat/test-report-extracts-dir.R b/tests/testthat/test-report-extracts-dir.R new file mode 100644 index 0000000..35b1465 --- /dev/null +++ b/tests/testthat/test-report-extracts-dir.R @@ -0,0 +1,53 @@ +# datadiff_report_html(extracts_dir =): the failing-row extracts are also +# written as plain CSV files. The HTML report's CSV buttons are data-URI +# downloads, which some viewers (Positron / Posit Workbench webview) block: +# real files on disk are the robust alternative. + +test_that("extracts_dir writes one CSV per failing step", { + ref <- data.frame(id = 1:3, x = c(1, 2, 3), s = c("a", "b", "c")) + cand <- data.frame(id = 1:3, x = c(1, 2, 9), s = c("a", "ZZ", "c")) + res <- suppressMessages(compare_datasets_from_yaml(ref, cand, key = "id")) + expect_false(res$all_passed) + + out_dir <- tempfile(pattern = "datadiff_extracts_") + on.exit(unlink(out_dir, recursive = TRUE), add = TRUE) + datadiff_report_html(res, file = NULL, extracts_dir = out_dir) + + csvs <- list.files(out_dir, pattern = "\\.csv$", full.names = TRUE) + expect_length(csvs, 2L) # x (tolerance) + s (equality) + expect_true(any(grepl("_x\\.csv$", csvs))) + expect_true(any(grepl("_s\\.csv$", csvs))) + + # Content: the x extract carries the failing row (id 3) + x_csv <- utils::read.csv(csvs[grepl("_x\\.csv$", csvs)]) + expect_true(3 %in% x_csv$id) +}) + +test_that("extracts_dir is not created on an all-pass comparison", { + ref <- data.frame(id = 1:3, x = c(1, 2, 3)) + res <- suppressMessages(compare_datasets_from_yaml(ref, ref, key = "id")) + expect_true(res$all_passed) + + out_dir <- tempfile(pattern = "datadiff_extracts_") + datadiff_report_html(res, file = NULL, extracts_dir = out_dir) + expect_false(dir.exists(out_dir)) +}) + +test_that("column names are sanitized in extract file names", { + ref <- data.frame(id = 1:2, x = c(1, 2)) + names(ref)[2] <- "montant (EUR)/annee" + cand <- ref + cand[[2]][2] <- 99 + res <- suppressWarnings(suppressMessages( + compare_datasets_from_yaml(ref, cand, key = "id") + )) + expect_false(res$all_passed) + + out_dir <- tempfile(pattern = "datadiff_extracts_") + on.exit(unlink(out_dir, recursive = TRUE), add = TRUE) + datadiff_report_html(res, file = NULL, extracts_dir = out_dir) + + csvs <- list.files(out_dir, pattern = "\\.csv$") + expect_length(csvs, 1L) + expect_false(grepl("[ ()/]", csvs)) +}) From 5e4e09bcd3e07031d5a32020bcd96c6f4bd04c7f Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:12:25 +0200 Subject: [PATCH 25/40] fix: extracts_dir - cleanup de test, pattern de nom documente, erreur d'extraction signalee Reponse a la review Copilot de la PR : on.exit dans le test tout-vert, zero-padding %04d documente dans la roxygen, et write_extract_csvs emet un warning explicite au lieu d'avaler silencieusement une erreur de get_data_extracts (extracts_dir semblait fonctionner en n'ecrivant rien). --- R/report.R | 7 ++++++- man/datadiff_report_html.Rd | 3 ++- tests/testthat/test-report-extracts-dir.R | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/R/report.R b/R/report.R index d96806a..3a1222a 100644 --- a/R/report.R +++ b/R/report.R @@ -246,7 +246,8 @@ print.datadiff_report <- function(x, ...) { #' file (via [pointblank::export_report()]). When `NULL`, nothing is written. #' @param extracts_dir Optional directory where the failing-row extracts are #' also written as plain CSV files, one per failing validation step, named -#' `extract__.csv`. The CSV buttons inside the HTML report are +#' `extract__.csv` with a zero-padded 4-digit step number +#' (e.g. `extract_0002_price.csv`). The CSV buttons inside the HTML report are #' `data:` URI downloads, which some viewers block (Positron / Posit #' Workbench webview): files on disk are the robust alternative. The #' directory is created when needed; nothing is created when there is no @@ -286,6 +287,10 @@ write_extract_csvs <- function(reponse, extracts_dir) { extracts <- tryCatch( pointblank::get_data_extracts(reponse), error = function(e) { + warning(sprintf( + "extracts_dir: could not read the data extracts (%s); no CSV written.", + conditionMessage(e) + ), call. = FALSE) list() } ) diff --git a/man/datadiff_report_html.Rd b/man/datadiff_report_html.Rd index f3c3001..0a40d3e 100644 --- a/man/datadiff_report_html.Rd +++ b/man/datadiff_report_html.Rd @@ -14,7 +14,8 @@ file (via \code{\link[pointblank:export_report]{pointblank::export_report()}}). \item{extracts_dir}{Optional directory where the failing-row extracts are also written as plain CSV files, one per failing validation step, named -\verb{extract__.csv}. The CSV buttons inside the HTML report are +\verb{extract__.csv} with a zero-padded 4-digit step number +(e.g. \code{extract_0002_price.csv}). The CSV buttons inside the HTML report are \verb{data:} URI downloads, which some viewers block (Positron / Posit Workbench webview): files on disk are the robust alternative. The directory is created when needed; nothing is created when there is no diff --git a/tests/testthat/test-report-extracts-dir.R b/tests/testthat/test-report-extracts-dir.R index 35b1465..7072b35 100644 --- a/tests/testthat/test-report-extracts-dir.R +++ b/tests/testthat/test-report-extracts-dir.R @@ -29,6 +29,7 @@ test_that("extracts_dir is not created on an all-pass comparison", { expect_true(res$all_passed) out_dir <- tempfile(pattern = "datadiff_extracts_") + on.exit(unlink(out_dir, recursive = TRUE), add = TRUE) datadiff_report_html(res, file = NULL, extracts_dir = out_dir) expect_false(dir.exists(out_dir)) }) From 5271f9bd750672eb7d09f14a39811559d23ffa70 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:19:34 +0200 Subject: [PATCH 26/40] docs: l'assainissement des noms de colonnes dans les fichiers d'extraits est documente Reponse a la review Copilot : le contrat extracts_dir mentionne desormais la substitution des caracteres hors [A-Za-z0-9_.-]. --- R/report.R | 3 ++- man/datadiff_report_html.Rd | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/R/report.R b/R/report.R index 3a1222a..9c512cc 100644 --- a/R/report.R +++ b/R/report.R @@ -247,7 +247,8 @@ print.datadiff_report <- function(x, ...) { #' @param extracts_dir Optional directory where the failing-row extracts are #' also written as plain CSV files, one per failing validation step, named #' `extract__.csv` with a zero-padded 4-digit step number -#' (e.g. `extract_0002_price.csv`). The CSV buttons inside the HTML report are +#' (e.g. `extract_0002_price.csv`). Column names are sanitized for the +#' file system: any character outside `[A-Za-z0-9_.-]` becomes `_`. The CSV buttons inside the HTML report are #' `data:` URI downloads, which some viewers block (Positron / Posit #' Workbench webview): files on disk are the robust alternative. The #' directory is created when needed; nothing is created when there is no diff --git a/man/datadiff_report_html.Rd b/man/datadiff_report_html.Rd index 0a40d3e..28d5180 100644 --- a/man/datadiff_report_html.Rd +++ b/man/datadiff_report_html.Rd @@ -15,7 +15,8 @@ file (via \code{\link[pointblank:export_report]{pointblank::export_report()}}). \item{extracts_dir}{Optional directory where the failing-row extracts are also written as plain CSV files, one per failing validation step, named \verb{extract__.csv} with a zero-padded 4-digit step number -(e.g. \code{extract_0002_price.csv}). The CSV buttons inside the HTML report are +(e.g. \code{extract_0002_price.csv}). Column names are sanitized for the +file system: any character outside \verb{[A-Za-z0-9_.-]} becomes \verb{_}. The CSV buttons inside the HTML report are \verb{data:} URI downloads, which some viewers block (Positron / Posit Workbench webview): files on disk are the robust alternative. The directory is created when needed; nothing is created when there is no From 97128a3a8d2d286bc555bb3460b33719d4804cd6 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:27:15 +0200 Subject: [PATCH 27/40] fix: nits de review - lookup par id de step, encodage UTF-8, test du chemin d'erreur - report_underlying_col cherche le step par son id (match sur vs$i) et non par position de ligne : la contiguite du validation_set est un invariant implicite de pointblank, pas un contrat - write.csv en fileEncoding UTF-8 (les extraits existent pour donner un fichier portable) - test de regression du warning sur reponse illisible (rien n'ecrit, pas de repertoire cree) --- R/report.R | 6 ++++-- tests/testthat/test-report-extracts-dir.R | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/R/report.R b/R/report.R index 9c512cc..75136ac 100644 --- a/R/report.R +++ b/R/report.R @@ -304,14 +304,16 @@ write_extract_csvs <- function(reponse, extracts_dir) { vs <- reponse$validation_set paths <- vapply(X = names(extracts), FUN = function(nm) { i <- as.integer(nm) - col <- report_underlying_col(vs$column[[i]][1]) + # Look the step up by its id (vs$i), not by row position: contiguity of + # the validation set is an implicit pointblank invariant, not a contract + col <- report_underlying_col(vs$column[[match(i, vs$i)]][1]) safe_col <- gsub("[^A-Za-z0-9_.-]", "_", x = col) path <- file.path( extracts_dir, sprintf("extract_%04d_%s.csv", i, safe_col) ) utils::write.csv(as.data.frame(extracts[[nm]]), file = path, - row.names = FALSE) + row.names = FALSE, fileEncoding = "UTF-8") path }, FUN.VALUE = character(1), USE.NAMES = FALSE) invisible(paths) diff --git a/tests/testthat/test-report-extracts-dir.R b/tests/testthat/test-report-extracts-dir.R index 7072b35..ca649f6 100644 --- a/tests/testthat/test-report-extracts-dir.R +++ b/tests/testthat/test-report-extracts-dir.R @@ -52,3 +52,21 @@ test_that("column names are sanitized in extract file names", { expect_length(csvs, 1L) expect_false(grepl("[ ()/]", csvs)) }) + +test_that("an unreadable reponse warns explicitly and writes nothing", { + res_broken <- list( + coverage = structure( + data.frame(column = "x", check = "tolerance", n = 1L, n_failed = 1L, + status = "FAIL", stringsAsFactors = FALSE), + class = c("datadiff_coverage", "data.frame") + ), + reponse = list() + ) + out_dir <- tempfile(pattern = "datadiff_extracts_") + on.exit(unlink(out_dir, recursive = TRUE), add = TRUE) + expect_warning( + datadiff_report_html(res_broken, file = NULL, extracts_dir = out_dir), + regexp = "could not read the data extracts" + ) + expect_false(dir.exists(out_dir)) +}) From b94b345622b007fed428778a7ae2459554412961 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 01:19:33 +0200 Subject: [PATCH 28/40] perf: verdict lazy par agregats SQL, plus de collect integral des booleens (issue #22) Le collect() de la table slim chargeait N x (T+E) logicals en RAM R (~2 Go pour 4M lignes x 125 colonnes sur le cas vert), contredisant la promesse without loading data into R memory du commentaire - alors que le verdict tient dans (T+E) paires (n, n_failed). - lazy_boolean_counts() : un scan d'agregats SQL sur la temp table (SUM(CASE WHEN ok THEN 0 ELSE 1 END) compte FALSE et NULL, semantique exacte des reducteurs locaux ; SUM sur table vide -> NULL -> 0) - build_coverage(counts =) : les comptes precalcules remplacent les accesseurs locaux sur le chemin lazy - cas vert : plus aucun collect, l'agent placeholder est de taille constante (nrow(res$reponse$tbl) <= 1 au lieu de N) - cas rouge : seules les colonnes booleennes EN ECHEC sont collectees pour l'agent (+ row_count_ok si check actif), extraits inchanges - les helpers datadiff_ok_col/eq_col deviennent length-guarded : le fantome recycle0 (paste0(character(0), suffixe) == suffixe) a mordu une 2e fois via lazy_boolean_counts ; la garde centrale immunise la classe et le suffix_all local devient inutile --- NEWS.md | 10 ++ R/compare_datasets_from_yaml.R | 88 ++++++++++++----- R/constants.R | 8 ++ R/coverage.R | 69 +++++++++++-- tests/testthat/test-lazy-aggregates.R | 136 ++++++++++++++++++++++++++ vignettes/datadiff.Rmd | 4 + 6 files changed, 285 insertions(+), 30 deletions(-) create mode 100644 tests/testthat/test-lazy-aggregates.R diff --git a/NEWS.md b/NEWS.md index e6da9c0..73be078 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,6 +10,16 @@ ## Performance +* The lazy verdict no longer loads the boolean table into R: the per-column + (n, n_failed) counts come from ONE SQL aggregate scan over the computed + slim table (`SUM(CASE WHEN ok THEN 0 ELSE 1 END)`, counting FALSE and NULL + alike, exactly the local reducers' semantics), and only the FAILING + columns' booleans are collected for the pointblank agent. A green lazy + comparison previously collected N x columns logicals (~2 GB of R memory + for 4M rows x 125 columns, 32x the figure the code comment promised); it + now keeps O(columns) in R, and `res$reponse` carries a constant-size + placeholder instead of N collected rows (issue #22). + * Four avoidable scans and transfers are gone from the lazy path (issue #24): the two systematic `COUNT(*)` full scans are skipped when nothing consumes them (keyed comparison with `check_count: false`; `validate_row_counts()` diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 021f5a3..91f3565 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -690,40 +690,45 @@ compare_datasets_from_yaml <- function(data_reference, is_lazy <- is_non_local(cmp) cmp_for_agent <- cmp if (is_lazy) { - # Length guards: paste0(character(0), "__ok") yields "__ok" (recycle0 is - # FALSE by default), a phantom name the previous any_of() silently ate - suffix_all <- function(cols, suffix) { - if (length(cols) == 0) { - return(character(0)) - } - paste0(cols, suffix) - } + # The name helpers are length-guarded against the recycle0 phantom + # (paste0(character(0), "__ok") yields "__ok"); all_of() below then + # fails loudly instead of silently dropping a phantom name val_cols <- c( - suffix_all(tol_cols, suffix = datadiff_suffix_ok), - suffix_all(eq_cols, suffix = datadiff_suffix_eq), + datadiff_ok_col(tol_cols), + datadiff_eq_col(eq_cols), if (isTRUE(row_validation_info$check_count)) { "row_count_ok" } else { character(0) } ) + if (length(val_cols) == 0) { + # Nothing to compute or aggregate (no value checks, no row-count flag): + # the verdict rests on the structural coverage rows alone, and a + # zero-column CREATE TABLE AS SELECT is invalid SQL on some backends + lazy_counts <- list() + cmp_for_agent <- data.frame() + } else { # all_of(): a validation column silently dropped here would later read as # NULL by the boolean accessors, i.e. a false all-pass; fail loudly instead cmp_slim <- dplyr::select(cmp, dplyr::all_of(val_cols)) tmp_tbl_name <- datadiff_tmp_table_name() # compute() sends CREATE TEMP TABLE AS SELECT ... to DuckDB: all computation # (join, boolean expressions) happens inside DuckDB's process, with disk - # spilling available for the large join. We then collect() the slim boolean - # result into R so that pointblank receives a plain data.frame - avoiding - # DuckDB connection-state issues (is_tbl_mssql crash) during interrogation. + # spilling available for the large join. The verdict is then derived from + # an SQL aggregate scan of this table; only the FAILING boolean columns + # are later collected into R (red path) so that pointblank receives a + # plain data.frame - avoiding DuckDB connection-state issues + # (is_tbl_mssql crash) during interrogation. cmp_slim_computed <- dplyr::compute(cmp_slim, name = tmp_tbl_name, temporary = TRUE) - # The slim table only feeds the collect() below. Drop it at exit so that - # repeated calls on a user-supplied connection do not accumulate temp - # tables for the lifetime of that connection. after = FALSE runs the drop - # BEFORE the exit handlers registered earlier, in particular before the - # dbDisconnect of the private connection on the Arrow path (on.exit - # add = TRUE fires FIFO by default, which would drop on a closed - # connection); the try() then only masks genuine failures. + # The slim table only feeds the aggregate scan and the failing-column + # collect below. Drop it at exit so that repeated calls on a user-supplied + # connection do not accumulate temp tables for the lifetime of that + # connection. after = FALSE runs the drop BEFORE the exit handlers + # registered earlier, in particular before the dbDisconnect of the private + # connection on the Arrow path (on.exit add = TRUE fires FIFO by default, + # which would drop on a closed connection); the try() then only masks + # genuine failures. on.exit( try( DBI::dbRemoveTable(dbplyr::remote_con(cmp_slim_computed), tmp_tbl_name), @@ -731,20 +736,31 @@ compare_datasets_from_yaml <- function(data_reference, ), add = TRUE, after = FALSE ) - cmp_for_agent <- dplyr::collect(cmp_slim_computed) + # The verdict needs only (n, n_failed) per column: one aggregate scan in + # the database replaces the collect of N x columns booleans into R (the + # collect made the green verdict cost O(rows) of R memory, contradicting + # the documented promise). Only the FAILING columns are collected later, + # on the red path, for the pointblank agent. + lazy_counts <- lazy_boolean_counts( + cmp_slim_computed, tol_cols = tol_cols, eq_cols = eq_cols + ) + cmp_for_agent <- data.frame() + } } # Faithful, O(columns) record of every check performed, built from the # boolean validation columns computed above. It is the SINGLE pass over the # data: the verdict and the failing-column sets below both derive from it, # so the same booleans are no longer scanned two or three times (and the - # local equality booleans no longer recomputed once per scan). + # local equality booleans no longer recomputed once per scan). On the lazy + # path the counts come from the SQL aggregate scan. coverage <- build_coverage( tbl = cmp_for_agent, tol_cols = tol_cols, eq_cols = eq_cols, missing_in_candidate = missing_in_candidate, type_mismatch_cols = type_mismatch_cols, row_validation_info = row_validation_info, row_count_ok = row_count_ok, - ref_suffix = ref_suffix, na_equal = na_equal + ref_suffix = ref_suffix, na_equal = na_equal, + counts = if (is_lazy) lazy_counts else NULL ) # Fast all-pass short-circuit. @@ -802,6 +818,32 @@ compare_datasets_from_yaml <- function(data_reference, cmp_for_agent, fail$tol, col_rules, ref_suffix, na_equal ) } + # Lazy path: collect ONLY the failing boolean columns (plus the row-count + # flag consumed by its validation step) for the agent; the passing + # majority stays in the database. + if (is_lazy) { + red_cols <- c( + datadiff_ok_col(fail$tol), + datadiff_eq_col(fail$eq), + if (isTRUE(row_validation_info$check_count)) { + "row_count_ok" + } else { + character(0) + } + ) + cmp_for_agent <- if (length(red_cols) > 0) { + dplyr::collect( + dplyr::select(cmp_slim_computed, dplyr::all_of(red_cols)) + ) + } else { + # Structural-only failure (missing columns / type mismatches with no + # failing value column and no row-count check): seed ONE row so the + # dummy FALSE columns added by setup_pointblank_agent carry a failing + # unit - a 0-row dummy step would interrogate 0 units and pass, + # flipping pointblank::all_passed() against the coverage verdict + data.frame(.datadiff_structural = FALSE) + } + } agent <- setup_pointblank_agent( cmp_for_agent, cols_reference, diff --git a/R/constants.R b/R/constants.R index 97d7dd9..91789fd 100644 --- a/R/constants.R +++ b/R/constants.R @@ -17,10 +17,18 @@ datadiff_prefix_missing_col <- "__missing_col_" datadiff_prefix_type_mismatch <- "__type_mismatch_" # Column name helpers: the only supported way to build these names. +# Length-guarded: paste0(character(0), suffix) yields the bare suffix +# (recycle0 is FALSE by default), a phantom column name. datadiff_ok_col <- function(col) { + if (length(col) == 0) { + return(character(0)) + } paste0(col, datadiff_suffix_ok) } datadiff_eq_col <- function(col) { + if (length(col) == 0) { + return(character(0)) + } paste0(col, datadiff_suffix_eq) } diff --git a/R/coverage.R b/R/coverage.R index b656024..0807d09 100644 --- a/R/coverage.R +++ b/R/coverage.R @@ -18,33 +18,87 @@ eq_col_counts <- function(tbl, col, ref_suffix, na_equal) { list(n = length(b), n_failed = sum(!b)) } +# (n, n_failed) for every boolean validation column of a LAZY table, computed +# in ONE SQL aggregate scan: SUM(CASE WHEN col THEN 0 ELSE 1 END) counts the +# not-TRUE values (FALSE and NULL alike), matching the local reducers +# (tolerance: is.na | !ok; equality: !b, NA-free by construction). This is +# what lets the lazy verdict avoid collecting N x columns booleans into R. +# Returns a named list keyed by the UNDERLYING column name: +# list( = list(n, n_failed), ...). +lazy_boolean_counts <- function(tbl_lazy, tol_cols, eq_cols) { + con <- dbplyr::remote_con(tbl_lazy) + sub <- dbplyr::sql_render(tbl_lazy) + q <- function(x) as.character(DBI::dbQuoteIdentifier(con, x = x)) + + bool_cols <- c(datadiff_ok_col(tol_cols), datadiff_eq_col(eq_cols)) + underlying <- c(tol_cols, eq_cols) + nf_exprs <- vapply(X = seq_along(bool_cols), FUN = function(i) { + sprintf("SUM(CASE WHEN %s THEN 0 ELSE 1 END) AS %s", + q(bool_cols[i]), q(sprintf("..datadiff_nf_%d", i))) + }, FUN.VALUE = character(1)) + + sql <- sprintf( + "SELECT COUNT(*) AS %s%s FROM (%s) AS %s", + q("..datadiff_total"), + if (length(nf_exprs) > 0) { + paste0(", ", paste(nf_exprs, collapse = ", ")) + } else { + "" + }, + sub, q("datadiff_counts") + ) + row <- DBI::dbGetQuery(con, statement = sql) + + total <- as.numeric(row[["..datadiff_total"]]) + counts <- vector("list", length(underlying)) + names(counts) <- underlying + for (i in seq_along(underlying)) { + nf <- as.numeric(row[[sprintf("..datadiff_nf_%d", i)]]) + # SUM over an empty table is NULL: zero rows means zero failures + counts[[i]] <- list(n = total, n_failed = if (is.na(nf)) { + 0 + } else { + nf + }) + } + counts +} + #' Build a faithful coverage summary of the checks performed #' #' @param tbl Local data.frame holding the boolean validation columns (and, on #' the local path, the raw + reference columns for equality recomputation). +#' Ignored when `counts` is supplied. #' @param tol_cols,eq_cols Character vectors of tolerance / equality columns. #' @param missing_in_candidate,type_mismatch_cols Structural failures. #' @param row_validation_info List with `check_count`. #' @param row_count_ok Logical row-count outcome. #' @param ref_suffix Suffix identifying reference columns. #' @param na_equal Logical; NA equality semantics for equality columns. +#' @param counts Optional precomputed per-column counts (named list +#' `col -> list(n, n_failed)` covering every tolerance and equality column), +#' as produced by `lazy_boolean_counts()`: the lazy path aggregates in SQL +#' instead of collecting the booleans. #' @return A `datadiff_coverage` data.frame with columns `column`, `check`, #' `n`, `n_failed`, `status` (one row per check performed). #' @noRd build_coverage <- function(tbl, tol_cols, eq_cols, missing_in_candidate, type_mismatch_cols, row_validation_info, row_count_ok, - ref_suffix, na_equal) { + ref_suffix, na_equal, counts = NULL) { columns <- character(0) checks <- character(0) - ns <- integer(0) - n_failed <- integer(0) + ns <- numeric(0) + n_failed <- numeric(0) add <- function(column, check, n, nf) { columns <<- c(columns, column) checks <<- c(checks, check) - ns <<- c(ns, as.integer(n)) - n_failed <<- c(n_failed, as.integer(nf)) + # numeric, not integer: the lazy counts come from SQL aggregates and can + # exceed 2^31 - 1, where as.integer() would overflow to NA and silently + # corrupt the coverage and the verdict + ns <<- c(ns, as.numeric(n)) + n_failed <<- c(n_failed, as.numeric(nf)) } # Existence checks: every common (non-type-mismatch) column gets a col_exists @@ -54,11 +108,12 @@ build_coverage <- function(tbl, tol_cols, eq_cols, add(c, "col_exists", 1L, 0L) } for (c in tol_cols) { - cnt <- tol_col_counts(tbl, col = c) + cnt <- counts[[c]] %||% tol_col_counts(tbl, col = c) add(c, "tolerance", cnt$n, cnt$n_failed) } for (c in eq_cols) { - cnt <- eq_col_counts(tbl, col = c, ref_suffix = ref_suffix, na_equal = na_equal) + cnt <- counts[[c]] %||% + eq_col_counts(tbl, col = c, ref_suffix = ref_suffix, na_equal = na_equal) add(c, "equality", cnt$n, cnt$n_failed) } for (c in missing_in_candidate) { diff --git a/tests/testthat/test-lazy-aggregates.R b/tests/testthat/test-lazy-aggregates.R new file mode 100644 index 0000000..d385e37 --- /dev/null +++ b/tests/testthat/test-lazy-aggregates.R @@ -0,0 +1,136 @@ +# Lazy verdict via SQL aggregates: the coverage counts come from one aggregate +# scan in the database, and the boolean slim table is only collected for the +# FAILING columns. A green lazy comparison must not load N rows of booleans +# into R ("without loading data into R memory" is the documented promise). + +test_that("a green lazy comparison does not collect the boolean table", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + n <- 5e4 + df <- data.frame(id = seq_len(n), a = rnorm(n), b = rnorm(n), + s = sample(letters, n, replace = TRUE)) + duckdb::dbWriteTable(con, "ref_agg", df) + duckdb::dbWriteTable(con, "cand_agg", df) + + res <- suppressMessages(compare_datasets_from_yaml( + dplyr::tbl(con, "ref_agg"), dplyr::tbl(con, "cand_agg"), key = "id" + )) + expect_true(res$all_passed) + # Coverage counts are intact (they now come from the SQL aggregates) + cov <- res$coverage + expect_identical(as.integer(cov$n[cov$column == "a" & cov$check == "tolerance"]), as.integer(n)) + expect_identical(as.integer(sum(cov$n_failed)), 0L) + # The agent is the constant-size pass placeholder, not N collected rows + expect_lte(nrow(res$reponse$tbl), 1L) +}) + +test_that("a red lazy comparison collects only the failing boolean columns", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + n <- 1e4 + df <- data.frame(id = seq_len(n), a = rnorm(n), b = rnorm(n), + s = sample(letters, n, replace = TRUE)) + df2 <- df + df2$a[c(3, 7)] <- 999 + duckdb::dbWriteTable(con, "ref_red2", df) + duckdb::dbWriteTable(con, "cand_red2", df2) + + res <- suppressMessages(compare_datasets_from_yaml( + dplyr::tbl(con, "ref_red2"), dplyr::tbl(con, "cand_red2"), key = "id" + )) + expect_false(res$all_passed) + + # Only the failing tolerance column (plus the row-count flag consumed by + # its validation step) reaches the local agent: b__ok and s__eq stay in + # the database + agent_cols <- names(res$reponse$tbl) + expect_true("a__ok" %in% agent_cols) + expect_false("b__ok" %in% agent_cols) + expect_false("s__eq" %in% agent_cols) + + # The verdict details are unchanged: 2 failing rows on a, everything else + # green in the coverage + cov <- res$coverage + expect_identical(as.integer(cov$n_failed[cov$column == "a" & cov$check == "tolerance"]), 2L) + expect_identical(as.integer(cov$n_failed[cov$column == "b" & cov$check == "tolerance"]), 0L) + expect_identical(as.integer(cov$n_failed[cov$column == "s" & cov$check == "equality"]), 0L) + + # And the extracts still surface the failing rows + ex <- pointblank::get_data_extracts(res$reponse) + expect_gte(length(ex), 1L) +}) + +test_that("SQL-aggregated counts match the collected booleans on SQLite too", { + skip_if_not_installed("RSQLite") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(RSQLite::SQLite(), ":memory:") + on.exit(DBI::dbDisconnect(con), add = TRUE) + + df <- data.frame(id = 1:50, x = c(rep(1, 45), rep(2, 5)), + s = rep(c("u", "v"), 25)) + ref <- data.frame(id = 1:50, x = rep(1, 50), s = rep(c("u", "v"), 25)) + DBI::dbWriteTable(con, "ref_sq", ref) + DBI::dbWriteTable(con, "cand_sq", df) + + res <- suppressMessages(compare_datasets_from_yaml( + dplyr::tbl(con, "ref_sq"), dplyr::tbl(con, "cand_sq"), key = "id" + )) + expect_false(res$all_passed) + cov <- res$coverage + expect_identical(as.integer(cov$n_failed[cov$column == "x" & cov$check == "tolerance"]), 5L) + expect_identical(as.integer(cov$n_failed[cov$column == "s" & cov$check == "equality"]), 0L) +}) + +test_that("structural-only lazy failure keeps a failing verdict (no value checks)", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + duckdb::dbWriteTable(con, "ref_struct", data.frame(id = 1:3, x = c(1, 2, 3), extra_ref = 1:3)) + duckdb::dbWriteTable(con, "cand_struct", data.frame(id = 1:3, x = c(1, 2, 3))) + + p <- tempfile(fileext = ".yaml") + on.exit(unlink(p), add = TRUE) + writeLines( + "version: 1\ndefaults: {keys: [id], na_equal: yes}\nrow_validation: {check_count: no}\nby_type:\n numeric: {abs: 0.001}\n", + con = p + ) + res <- suppressMessages(suppressWarnings(compare_datasets_from_yaml( + dplyr::tbl(con, "ref_struct"), dplyr::tbl(con, "cand_struct"), + key = "id", path = p + ))) + # The missing column is the only failure: the agent verdict must agree + # with the coverage, not pass on a 0-unit dummy step + expect_false(res$all_passed) + expect_false(pointblank::all_passed(res$reponse)) + expect_identical(res$missing_in_candidate, "extra_ref") +}) + +test_that("key-only lazy comparison runs without value columns", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + + duckdb::dbWriteTable(con, "ref_keys", data.frame(id = 1:3)) + duckdb::dbWriteTable(con, "cand_keys", data.frame(id = 1:3)) + + p <- tempfile(fileext = ".yaml") + on.exit(unlink(p), add = TRUE) + writeLines( + "version: 1\ndefaults: {keys: [id], na_equal: yes}\nrow_validation: {check_count: no}\n", + con = p + ) + res <- suppressMessages(compare_datasets_from_yaml( + dplyr::tbl(con, "ref_keys"), dplyr::tbl(con, "cand_keys"), + key = "id", path = p + )) + expect_true(res$all_passed) +}) diff --git a/vignettes/datadiff.Rmd b/vignettes/datadiff.Rmd index ddb84ad..530fbf5 100644 --- a/vignettes/datadiff.Rmd +++ b/vignettes/datadiff.Rmd @@ -165,6 +165,10 @@ failed_rows The `type = "pass"` variant returns rows that passed all steps. This is useful to understand the scope of the problem before investigating further. +Note: on an all-pass LAZY comparison, `reponse` carries a constant-size +placeholder rather than the source rows (the verdict is computed by SQL +aggregates without loading the data into R), so sundering is only +meaningful on a failing comparison. For each failing tolerance column, the step extracts returned by `pointblank::get_data_extracts()` (and the CSV download in the HTML report) From 72a72b85cfa07ffdd7c2ca8115da5e8daeaaea10 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 01:27:43 +0200 Subject: [PATCH 29/40] fix: equal_mode normalized implique case_insensitive + trim sauf override (issue #27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Le champ n'etait lu qu'a un endroit ou identical(eq_mode, "normalized") activait normalize_text(case_insensitive = FALSE, trim = FALSE) - l'identite. Un utilisateur ecrivant equal_mode: normalized seul n'obtenait rien, silencieusement, et test-utils verrouillait meme l'inertie. La vignette documentait pourtant deja « Apply both transformations » : le code s'aligne sur sa doc. - normalized implique les deux normalisations pour les colonnes character, un flag explicite (meme FALSE) gagne sur le mode - date/datetime/logical : *_equal_mode documentes acceptes-mais-inertes (la normalisation ne touche que character) - test verrouillant l'inertie retourne + reprex de l'issue en test --- NEWS.md | 14 ++++++++ R/compare_datasets_from_yaml.R | 24 +++++++++++-- R/preprocessing.R | 20 ++++++++--- man/write_rules_template.Rd | 15 +++++--- tests/testthat/test-utils.R | 64 +++++++++++++++++++++++++++++++--- 5 files changed, 121 insertions(+), 16 deletions(-) diff --git a/NEWS.md b/NEWS.md index 73be078..a306e59 100644 --- a/NEWS.md +++ b/NEWS.md @@ -88,6 +88,20 @@ ## Bug fixes +* `equal_mode: normalized` now does what the vignette always said: it implies + `case_insensitive = TRUE` and `trim = TRUE` for character columns unless + those flags are set explicitly (an explicit value wins over the mode). It + previously activated a branch that applied the identity transformation: a + user writing `equal_mode: normalized` alone got nothing, silently, and a + test even locked that inertia. `write_rules_template()` stops co-writing + the default FALSE flags next to a "normalized" mode (they neutralised the + implication). Note: a hand-written YAML using `equal_mode: normalized` + without explicit flags now normalizes where it previously compared exact, + so verdicts on such configs can flip from FAIL to PASS; re-run rather than + assume stability. The `date`/`datetime`/`logical` equal-mode parameters are + documented as accepted-but-inert for non-character types (text + normalization only touches character columns) (issue #27). + * Factor columns are now compared as the character values they display: preprocessing converts them, so the text normalization rules (`case_insensitive`, `trim`) apply to them and the verdict no longer depends diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 91f3565..259d554 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -18,12 +18,19 @@ #' @param numeric_abs Default absolute tolerance for numeric columns #' @param numeric_rel Default relative tolerance for numeric columns #' @param integer_abs Default absolute tolerance for integer columns -#' @param character_equal_mode Default comparison mode for character columns ("exact", "normalized") +#' @param character_equal_mode Default comparison mode for character columns: +#' "exact", or "normalized" which implies `case_insensitive = TRUE` and +#' `trim = TRUE` unless those flags are set explicitly (an explicit value +#' always wins over the mode) #' @param character_case_insensitive Logical for case-insensitive character comparison #' @param character_trim Logical for trimming whitespace in character comparison -#' @param date_equal_mode Default comparison mode for date columns +#' @param date_equal_mode Default comparison mode for date columns. Only +#' "exact" has an effect: text normalization applies to character columns +#' only, so "normalized" is accepted but changes nothing for this type. #' @param datetime_equal_mode Default comparison mode for datetime columns +#' (same caveat as `date_equal_mode`) #' @param logical_equal_mode Default comparison mode for logical columns +#' (same caveat as `date_equal_mode`) #' @return The \code{path} to the written YAML file, returned invisibly. #' @importFrom yaml write_yaml #' @importFrom stats setNames @@ -67,6 +74,17 @@ write_rules_template <- function(data_reference, } types <- detect_column_types(.ref_schema) + # With equal_mode "normalized", writing the default FALSE flags would + # neutralise the mode's implication (an explicit flag wins over the mode): + # only include the flags the caller actually supplied + character_rules <- list(equal_mode = character_equal_mode) + if (!identical(character_equal_mode, "normalized") || + !missing(character_case_insensitive)) { + character_rules$case_insensitive <- character_case_insensitive + } + if (!identical(character_equal_mode, "normalized") || !missing(character_trim)) { + character_rules$trim <- character_trim + } y <- list( version = version, defaults = list(na_equal = na_equal_default, @@ -78,7 +96,7 @@ write_rules_template <- function(data_reference, by_type = list( numeric = list(abs = numeric_abs, rel = numeric_rel), integer = list(abs = integer_abs), - character = list(equal_mode = character_equal_mode, case_insensitive = character_case_insensitive, trim = character_trim), + character = character_rules, date = list(equal_mode = date_equal_mode), datetime = list(equal_mode = datetime_equal_mode), logical = list(equal_mode = logical_equal_mode) diff --git a/R/preprocessing.R b/R/preprocessing.R index 678068d..10e7c54 100644 --- a/R/preprocessing.R +++ b/R/preprocessing.R @@ -65,12 +65,22 @@ preprocess_dataframe <- function(df, col_rules, schema = NULL) { } for (nm in names(col_rules)) { cr <- col_rules[[nm]] - eq_mode <- cr$equal_mode %||% "exact" - case_insensitive <- isTRUE(cr$case_insensitive) - trim <- isTRUE(cr$trim) + normalized <- identical(cr$equal_mode %||% "exact", "normalized") - # Apply normalization if equal_mode is "normalized" OR if case_insensitive/trim is TRUE - should_normalize <- identical(eq_mode, "normalized") || case_insensitive || trim + # equal_mode "normalized" implies BOTH text normalizations unless the + # rule sets them explicitly (an explicit FALSE wins over the mode) + case_insensitive <- if (is.null(cr$case_insensitive)) { + normalized + } else { + isTRUE(cr$case_insensitive) + } + trim <- if (is.null(cr$trim)) { + normalized + } else { + isTRUE(cr$trim) + } + + should_normalize <- case_insensitive || trim # Determine column type: use schema for lazy tables, otherwise inspect df # directly. A factor in the schema counts as character: the local values diff --git a/man/write_rules_template.Rd b/man/write_rules_template.Rd index 30e42c9..b153de3 100644 --- a/man/write_rules_template.Rd +++ b/man/write_rules_template.Rd @@ -54,17 +54,24 @@ comparison. If NULL, comparison is positional (row by row).} \item{integer_abs}{Default absolute tolerance for integer columns} -\item{character_equal_mode}{Default comparison mode for character columns ("exact", "normalized")} +\item{character_equal_mode}{Default comparison mode for character columns: +"exact", or "normalized" which implies \code{case_insensitive = TRUE} and +\code{trim = TRUE} unless those flags are set explicitly (an explicit value +always wins over the mode)} \item{character_case_insensitive}{Logical for case-insensitive character comparison} \item{character_trim}{Logical for trimming whitespace in character comparison} -\item{date_equal_mode}{Default comparison mode for date columns} +\item{date_equal_mode}{Default comparison mode for date columns. Only +"exact" has an effect: text normalization applies to character columns +only, so "normalized" is accepted but changes nothing for this type.} -\item{datetime_equal_mode}{Default comparison mode for datetime columns} +\item{datetime_equal_mode}{Default comparison mode for datetime columns +(same caveat as \code{date_equal_mode})} -\item{logical_equal_mode}{Default comparison mode for logical columns} +\item{logical_equal_mode}{Default comparison mode for logical columns +(same caveat as \code{date_equal_mode})} } \value{ The \code{path} to the written YAML file, returned invisibly. diff --git a/tests/testthat/test-utils.R b/tests/testthat/test-utils.R index b56b33a..128dec1 100644 --- a/tests/testthat/test-utils.R +++ b/tests/testthat/test-utils.R @@ -47,15 +47,42 @@ test_that("preprocess_dataframe applies transformations correctly", { expect_equal(result$case_col, c("mixed", "case")) }) -test_that("preprocess_dataframe handles equal_mode normalized", { +test_that("equal_mode normalized implies case_insensitive and trim by default", { df <- data.frame(text_col = c(" HELLO ", "WORLD")) rules <- list(text_col = list(equal_mode = "normalized")) - result <- preprocess_dataframe(df, rules) + expect_equal(result$text_col, c("hello", "world")) + + # An explicit flag overrides the implied default + rules_no_trim <- list(text_col = list(equal_mode = "normalized", trim = FALSE)) + result_no_trim <- preprocess_dataframe(df, rules_no_trim) + expect_equal(result_no_trim$text_col, c(" hello ", "world")) - # Should not apply any transformation since case_insensitive and trim are not set - expect_equal(result$text_col, c(" HELLO ", "WORLD")) + rules_no_case <- list(text_col = list(equal_mode = "normalized", + case_insensitive = FALSE)) + result_no_case <- preprocess_dataframe(df, rules_no_case) + expect_equal(result_no_case$text_col, c("HELLO", "WORLD")) +}) + +test_that("equal_mode normalized alone changes the verdict (issue reprex)", { + ref <- data.frame(id = 1:2, s = c("A", " b"), stringsAsFactors = FALSE) + cand <- data.frame(id = 1:2, s = c("a", "b"), stringsAsFactors = FALSE) + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(' +version: 1 +defaults: {keys: [id], na_equal: true} +row_validation: {check_count: false} +by_type: + character: {equal_mode: normalized} +', con = yaml_path) + + res <- suppressMessages( + compare_datasets_from_yaml(ref, cand, key = "id", path = yaml_path) + ) + expect_true(res$all_passed) }) test_that("preprocess_dataframe handles mixed rules", { @@ -112,3 +139,32 @@ test_that("preprocess_dataframe applies stringr trim on Arrow objects", { expect_equal(result$name, c("Alice", "Bob")) }) + +test_that("a normalized template does not neutralise its own mode", { + ref <- data.frame(id = 1:2, s = c("A", " b"), stringsAsFactors = FALSE) + cand <- data.frame(id = 1:2, s = c("a", "b"), stringsAsFactors = FALSE) + + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + write_rules_template(ref, key = "id", path = yaml_path, + character_equal_mode = "normalized") + + rules <- read_rules(yaml_path) + # The default FALSE flags must not be co-written next to the mode + expect_null(rules$by_type$character$case_insensitive) + expect_null(rules$by_type$character$trim) + + res <- suppressMessages( + compare_datasets_from_yaml(ref, cand, key = "id", path = yaml_path) + ) + expect_true(res$all_passed) + + # An explicitly supplied flag still wins over the mode + yaml_path2 <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path2), add = TRUE) + write_rules_template(ref, key = "id", path = yaml_path2, + character_equal_mode = "normalized", + character_trim = FALSE) + rules2 <- read_rules(yaml_path2) + expect_false(rules2$by_type$character$trim) +}) From e30f8c39deaea4a86842c1d394594bb46736689e Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:11:55 +0200 Subject: [PATCH 30/40] fix: setup_pointblank_agent - cols_reference deprecie, steps egalite sur booleen derive (issue #25) - cols_reference jamais lu depuis son introduction : NULL par defaut + warning de depreciation quand fourni ; l'appelant pipeline passe en arguments nommes sans lui ; variables mortes cols_reference/ cols_candidate supprimees de l'appelant - les steps d'egalite locaux valident un booleen __eq derive en amont de create_agent (semantique NA partagee : unilateral echoue, bilateral suit na_equal) au lieu d'embarquer le vecteur de reference entier dans chaque step (O(lignes) par step, serialise avec le rapport, na_pass incapable d'exprimer la distinction) ; un cmp lazy sans __eq precalcule recoit une erreur de contrat explicite - exemple roxygen executable et representatif (il ciblait b__ok inexistant : l'agent produit echouait a l'interrogation) - get_col_names hisse hors boucle, calcule apres toutes les mutations - tests : depreciation, semantique NA des appels directs (rouge avant), appels legacy du fichier de tests passes en arguments nommes --- NEWS.md | 11 ++++ R/compare_datasets_from_yaml.R | 23 +++---- R/pointblank_setup.R | 85 +++++++++++++++++++------- man/setup_pointblank_agent.Rd | 33 +++++++--- tests/testthat/test-pointblank-setup.R | 73 ++++++++++++++++++---- 5 files changed, 171 insertions(+), 54 deletions(-) diff --git a/NEWS.md b/NEWS.md index a306e59..5cc91b5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -88,6 +88,17 @@ ## Bug fixes +* `setup_pointblank_agent()` cleanup: the `cols_reference` argument was never + read (verified by grep since its introduction) and is now deprecated + (warning when supplied, removal planned); the local equality steps validate + a derived `__eq` boolean with the shared NA semantics instead of + embedding the whole reference vector in each step (O(rows) per step, + serialised with the report, and unable to express the one-sided/two-sided + NA distinction); the roxygen example is now executable and representative + (it used to target a `__ok` column that did not exist); `get_col_names()` + is hoisted out of the per-column loop; the dead `cols_reference`/ + `cols_candidate` locals of the caller are gone (issue #25). + * `equal_mode: normalized` now does what the vignette always said: it implies `case_insensitive = TRUE` and `trim = TRUE` for character columns unless those flags are set explicitly (an explicit value wins over the mode). It diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 259d554..0970389 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -540,8 +540,6 @@ compare_datasets_from_yaml <- function(data_reference, # Analyze columns col_analysis <- analyze_columns(data_reference, data_candidate, ignore_columns = ignore_columns) - cols_reference <- col_analysis$cols_reference - cols_candidate <- col_analysis$cols_candidate missing_in_candidate <- col_analysis$missing_in_candidate extra_in_candidate <- col_analysis$extra_in_candidate common_cols <- col_analysis$common_cols @@ -864,17 +862,16 @@ compare_datasets_from_yaml <- function(data_reference, } agent <- setup_pointblank_agent( cmp_for_agent, - cols_reference, - fail$eq, - fail$tol, - row_validation_info, - ref_suffix, - warn_at, - stop_at, - label, - na_equal, - lang, - locale, + common_cols = fail$eq, + tol_cols = fail$tol, + row_validation_info = row_validation_info, + ref_suffix = ref_suffix, + warn_at = warn_at, + stop_at = stop_at, + label = label, + na_equal = na_equal, + lang = lang, + locale = locale, missing_in_candidate = missing_in_candidate, type_mismatch_cols = type_mismatch_cols, add_col_exists_steps = FALSE diff --git a/R/pointblank_setup.R b/R/pointblank_setup.R index b3dc16c..2173113 100644 --- a/R/pointblank_setup.R +++ b/R/pointblank_setup.R @@ -3,9 +3,19 @@ #' Creates and configures a pointblank validation agent with all necessary validation steps #' including column existence checks, exact value comparisons, and tolerance validations. #' +#' The equality steps validate a `__eq` boolean (one-sided NA fails, +#' two-sided NA follows `na_equal`, matching the tolerance kernel and the lazy +#' SQL): when `cmp` does not already carry it, it is derived on the fly for a +#' local data.frame. Tolerance steps expect the `__ok` booleans to be +#' precomputed (see [add_tolerance_columns()]). A lazy `cmp` must carry every +#' boolean column already. +#' #' @param cmp Comparison dataframe with candidate and reference data -#' @param cols_reference Character vector of reference column names -#' @param common_cols Character vector of columns present in both datasets +#' @param cols_reference Deprecated and unused; supplying any non-NULL value +#' raises a warning (an explicit NULL is silent) and the argument will be +#' removed in a future release +#' @param common_cols Character vector of equality columns to validate (the +#' internal pipeline passes only the failing ones on its failure path) #' @param tol_cols Character vector of columns with tolerance rules #' @param row_validation_info List with row validation information from validate_row_counts #' @param ref_suffix Suffix for reference columns @@ -29,21 +39,36 @@ #' data columns. #' @return Configured pointblank agent ready for interrogation #' @examples -#' cmp <- data.frame(a = 1:3, a__reference = 1:3, b = c(1.1, 2.2, 3.3), -#' b__reference = c(1.0, 2.0, 3.0)) +#' cmp <- data.frame( +#' a = 1:3, a__reference = 1:3, +#' b = c(1.1, 2.2, 3.3), b__reference = c(1.0, 2.0, 3.0), +#' b__ok = c(FALSE, FALSE, FALSE) +#' ) #' row_info <- list(check_count = FALSE) -#' setup_pointblank_agent(cmp, c("a", "b"), c("a", "b"), "b", row_info, -#' "__reference", 0.1, 0.1, "Test", TRUE) +#' agent <- setup_pointblank_agent( +#' cmp, +#' common_cols = "a", tol_cols = "b", row_validation_info = row_info, +#' ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, +#' label = "Test", na_equal = TRUE +#' ) +#' pointblank::all_passed(pointblank::interrogate(agent)) #' @importFrom pointblank create_agent col_exists col_vals_equal action_levels #' @importFrom tidyselect all_of #' @importFrom rlang := #' @export -setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, +setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_cols, row_validation_info = NULL, ref_suffix, warn_at, stop_at, label, na_equal, lang = "fr", locale = "fr_FR", missing_in_candidate = character(0), type_mismatch_cols = character(0), add_col_exists_steps = TRUE) { + if (!is.null(cols_reference)) { + warning( + "The 'cols_reference' argument of setup_pointblank_agent() is deprecated and unused; it will be removed in a future release.", + call. = FALSE + ) + } + # Add dummy columns for missing columns BEFORE creating the agent # These columns are set to FALSE and we'll check they equal TRUE (will fail) for (c in missing_in_candidate) { @@ -66,6 +91,25 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, } } + # Derive the missing __eq booleans BEFORE creating the agent, so every + # equality step validates the same boolean the verdict logic uses (shared NA + # semantics) instead of embedding the whole reference vector in the step + # (O(n_rows) per step, serialised with the report, and na_pass cannot + # express the one-sided/two-sided NA distinction). + eq_cols <- setdiff(x = common_cols, y = tol_cols) + if (!is_non_local(cmp)) { + for (c in eq_cols) { + eq_col <- datadiff_eq_col(c) + if (!(eq_col %in% get_col_names(cmp))) { + cmp[[eq_col]] <- eq_col_bool( + cmp, col = c, ref_suffix = ref_suffix, na_equal = na_equal + ) + } + } + } + # Hoisted once, AFTER every cmp mutation above + col_names <- get_col_names(cmp) + agent <- create_agent(tbl = cmp, label = label, actions = action_levels(warn_at = warn_at, stop_at = stop_at), lang = lang, @@ -105,22 +149,19 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, ) } - eq_cols <- setdiff(x = common_cols, y = tol_cols) for (c in eq_cols) { - eq_precomputed <- datadiff_eq_col(c) - if (eq_precomputed %in% get_col_names(cmp)) { - # Lazy path: use pre-computed boolean equality column - agent <- agent %>% - col_vals_equal(columns = all_of(eq_precomputed), value = TRUE, na_pass = FALSE) - } else { - # Local path: compare directly against reference column values - agent <- agent %>% - col_vals_equal( - columns = all_of(c), - value = cmp[[paste0(c, ref_suffix)]], - na_pass = na_equal - ) + eq_col <- datadiff_eq_col(c) + if (!(eq_col %in% col_names)) { + # Only reachable with a lazy cmp missing its precomputed boolean: + # the documented contract requires it (deriving it here would need + # the raw columns, absent from the slim lazy table) + stop(sprintf( + "equality boolean column '%s' missing from cmp (a lazy cmp must carry the precomputed __eq columns)", + eq_col + ), call. = FALSE) } + agent <- agent %>% + col_vals_equal(columns = all_of(eq_col), value = TRUE, na_pass = FALSE) } for (c in tol_cols) { @@ -130,7 +171,7 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols, # Add row count validation if needed # Use colnames() instead of names() to work correctly with lazy tables - if (!is.null(row_validation_info) && isTRUE(row_validation_info$check_count) && "row_count_ok" %in% get_col_names(cmp)) { + if (!is.null(row_validation_info) && isTRUE(row_validation_info$check_count) && "row_count_ok" %in% col_names) { agent <- agent %>% col_vals_equal(columns = all_of("row_count_ok"), value = TRUE, na_pass = FALSE) } diff --git a/man/setup_pointblank_agent.Rd b/man/setup_pointblank_agent.Rd index 0f22844..7ef1bc3 100644 --- a/man/setup_pointblank_agent.Rd +++ b/man/setup_pointblank_agent.Rd @@ -6,7 +6,7 @@ \usage{ setup_pointblank_agent( cmp, - cols_reference, + cols_reference = NULL, common_cols, tol_cols, row_validation_info = NULL, @@ -25,9 +25,12 @@ setup_pointblank_agent( \arguments{ \item{cmp}{Comparison dataframe with candidate and reference data} -\item{cols_reference}{Character vector of reference column names} +\item{cols_reference}{Deprecated and unused; supplying any non-NULL value +raises a warning (an explicit NULL is silent) and the argument will be +removed in a future release} -\item{common_cols}{Character vector of columns present in both datasets} +\item{common_cols}{Character vector of equality columns to validate (the +internal pipeline passes only the failing ones on its failure path)} \item{tol_cols}{Character vector of columns with tolerance rules} @@ -69,10 +72,26 @@ Configured pointblank agent ready for interrogation Creates and configures a pointblank validation agent with all necessary validation steps including column existence checks, exact value comparisons, and tolerance validations. } +\details{ +The equality steps validate a \verb{__eq} boolean (one-sided NA fails, +two-sided NA follows \code{na_equal}, matching the tolerance kernel and the lazy +SQL): when \code{cmp} does not already carry it, it is derived on the fly for a +local data.frame. Tolerance steps expect the \verb{__ok} booleans to be +precomputed (see \code{\link[=add_tolerance_columns]{add_tolerance_columns()}}). A lazy \code{cmp} must carry every +boolean column already. +} \examples{ -cmp <- data.frame(a = 1:3, a__reference = 1:3, b = c(1.1, 2.2, 3.3), - b__reference = c(1.0, 2.0, 3.0)) +cmp <- data.frame( + a = 1:3, a__reference = 1:3, + b = c(1.1, 2.2, 3.3), b__reference = c(1.0, 2.0, 3.0), + b__ok = c(FALSE, FALSE, FALSE) +) row_info <- list(check_count = FALSE) -setup_pointblank_agent(cmp, c("a", "b"), c("a", "b"), "b", row_info, - "__reference", 0.1, 0.1, "Test", TRUE) +agent <- setup_pointblank_agent( + cmp, + common_cols = "a", tol_cols = "b", row_validation_info = row_info, + ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, + label = "Test", na_equal = TRUE +) +pointblank::all_passed(pointblank::interrogate(agent)) } diff --git a/tests/testthat/test-pointblank-setup.R b/tests/testthat/test-pointblank-setup.R index 3b28d8b..ec7611b 100644 --- a/tests/testthat/test-pointblank-setup.R +++ b/tests/testthat/test-pointblank-setup.R @@ -12,14 +12,14 @@ test_that("setup_pointblank_agent creates valid agent", { text__reference = c("a", "b", "c") ) - cols_reference <- c("id", "value", "text") common_cols <- c("id", "value", "text") tol_cols <- c("value") rules <- list() ref_suffix <- "__reference" # Test agent creation - agent <- setup_pointblank_agent(cmp, cols_reference, common_cols, tol_cols, rules, ref_suffix, + agent <- setup_pointblank_agent(cmp, common_cols = common_cols, tol_cols = tol_cols, + row_validation_info = rules, ref_suffix = ref_suffix, warn_at = 0.1, stop_at = 0.1, label = "Test", na_equal = TRUE) # Check that agent was created @@ -40,13 +40,13 @@ test_that("setup_pointblank_agent handles different column types", { text__reference = c("a", "b") ) - cols_reference <- c("id", "text") common_cols <- c("id", "text") tol_cols <- character(0) # No tolerance columns rules <- list() ref_suffix <- "__reference" - agent <- setup_pointblank_agent(cmp, cols_reference, common_cols, tol_cols, rules, ref_suffix, + agent <- setup_pointblank_agent(cmp, common_cols = common_cols, tol_cols = tol_cols, + row_validation_info = rules, ref_suffix = ref_suffix, warn_at = 0.5, stop_at = 0.5, label = "Test Exact", na_equal = FALSE) expect_s3_class(agent, "ptblank_agent") @@ -57,7 +57,8 @@ test_that("setup_pointblank_agent handles empty inputs", { # Test with minimal data cmp <- data.frame(id = 1, id__reference = 1) - agent <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 1.0, stop_at = 1.0, label = "Minimal", na_equal = TRUE) expect_s3_class(agent, "ptblank_agent") @@ -67,7 +68,8 @@ test_that("setup_pointblank_agent handles empty inputs", { test_that("setup_pointblank_agent uses French language by default", { cmp <- data.frame(id = 1, id__reference = 1) - agent <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test", na_equal = TRUE) expect_s3_class(agent, "ptblank_agent") @@ -79,7 +81,8 @@ test_that("setup_pointblank_agent accepts custom language parameters", { cmp <- data.frame(id = 1, id__reference = 1) # Test English - agent_en <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_en <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test EN", na_equal = TRUE, lang = "en", locale = "en_US") @@ -88,7 +91,8 @@ test_that("setup_pointblank_agent accepts custom language parameters", { expect_equal(agent_en$locale, "en_US") # Test German - agent_de <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_de <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test DE", na_equal = TRUE, lang = "de", locale = "de_DE") @@ -101,21 +105,24 @@ test_that("setup_pointblank_agent accepts various supported languages", { cmp <- data.frame(id = 1, id__reference = 1) # Test Spanish - agent_es <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_es <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test ES", na_equal = TRUE, lang = "es", locale = "es_ES") expect_equal(agent_es$lang, "es") expect_equal(agent_es$locale, "es_ES") # Test Portuguese - agent_pt <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_pt <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test PT", na_equal = TRUE, lang = "pt", locale = "pt_BR") expect_equal(agent_pt$lang, "pt") expect_equal(agent_pt$locale, "pt_BR") # Test Italian - agent_it <- setup_pointblank_agent(cmp, c("id"), c("id"), character(0), list(), "__reference", + agent_it <- setup_pointblank_agent(cmp, common_cols = c("id"), tol_cols = character(0), + row_validation_info = list(), ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, label = "Test IT", na_equal = TRUE, lang = "it", locale = "it_IT") expect_equal(agent_it$lang, "it") @@ -135,7 +142,6 @@ test_that("setup_pointblank_agent handles lazy table with missing_in_candidate a # Passing a lazy table exercises the dplyr::mutate() path (lines 52 and 63) agent <- setup_pointblank_agent( lazy_tbl, - cols_reference = character(0), common_cols = character(0), tol_cols = c("val"), row_validation_info = list(check_count = FALSE), @@ -150,3 +156,46 @@ test_that("setup_pointblank_agent handles lazy table with missing_in_candidate a expect_s3_class(agent, "ptblank_agent") }) + +test_that("cols_reference is deprecated and unused", { + cmp <- data.frame(a = 1:3, a__reference = 1:3) + row_info <- list(check_count = FALSE) + expect_warning( + setup_pointblank_agent(cmp, cols_reference = c("a"), common_cols = "a", + tol_cols = character(0), row_validation_info = row_info, + ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, + label = "x", na_equal = TRUE), + regexp = "deprecated" + ) + # Omitting it works, produces the same agent shape and stays silent + expect_no_warning( + agent <- setup_pointblank_agent(cmp, common_cols = "a", + tol_cols = character(0), row_validation_info = row_info, + ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, + label = "x", na_equal = TRUE) + ) + expect_s3_class(agent, "ptblank_agent") +}) + +test_that("direct equality steps use the shared NA semantics via a derived boolean", { + # One-sided NA with na_equal = TRUE: must FAIL (kernel/lazy semantics), + # not pass through col_vals_equal's na_pass + cmp <- data.frame( + s = c("a", NA, "c"), + s__reference = c("a", "b", "c"), + stringsAsFactors = FALSE + ) + row_info <- list(check_count = FALSE) + agent <- setup_pointblank_agent(cmp, common_cols = "s", + tol_cols = character(0), row_validation_info = row_info, + ref_suffix = "__reference", warn_at = 0.1, stop_at = 0.1, + label = "na semantics", na_equal = TRUE) + reponse <- pointblank::interrogate(agent) + expect_false(pointblank::all_passed(reponse)) + + # The step no longer embeds the reference VECTOR in the agent: the equality + # is validated through a derived __eq boolean column + vs <- reponse$validation_set + step_cols <- vapply(vs$column, function(cc) cc[1], character(1)) + expect_true("s__eq" %in% step_cols) +}) From e20f848085a1db4aeeb7066bc1ac4b13d3fefe36 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:17:51 +0200 Subject: [PATCH 31/40] fix: contradictions NEWS/code resolues - %||% interne, label comparison, doc lang exacte (issue #26) Le commit Perf (#1) post-0.4.4 avait re-exporte %||% (masquant rlang::%||% et base::%||% R >= 4.4 au chargement), re-bascule le label par defaut en francais et laisse la vignette pretendre un defaut anglais - le tout sans entree NEWS, contredisant les annonces de la 0.4.4. - %||% re-internalise (conforme NEWS 0.4.4) ; le test qui verrouillait l'export accidentel verrouille desormais l'inverse - prefixe de label du template : comparison (annonce 0.4.4) - vignette : defaut de langue reel documente (fr / fr_FR + options) - normalize_text, validate_row_counts, setup_pointblank_agent documentes dans la section utilitaires de la vignette avec cas d'usage reels (ils n'apparaissaient nulle part) --- NAMESPACE | 1 - NEWS.md | 12 ++++++++++ R/compare_datasets_from_yaml.R | 4 ++-- R/utils.R | 18 +++++++------- man/or_operator.Rd | 20 ---------------- tests/testthat/test-input-validation.R | 16 ++++++------- tests/testthat/test-utils.R | 5 ++-- vignettes/datadiff.Rmd | 33 +++++++++++++++++++++++++- 8 files changed, 66 insertions(+), 43 deletions(-) delete mode 100644 man/or_operator.Rd diff --git a/NAMESPACE b/NAMESPACE index 9234e31..9c201c6 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -2,7 +2,6 @@ S3method(print,datadiff_coverage) S3method(print,datadiff_report) -export("%||%") export(add_tolerance_columns) export(analyze_columns) export(compare_datasets_from_yaml) diff --git a/NEWS.md b/NEWS.md index 5cc91b5..cbb9a2c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -88,6 +88,18 @@ ## Bug fixes +* Three NEWS-vs-code contradictions introduced by the "Perf (#1)" commit after + the 0.4.4 release are resolved in the direction 0.4.4 had announced or the + docs now tell the truth: `%||%` is internal again (exporting it masked + rlang's and base R's own operator at load time; the test that locked the + accidental re-export now locks the opposite), the default rules-template + label prefix is "comparison" (was still the French "comparaison"), and the + vignette states the real report language default (`lang = "fr"`, it claimed + English). The three exported helpers that no user-facing doc mentioned + (`normalize_text()`, `validate_row_counts()`, `setup_pointblank_agent()`) + are now documented in the vignette's utility-functions section with real + use cases (issue #26). + * `setup_pointblank_agent()` cleanup: the `cols_reference` argument was never read (verified by grep since its introduction) and is now deprecated (warning when supplied, removal planned); the local equality steps validate diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 0970389..6f2d58d 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -70,8 +70,8 @@ write_rules_template <- function(data_reference, } } validate_label(label) - if (is.null(label) || label == "") {label <- paste("comparaison", deparse1(substitute(data_reference))) - + if (is.null(label) || label == "") { + label <- paste("comparison", deparse1(substitute(data_reference))) } types <- detect_column_types(.ref_schema) # With equal_mode "normalized", writing the default FALSE flags would diff --git a/R/utils.R b/R/utils.R index a8dba33..f38d9af 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,12 +1,12 @@ -#' Operator for default values -#' -#' Returns y if x is NULL, otherwise returns x -#' @name or_operator -#' @param x Value to check -#' @param y Default value to return if x is NULL -#' @return x if not NULL, otherwise y -#' @export -`%||%` <- function(x, y) if (is.null(x)) y else x +# Default-value operator: y if x is NULL, otherwise x. Internal: exporting it +# would mask rlang's and base R's (R >= 4.4) own %||% at load time. +`%||%` <- function(x, y) { + if (is.null(x)) { + y + } else { + x + } +} get_col_names <- function(x) { nms <- colnames(x) diff --git a/man/or_operator.Rd b/man/or_operator.Rd deleted file mode 100644 index 20863e8..0000000 --- a/man/or_operator.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R -\name{or_operator} -\alias{or_operator} -\alias{\%||\%} -\title{Operator for default values} -\usage{ -x \%||\% y -} -\arguments{ -\item{x}{Value to check} - -\item{y}{Default value to return if x is NULL} -} -\value{ -x if not NULL, otherwise y -} -\description{ -Returns y if x is NULL, otherwise returns x -} diff --git a/tests/testthat/test-input-validation.R b/tests/testthat/test-input-validation.R index 2811fc0..3f1d49f 100644 --- a/tests/testthat/test-input-validation.R +++ b/tests/testthat/test-input-validation.R @@ -138,18 +138,18 @@ test_that("write_rules_template handles label edge cases", { df <- data.frame(id = 1:2, value = 1:2) # Test NULL label (should use default) - template_path <- "test_null_label.yaml" + template_path <- tempfile(fileext = ".yaml") + on.exit(unlink(template_path), add = TRUE) write_rules_template(df, key = "id", label = NULL, path = template_path) rules <- read_rules(template_path) - expect_match(rules$defaults$label, "comparaison") - unlink(template_path) + expect_match(rules$defaults$label, "comparison") # Test empty label (should use default) - template_path <- "test_empty_label.yaml" - write_rules_template(df, key = "id", label = "", path = template_path) - rules <- read_rules(template_path) - expect_match(rules$defaults$label, "comparaison") - unlink(template_path) + template_path2 <- tempfile(fileext = ".yaml") + on.exit(unlink(template_path2), add = TRUE) + write_rules_template(df, key = "id", label = "", path = template_path2) + rules <- read_rules(template_path2) + expect_match(rules$defaults$label, "comparison") # Test very long label long_label <- paste(rep("very_long_label", 100), collapse = "_") diff --git a/tests/testthat/test-utils.R b/tests/testthat/test-utils.R index 128dec1..e52e62f 100644 --- a/tests/testthat/test-utils.R +++ b/tests/testthat/test-utils.R @@ -5,8 +5,9 @@ test_that("%||% operator works correctly", { expect_equal(object = NULL %||% NULL, expected = NULL) }) -test_that("%||% is exported from the package namespace", { - expect_true("%||%" %in% getNamespaceExports("datadiff")) +test_that("%||% is internal (not exported), per NEWS 0.4.4", { + # Exporting it masks rlang's and base R's (R >= 4.4) own %||% at load time + expect_false("%||%" %in% getNamespaceExports("datadiff")) }) test_that("normalize_text handles basic cases", { diff --git a/vignettes/datadiff.Rmd b/vignettes/datadiff.Rmd index 530fbf5..6003f36 100644 --- a/vignettes/datadiff.Rmd +++ b/vignettes/datadiff.Rmd @@ -801,11 +801,42 @@ cmp_annotated <- add_tolerance_columns(cmp, "value", rules_debug, cmp_annotated[, c("value__absdiff", "value__thresh", "value__ok")] ``` +### `normalize_text()` + +The text-normalization primitive used by `preprocess_dataframe()`, handy on +its own to normalize a vector the same way a comparison would: + +```{r normalize-text} +normalize_text(c(" Hello ", "WORLD"), case_insensitive = TRUE, trim = TRUE) +``` + +### `validate_row_counts()` + +Computes the row-count validation information (counts, expected count, +tolerance) for a pair of datasets and a rules object, without running the +full comparison: + +```{r validate-row-counts} +rules_rc <- list(row_validation = list(check_count = TRUE, + expected_count = NULL, tolerance = 0)) +validate_row_counts(ref, cand, rules = rules_rc) +``` + +### `setup_pointblank_agent()` + +Builds the configured (not yet interrogated) pointblank agent the comparison +uses, for advanced consumers who want to add their own steps before +interrogating. The equality steps validate `__eq` booleans and the +tolerance steps expect precomputed `__ok` booleans (see +`add_tolerance_columns()`); most users never need to call it directly. + --- ## 14. Language and locale -By default, pointblank reports are rendered in English. You can change +By default, pointblank reports are rendered in French (`lang = "fr"`, +`locale = "fr_FR"`; override globally with +`options(datadiff.lang = "en", datadiff.locale = "en_US")`). You can change the language per call or globally for a session. ### Per call From 477f840c269c4689ae3bbae0477f686b4ca0b703 Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:18:51 +0200 Subject: [PATCH 32/40] docs: RFC surface d'API 0.5 (issue #33) Document de design a discuter, sans implementation : nom compare_datasets() + alias retrocompatible, objets d'options extract_opts()/engine_opts(), retour de classe datadiff_result (passed/report, depreciation de reponse/all_passed dupliques), fail_at unique, bascule lang en a la 0.5, cycle de depreciation en 3 versions. Les quick wins non cassants sont rattaches aux issues deja traitees (#16/#20/#25/#26/#27/#30). --- .Rbuildignore | 1 + dev/RFC-api-0.5.md | 117 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 dev/RFC-api-0.5.md diff --git a/.Rbuildignore b/.Rbuildignore index 293d000..48144d0 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -13,3 +13,4 @@ ^README\.Rmd$ ^doc$ ^Meta$ +^dev$ diff --git a/dev/RFC-api-0.5.md b/dev/RFC-api-0.5.md new file mode 100644 index 0000000..5fa6787 --- /dev/null +++ b/dev/RFC-api-0.5.md @@ -0,0 +1,117 @@ +# RFC : surface d'API datadiff 0.5 + +Statut : proposition a discuter (issue #33). Rien ici n'est implemente ; +les quick wins non cassants identifies sont listes en fin de document avec +leur issue de rattachement. + +## Constat + +La fonction centrale `compare_datasets_from_yaml()` a grossi par accretion : +17 parametres a plat, un nom qui ne reflete plus l'usage (le YAML est +optionnel depuis 0.1.5), un retour a 8 champs partiellement redondants, et un +melange de langues (arguments anglais, champ `$reponse` en francais, rapport +par defaut en francais, messages en anglais). + +## Proposition + +### 1. Nom et alias + +```r +compare_datasets(reference, candidate, key = NULL, rules = NULL, ...) +``` + +- `compare_datasets()` devient le point d'entree documente. +- `compare_datasets_from_yaml()` reste exporte comme alias retrocompatible + (one-liner qui delegue), documente dans une section "Legacy". +- `reference`/`candidate` remplacent `data_reference`/`data_candidate` + (les anciens noms restent acceptes par l'alias). + +### 2. Objets d'options pour les passe-plats + +Cinq parametres sont du pur passe-plat vers `pointblank::interrogate()` et un +n'agit que sur le chemin Arrow. Regroupement : + +```r +# NB : lang/locale montres avec les defauts ACTUELS (0.4.x, fr/fr_FR) ; +# la section 5 propose de basculer le defaut 0.5 vers en/en_US +compare_datasets( + reference, candidate, key = NULL, rules = NULL, + extract = extract_opts(failed = TRUE, first_n = NULL, sample_n = NULL, + sample_frac = NULL, limit = 5000), + engine = engine_opts(duckdb_memory_limit = "8GB"), + lang = getOption("datadiff.lang", "fr"), + locale = getOption("datadiff.locale", "fr_FR") +) +``` + +- `extract_opts()` / `engine_opts()` sont des constructeurs valides + (erreurs franches, defauts documentes en un seul endroit). +- La signature centrale descend de 17 a ~8 parametres. + +### 3. Retour : classe `datadiff_result` + +Etat actuel : `all_passed` existe en 3 exemplaires (`$all_passed`, +`$summary$all_passed`, `pointblank::all_passed($reponse)`) ; `$agent` n'est +pas interroge (et il est factice sur le fast-path all-pass) sans usage +utilisateur identifie ; `$reponse` est du franglais. + +Cible : + +```r +res$passed # le verdict, une seule fois +res$coverage # inchange +res$summary # inchange (sans all_passed duplique) +res$report # l'agent interroge (ex-$reponse), print() paresseux inchange +res$applied_rules, res$missing_in_candidate, res$extra_in_candidate +``` + +- `$reponse` et `$all_passed` restent presents une version avec un warning de + depreciation a l'acces (active binding ou methode `$.datadiff_result`). +- `$agent` est retire du contrat documente (garde interne si necessaire). + +### 4. warn_at / stop_at + +Les deux valent 1e-14 : WARN et STOP se declenchent toujours ensemble, le +niveau WARN n'apporte rien. Proposition : un unique `fail_at = 1e-14` +(fraction), les deux niveaux pointblank cales dessus ; `warn_at`/`stop_at` +acceptes par l'alias legacy avec warning si differents. + +### 5. Langue + +- Deux ecoles : (a) tout anglais par defaut (`lang = "en"`), coherent avec + les messages ; (b) statu quo francais documente. La 0.4.4 avait annonce (a) + puis un commit l'a re-bascule sans NEWS (issue #26 : la doc est desormais + alignee sur le defaut reel "fr"). +- Proposition : basculer a `"en"` au moment du passage 0.5 (major-ish), via + `getOption("datadiff.lang", "en")`, avec entree NEWS Breaking changes. + +### 6. Messages + +Uniformiser sur le style des bons warnings existants (doublons de cles, +type_mismatch : contexte + consequence + action). En particulier remplacer +`message("key is missing")` par une note explicite unique documentant le mode +positionnel, ou la supprimer (le mode positionnel est un choix legitime). + +### 7. write_rules_template() + +- 19 parametres au nommage incoherent (`na_equal_default` vs `numeric_abs`) : + harmoniser en `0.5` (`na_equal`, `numeric_abs`, ...) via l'alias. +- Ne plus ecrire par defaut `rules.yaml` dans le repertoire courant : + `path` obligatoire ou defaut `tempfile()`. +- `ref_suffix` : detail d'implementation, a retirer de la signature publique. + +## Cycle de depreciation + +1. 0.5.0 : nouvelle surface + alias retrocompatibles complets, warnings de + depreciation conditionnels (une fois par session), vignette reecrite sur la + nouvelle surface, annexe migration. +2. 0.6.0 : warnings inconditionnels. +3. 1.0.0 : retrait des alias. + +## Quick wins deja traites ou rattaches a d'autres issues + +- Precedence argument > YAML (issue #20, fait en 0.4.10). +- Validation label / key / duckdb_memory_limit (issues #16/#20/#30, fait). +- Contradictions NEWS %||% / lang / "comparaison" (issue #26). +- Parametre mort cols_reference (issue #25). +- equal_mode inerte (issue #27). From 38be24f541751030be8a0cff960f541018b7247b Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:41:16 +0200 Subject: [PATCH 33/40] docs+fix: ecarts doc/code resolus, read_rules parle a l'editeur de YAML (issue #32) - @return complet (all_passed en tete, agent decrit fidelement : configure mais non interroge) - README : Quick Start zero-config d'abord, cle explicite partout ; la section Dev n'embarque plus de sorties check/covr perissables (figees a la 0.4.2 depuis mars) - vignette : restriction locale des diagnostics __absdiff/__thresh dans les extraits ; caveat NaN DuckDB-only du chemin lazy (releve au triage de la PR #41) - read_rules : erreur de version explicite (fichier + version declaree + version supportee, fini le stopifnot brut) ; warning sur les champs inconnus racine/defaults (typos by_nmae, no_equal) en nommant les champs connus - la friction principale d'un YAML edite a la main - write_rules_template refuse version != 1 en amont (il ecrivait des templates que read_rules refusait ensuite) --- NEWS.md | 20 ++++++ R/compare_datasets_from_yaml.R | 61 ++++++++++++++++- README.Rmd | 37 ++++------- README.md | 105 ++++++++++++------------------ man/compare_datasets_from_yaml.Rd | 6 +- tests/testthat/test-main.R | 65 ++++++++++++++++++ vignettes/datadiff.Rmd | 11 ++++ 7 files changed, 216 insertions(+), 89 deletions(-) diff --git a/NEWS.md b/NEWS.md index cbb9a2c..f2085af 100644 --- a/NEWS.md +++ b/NEWS.md @@ -86,6 +86,26 @@ projection (`paste0(character(0), "__ok")` yields `"__ok"`) that `any_of()` had been eating since 0.4.8 (issue #17). +## Documentation and input friction + +* Doc-vs-code drift resolved (issue #32): `@return` of + `compare_datasets_from_yaml()` now lists all 8 elements including + `all_passed` (the first one) and describes `agent` truthfully (configured, + NOT interrogated - `reponse` is); the README Quick Start shows the + zero-configuration mode first and passes `key` explicitly everywhere; the + README "Dev part" no longer embeds check/coverage outputs that go stale + (it froze results from 0.4.2); the vignette documents the local-only + restriction of the `__absdiff`/`__thresh` extract diagnostics and the + DuckDB-only NaN caveat of the lazy booleans. + +* `read_rules()` speaks to the person who edits the YAML by hand: an + unsupported `version` gets an explicit error naming the file, the declared + and the supported versions (was a raw `stopifnot` output), and unknown + top-level or `defaults` fields (typos like `by_nmae:` or `no_equal:`) get a + warning naming them and the known fields, instead of being silently + ignored. `write_rules_template()` rejects a `version` other than 1 upfront + (it happily wrote version 2 templates that `read_rules()` then refused). + ## Bug fixes * Three NEWS-vs-code contradictions introduced by the "Perf (#1)" commit after diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 6f2d58d..14727d6 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -50,6 +50,13 @@ write_rules_template <- function(data_reference, datetime_equal_mode = "exact", logical_equal_mode = "exact") { + if (!identical(suppressWarnings(as.numeric(version)), 1)) { + stop(sprintf( + "Parameter 'version' must be 1 (the only rules version read_rules() accepts), got %s.", + version + ), call. = FALSE) + } + # Validate key if provided # Use a 0-row collect to retrieve column names and types: works for both local # data.frames and lazy tables (tbl_lazy) without loading all rows. @@ -122,7 +129,53 @@ write_rules_template <- function(data_reference, #' @export read_rules <- function(path) { r <- read_yaml(path) - stopifnot(is.list(r), !is.null(r$version), r$version == 1) + if (!is.list(r)) { + stop(sprintf("'%s' does not contain a YAML mapping of rules.", path), + call. = FALSE) + } + version_ok <- is.atomic(r$version) && + length(r$version) == 1 && + identical(suppressWarnings(as.numeric(r$version)), 1) + if (!version_ok) { + shown <- if (is.atomic(r$version) && length(r$version) == 1) { + as.character(r$version) + } else if (is.null(r$version)) { + "" + } else { + # lists / vectors from YAML like [1, 2] or {v: 1}: printable summary + paste(class(r$version)[1], "of length", length(r$version)) + } + stop(sprintf( + "'%s' declares rules version %s; this version of {datadiff} supports version 1 only.", + path, shown + ), call. = FALSE) + } + # Hand-edited YAML again: a scalar where a mapping is expected + # (defaults: yes) would crash far away on a $ access; refuse it here + for (section in c("defaults", "by_type", "by_name", "row_validation")) { + if (!is.null(r[[section]]) && !is.list(r[[section]])) { + stop(sprintf( + "'%s': the '%s' section must be a YAML mapping (key: value lines), not a single value.", + path, section + ), call. = FALSE) + } + } + # A hand-edited YAML is the main input of this package: flag the fields the + # comparison will silently ignore (typos like by_nmae or no_equal) + known_top <- c("version", "defaults", "by_type", "by_name", "row_validation") + unknown_top <- setdiff(names(r), known_top) + known_defaults <- c("na_equal", "ignore_columns", "keys", "key", "label") + unknown_defaults <- setdiff(names(r$defaults %||% list()), known_defaults) + unknown <- c(unknown_top, unknown_defaults) + if (length(unknown) > 0) { + warning(sprintf( + "'%s' contains field(s) the comparison will ignore: %s. Check for typos (known top-level fields: %s; known defaults fields: %s).", + path, + paste(sprintf("'%s'", unknown), collapse = ", "), + paste(known_top, collapse = ", "), + paste(known_defaults, collapse = ", ") + ), call. = FALSE) + } r$defaults <- r$defaults %||% list() r$by_type <- r$by_type %||% list() r$by_name <- r$by_name %||% list() @@ -292,7 +345,11 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' effect when Arrow datasets are used: plain `data.frame`s or `tbl_lazy` #' inputs ignore a valid value. #' @return A list containing: -#' \item{agent}{Configured pointblank agent with validation results} +#' \item{all_passed}{Logical; \code{TRUE} when every check passed. The +#' first element of the returned list and the single verdict consumers +#' should read.} +#' \item{agent}{The configured pointblank agent as built (NOT interrogated: +#' the verdict lives in \code{reponse}, which is the interrogated one)} #' \item{reponse}{Interrogated pointblank agent (class \code{datadiff_report}): #' usable by \code{pointblank::all_passed()} / \code{get_data_extracts()}; #' printing it lazily renders a full pointblank-style report from diff --git a/README.Rmd b/README.Rmd index f0f9712..8a77056 100644 --- a/README.Rmd +++ b/README.Rmd @@ -46,16 +46,20 @@ candidate <- data.frame( category = c("a", "b", "c") # Different case ) -# Generate a rules template +# Zero configuration: near-exact rules generated from the reference structure +result <- compare_datasets_from_yaml(reference, candidate, key = "id") +result$all_passed # FALSE here: exact rules reject the differences above + +# To tune tolerances, generate a rules template and edit it +rules_path <- tempfile(fileext = ".yaml") write_rules_template(reference, key = "id", - path = "validation_rules.yaml", + path = rules_path, numeric_abs = 0.01, character_case_insensitive = TRUE) -# Edit the rules file to configure validation (e.g., set tolerances, case sensitivity) - -# Compare datasets -result <- compare_datasets_from_yaml(reference, candidate, path = "validation_rules.yaml") +# Compare datasets with the tuned rules (key stays explicit) +result <- compare_datasets_from_yaml(reference, candidate, + key = "id", path = rules_path) result$all_passed # overall verdict (TRUE/FALSE) result$coverage # one row per check performed: column, type, n, n_failed, status @@ -401,24 +405,11 @@ MIT License ## Dev part -This `README` has been compiled on the - -```{r} -Sys.time() -``` - -Here are the test & coverage results: +Run the checks and the coverage locally (embedded outputs went stale as the +package evolved, so this README no longer freezes them at compile time): -```{r error = TRUE} -devtools::check(quiet = TRUE) -``` - -```{r echo = FALSE} -unloadNamespace("datadiff") -``` - -```{r error = TRUE} -Sys.setenv("NOT_CRAN" = TRUE) +```r +devtools::check() covr::package_coverage() ``` diff --git a/README.md b/README.md index 3c3cee6..814bffd 100644 --- a/README.md +++ b/README.md @@ -37,16 +37,20 @@ candidate <- data.frame( category = c("a", "b", "c") # Different case ) -# Generate a rules template +# Zero configuration: near-exact rules generated from the reference structure +result <- compare_datasets_from_yaml(reference, candidate, key = "id") +result$all_passed # FALSE here: exact rules reject the differences above + +# To tune tolerances, generate a rules template and edit it +rules_path <- tempfile(fileext = ".yaml") write_rules_template(reference, key = "id", - path = "validation_rules.yaml", + path = rules_path, numeric_abs = 0.01, character_case_insensitive = TRUE) -# Edit the rules file to configure validation (e.g., set tolerances, case sensitivity) - -# Compare datasets -result <- compare_datasets_from_yaml(reference, candidate, path = "validation_rules.yaml") +# Compare datasets with the tuned rules (key stays explicit) +result <- compare_datasets_from_yaml(reference, candidate, + key = "id", path = rules_path) result$all_passed # overall verdict (TRUE/FALSE) result$coverage # one row per check performed: column, type, n, n_failed, status @@ -142,13 +146,13 @@ by_name: `by_name` taking precedence. A column not listed in `by_name` uses its `by_type` defaults unchanged. -| Column | Effective rule | Source | -|----|----|----| -| `id` | `abs: 0` | `by_type.integer` | -| `amount` | `abs: 0.01, rel: 0` | `by_name.amount` overrides `by_type.numeric` | -| `unit_price` | `abs: 0.001, rel: 0.0001` | `by_name.unit_price` overrides `by_type.numeric` | -| `category` | `case_insensitive: yes, trim: yes` | `by_name.category` overrides `by_type.character` | -| `created_at` | `equal_mode: exact` | `by_type.date` (by_name is redundant here) | +| Column | Effective rule | Source | +|--------------|------------------------------------|--------------------------------------------------| +| `id` | `abs: 0` | `by_type.integer` | +| `amount` | `abs: 0.01, rel: 0` | `by_name.amount` overrides `by_type.numeric` | +| `unit_price` | `abs: 0.001, rel: 0.0001` | `by_name.unit_price` overrides `by_type.numeric` | +| `category` | `case_insensitive: yes, trim: yes` | `by_name.category` overrides `by_type.character` | +| `created_at` | `equal_mode: exact` | `by_type.date` (by_name is redundant here) | ### Numeric tolerance: formula, edge effects, and best practices @@ -162,10 +166,10 @@ threshold**: The two parameters **add up**: they are not two independent guards. -| Parameter | Role | Default value | -|----|----|----| -| `abs` | Absolute tolerance: fixed floor, independent of the magnitude of values | `1e-9` | -| `rel` | Relative tolerance: fraction of the reference value added to the threshold | `0` | +| Parameter | Role | Default value | +|-----------|----------------------------------------------------------------------------|---------------| +| `abs` | Absolute tolerance: fixed floor, independent of the magnitude of values | `1e-9` | +| `rel` | Relative tolerance: fraction of the reference value added to the threshold | `0` | #### Pure absolute mode (recommended by default) @@ -201,11 +205,11 @@ by_type: rel: 0.01 # tolerance of 1% of the reference value ``` -| Reference | Candidate | Difference | Threshold (1%) | Result | -|----|---:|---:|---:|----| -| 100.00 | 100.50 | 0.50 | 1.00 | OK | -| 1 000 000.00 | 1 005 000.00 | 5 000 | 10 000 | OK | -| 0.00 | 0.001 | 0.001 | **0** | **ERROR** (implicit division by zero) | +| Reference | Candidate | Difference | Threshold (1%) | Result | +|--------------|-------------:|-----------:|---------------:|---------------------------------------| +| 100.00 | 100.50 | 0.50 | 1.00 | OK | +| 1 000 000.00 | 1 005 000.00 | 5 000 | 10 000 | OK | +| 0.00 | 0.001 | 0.001 | **0** | **ERROR** (implicit division by zero) | > **Warning**: if the reference value is `0`, the relative threshold is > `0`, so any difference, even tiny, will be detected as an error. @@ -286,7 +290,7 @@ row_validation: This validates that the candidate dataset has between 950 and 1050 rows (1000 +/- 50). -If `expected_count` is not specified, the reference dataset's row count +If `expected_count` is not specified, the reference dataset’s row count is used as the expected value. ## Comparing Parquet files (large datasets) @@ -294,7 +298,7 @@ is used as the expected value. `datadiff` can compare Parquet files that are too large to fit in RAM. The recommended approach uses `arrow::open_dataset()` - **do not call `arrow::to_duckdb()` yourself** before passing to `datadiff`; the -package handles the Arrow -> DuckDB conversion internally with a single +package handles the Arrow -\> DuckDB conversion internally with a single connection. ### Recommended strategy: Arrow Dataset (lazy, out-of-core) @@ -322,12 +326,12 @@ Internally, `compare_datasets_from_yaml()`: 1. Opens a private DuckDB connection (`fresh_con`). 2. Materialises each Parquet dataset as a DuckDB physical temp table - via `read_parquet()` - all memory is managed by DuckDB's buffer + via `read_parquet()` - all memory is managed by DuckDB’s buffer pool, so disk spilling works correctly. 3. Runs the full join + 125 boolean expressions as a single lazy SQL query. 4. `dplyr::compute()` materialises only the slim boolean result table - (~125 logical columns * N rows) - the wide source data is never + (~125 logical columns \* N rows) - the wide source data is never loaded into R. 5. `dplyr::collect()` brings the slim boolean table (~few GB) into R and passes a plain `data.frame` to pointblank - no live DuckDB @@ -336,7 +340,7 @@ Internally, `compare_datasets_from_yaml()`: ### Memory tuning: `duckdb_memory_limit` -DuckDB's default memory cap (80 % of total RAM) can leave insufficient +DuckDB’s default memory cap (80 % of total RAM) can leave insufficient headroom when R, Arrow, and the OS are already using significant memory. The `duckdb_memory_limit` parameter controls how much RAM DuckDB may use before spilling intermediate results to `tempdir()`: @@ -365,12 +369,12 @@ limit only applies when Arrow datasets are used - it has no effect for ### Strategy comparison -| Strategy | Input type | RAM usage | Requires | -|----|----|----|----| -| **Arrow Dataset** ✅ recommended | `arrow::open_dataset()` | Slim boolean table only (~few GB) | arrow, duckdb | -| Arrow Table | `arrow::read_parquet(as_data_frame=FALSE)` | Same as Arrow Dataset | arrow, duckdb | -| Lazy table (dbplyr) | `tbl(con, "table_name")` | Slim boolean table only | DBI, dbplyr | -| `data.frame` | `read.csv()`, `readr::read_csv()`, etc. | Full data in RAM | - | +| Strategy | Input type | RAM usage | Requires | +|----------------------------------|--------------------------------------------|-----------------------------------|---------------| +| **Arrow Dataset** ✅ recommended | `arrow::open_dataset()` | Slim boolean table only (~few GB) | arrow, duckdb | +| Arrow Table | `arrow::read_parquet(as_data_frame=FALSE)` | Same as Arrow Dataset | arrow, duckdb | +| Lazy table (dbplyr) | `tbl(con, "table_name")` | Slim boolean table only | DBI, dbplyr | +| `data.frame` | `read.csv()`, `readr::read_csv()`, etc. | Full data in RAM | \- | ### What NOT to do @@ -389,7 +393,7 @@ ref_df <- dplyr::collect(ds_ref) # loads full 4 GB into R RAM DuckDB spills to `tempdir()` when the memory limit is reached. On Windows this is typically `C:\Users\\AppData\Local\Temp`. Ensure -that directory has sufficient free disk space (up to ~2-3* the size of +that directory has sufficient free disk space (up to ~2-3\* the size of your Parquet files in the worst case). ## Main Functions @@ -421,36 +425,11 @@ MIT License ## Dev part -This `README` has been compiled on the - -``` r -Sys.time() -#> [1] "2026-03-11 18:07:04 CET" -``` - -Here are the test & coverage results: - -``` r -devtools::check(quiet = TRUE) -#> ℹ Loading datadiff -#> ── R CMD check results ───────────────────────────────────── datadiff 0.4.2 ──── -#> Duration: 4m 47.4s -#> -#> ❯ checking for future file timestamps ... NOTE -#> unable to verify current time -#> -#> 0 errors ✔ | 0 warnings ✔ | 1 note ✖ -``` +Run the checks and the coverage locally (embedded outputs went stale as +the package evolved, so this README no longer freezes them at compile +time): ``` r -Sys.setenv("NOT_CRAN" = TRUE) +devtools::check() covr::package_coverage() -#> datadiff Coverage: 98.98% -#> R/preprocessing.R: 97.22% -#> R/compare_datasets_from_yaml.R: 98.58% -#> R/data_types.R: 100.00% -#> R/pointblank_setup.R: 100.00% -#> R/tolerance.R: 100.00% -#> R/utils.R: 100.00% -#> R/validation.R: 100.00% ``` diff --git a/man/compare_datasets_from_yaml.Rd b/man/compare_datasets_from_yaml.Rd index 91bf198..e550b56 100644 --- a/man/compare_datasets_from_yaml.Rd +++ b/man/compare_datasets_from_yaml.Rd @@ -89,7 +89,11 @@ inputs ignore a valid value.} } \value{ A list containing: -\item{agent}{Configured pointblank agent with validation results} +\item{all_passed}{Logical; \code{TRUE} when every check passed. The +first element of the returned list and the single verdict consumers +should read.} +\item{agent}{The configured pointblank agent as built (NOT interrogated: +the verdict lives in \code{reponse}, which is the interrogated one)} \item{reponse}{Interrogated pointblank agent (class \code{datadiff_report}): usable by \code{pointblank::all_passed()} / \code{get_data_extracts()}; printing it lazily renders a full pointblank-style report from diff --git a/tests/testthat/test-main.R b/tests/testthat/test-main.R index e505206..2471942 100644 --- a/tests/testthat/test-main.R +++ b/tests/testthat/test-main.R @@ -60,3 +60,68 @@ test_that("read_rules validates YAML structure", { expect_error(object = read_rules(invalid_path)) }) + +test_that("read_rules gives an actionable error for unsupported versions", { + p <- tempfile(fileext = ".yaml") + on.exit(unlink(p), add = TRUE) + writeLines('version: 2\ndefaults: {na_equal: yes}', con = p) + err <- tryCatch(read_rules(p), error = function(e) { + conditionMessage(e) + }) + expect_match(err, "version", ignore.case = TRUE) + expect_match(err, "2", fixed = TRUE) # the offending version + expect_match(err, "1", fixed = TRUE) # the supported version + expect_no_match(err, "is not TRUE", fixed = TRUE) # no raw stopifnot output +}) + +test_that("read_rules warns on unknown top-level and defaults fields", { + p <- tempfile(fileext = ".yaml") + on.exit(unlink(p), add = TRUE) + writeLines(' +version: 1 +defaults: {keys: [id], na_equal: yes, no_equal: yes} +by_nmae: + numeric: {abs: 0.1} +', con = p) + warns <- character(0) + withCallingHandlers( + rules <- read_rules(p), + warning = function(w) { + warns <<- c(warns, conditionMessage(w)) + invokeRestart("muffleWarning") + } + ) + expect_true(any(grepl("by_nmae", warns))) # typo of by_name flagged + expect_true(any(grepl("no_equal", warns))) # typo in defaults flagged +}) + +test_that("write_rules_template rejects an unsupported version upfront", { + df <- data.frame(id = 1:2, x = c(1, 2)) + expect_error( + write_rules_template(df, key = "id", path = tempfile(fileext = ".yaml"), + version = 2), + regexp = "version" + ) +}) + +test_that("read_rules stays actionable on non-scalar versions", { + p <- tempfile(fileext = ".yaml") + on.exit(unlink(p), add = TRUE) + writeLines('version: {v: 1}\ndefaults: {na_equal: yes}', con = p) + err <- tryCatch(read_rules(p), error = function(e) { + conditionMessage(e) + }) + expect_match(err, "version", ignore.case = TRUE) + expect_no_match(err, "invalid format", fixed = TRUE) # no raw sprintf crash +}) + +test_that("read_rules rejects non-mapping sections explicitly", { + p <- tempfile(fileext = ".yaml") + on.exit(unlink(p), add = TRUE) + writeLines('version: 1\ndefaults: yes', con = p) + err <- tryCatch(read_rules(p), error = function(e) { + conditionMessage(e) + }) + expect_match(err, "defaults") + expect_match(err, "mapping", ignore.case = TRUE) +}) diff --git a/vignettes/datadiff.Rmd b/vignettes/datadiff.Rmd index 6003f36..d30a3f8 100644 --- a/vignettes/datadiff.Rmd +++ b/vignettes/datadiff.Rmd @@ -182,6 +182,14 @@ extracts <- pointblank::get_data_extracts(result_fail$reponse) extracts[[1]] ``` +### Measured deviations in the extracts + +On the in-memory (data.frame) path, the extracts of a failing tolerance +column also carry the measured deviation and the applied threshold +(`__absdiff`, `__thresh`). The lazy path never had these +diagnostics: its agent works on a slim boolean table and its extracts only +show the boolean columns. + ### Controlling how many failing rows are extracted For large datasets, extracting all failing rows can consume significant memory. @@ -912,6 +920,9 @@ DBI::dbDisconnect(con) ``` This works with any DBI-compatible backend: SQLite, PostgreSQL, Snowflake, etc. +Caveat: the NaN-aware semantics of the lazy booleans are wired for DuckDB +(`isnan()`); a non-DuckDB backend that stores NaN keeps NULL-only rules for +those values (see the `add_bool_cols_sql()` notes). ### Arrow / Parquet (out-of-core) From b9283479bae785ec1e716f452c9e1893f79a43af Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:48:43 +0200 Subject: [PATCH 34/40] chore: menage packaging et scories (issue #34) - .Rbuildignore ^dev$ (le RFC et les benchs ne partent plus dans le tarball) ; inst/templates/rules_template.yaml orphelin supprime - imports : dplyr arrange/across retires, stats::setNames remplace par une construction base ; numeric_abs = 1e-9 lisible - is_non_local reutilise is_arrow (liste de classes Arrow definie une fois) ; abs(ref_vals) et la constante IEEE hisses hors des kernels et boucles ; tol_col_counts compte sans allocations intermediaires ; format_key_examples tronque avant de formater - preprocess_dataframe lazy : UNE seule mutate() composee pour toutes les colonnes normalisees (anti-pattern O(colonnes) dbplyr) - variables de boucle c renommees (masquaient base::c) dans pointblank_setup, coverage, tolerance, compare - note d'overflow integer documentee sur le kernel - ecartes explicitement (wontfix motives dans NEWS) : rendu gt en session non interactive (print = rendu), garde DBI transitive via duckdb --- DESCRIPTION | 1 - NAMESPACE | 3 - NEWS.md | 20 ++++++ R/compare_datasets_from_yaml.R | 13 ++-- R/coverage.R | 30 +++++---- R/duplicate_keys.R | 11 +-- R/pointblank_setup.R | 38 +++++------ R/preprocessing.R | 19 ++++-- R/tolerance.R | 104 ++++++++++++++++------------- R/utils.R | 10 +-- inst/templates/rules_template.yaml | 37 ---------- 11 files changed, 142 insertions(+), 144 deletions(-) delete mode 100644 inst/templates/rules_template.yaml diff --git a/DESCRIPTION b/DESCRIPTION index f8fa1f1..006a831 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,7 +18,6 @@ Imports: dplyr (>= 1.0.0), pointblank (>= 0.12.3), rlang (>= 0.4.0), - stats, tidyselect (>= 1.0.0), utils, yaml (>= 2.2.0) diff --git a/NAMESPACE b/NAMESPACE index 9c201c6..1d82dd0 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -15,8 +15,6 @@ export(setup_pointblank_agent) export(validate_row_counts) export(write_rules_template) importFrom(dplyr,"%>%") -importFrom(dplyr,across) -importFrom(dplyr,arrange) importFrom(dplyr,collect) importFrom(dplyr,left_join) importFrom(pointblank,action_levels) @@ -26,7 +24,6 @@ importFrom(pointblank,create_agent) importFrom(pointblank,interrogate) importFrom(rlang,":=") importFrom(rlang,.data) -importFrom(stats,setNames) importFrom(tidyselect,all_of) importFrom(utils,modifyList) importFrom(yaml,read_yaml) diff --git a/NEWS.md b/NEWS.md index f2085af..7fa53e5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -86,6 +86,26 @@ projection (`paste0(character(0), "__ok")` yields `"__ok"`) that `any_of()` had been eating since 0.4.8 (issue #17). +## Chores + +* Packaging and code cleanup (issue #34): `dev/` is excluded from the tarball + (`.Rbuildignore`); the orphan `inst/templates/rules_template.yaml` + (referenced nowhere) is gone; unused `@importFrom` entries dropped + (`dplyr::arrange`/`across`, `stats::setNames` replaced by a base + construction); the `numeric_abs` default reads `1e-9` instead of a + 9-zero decimal literal; `is_non_local()` reuses `is_arrow()` instead of + duplicating the Arrow class list; `abs(ref_vals)` and the IEEE fp constant + are hoisted out of the kernels' expressions and loops; + `tol_col_counts()` counts failures without allocating intermediate + vectors; `format_key_examples()` truncates to 3 groups before formatting; + the lazy `preprocess_dataframe()` composes ONE `mutate()` for all + normalized columns instead of one or two layers per column (the dbplyr + query-construction anti-pattern eliminated elsewhere in 0.4.8); loop + variables no longer shadow `base::c`; the tolerance kernel documents the + integer-overflow caveat. Deliberately not changed: `print()` of a report + builds the gt object in non-interactive sessions too (printing IS the + render), and the `DBI` availability guard stays transitive via `duckdb`. + ## Documentation and input friction * Doc-vs-code drift resolved (issue #32): `@return` of diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 14727d6..0954a26 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -33,7 +33,6 @@ #' (same caveat as `date_equal_mode`) #' @return The \code{path} to the written YAML file, returned invisibly. #' @importFrom yaml write_yaml -#' @importFrom stats setNames #' @importFrom dplyr collect #' @export #' @examples @@ -43,7 +42,7 @@ write_rules_template <- function(data_reference, key = NULL, label = NULL, path = "rules.yaml", version = 1L, na_equal_default = TRUE, ignore_columns_default = character(0), check_count_default = TRUE, expected_count_default = NULL, row_count_tolerance_default = 0, - numeric_abs = 0.000000001, numeric_rel = 0, + numeric_abs = 1e-9, numeric_rel = 0, integer_abs = 0L, character_equal_mode = "exact", character_case_insensitive = FALSE, character_trim = FALSE, date_equal_mode = "exact", @@ -108,7 +107,7 @@ write_rules_template <- function(data_reference, datetime = list(equal_mode = datetime_equal_mode), logical = list(equal_mode = logical_equal_mode) ), - by_name = setNames(replicate(length(types), list(), simplify = FALSE), names(types)) + by_name = as.list(structure(replicate(length(types), list(), simplify = FALSE), names = names(types))) ) write_yaml(x = y, file = path) invisible(path) @@ -363,7 +362,7 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' the fast path skips the per-column agent.} #' \item{summary}{Aggregate counts from \code{coverage} (n_checks, n_pass, #' n_fail, n_rows_failed_total, all_passed).} -#' @importFrom dplyr arrange across left_join %>% +#' @importFrom dplyr left_join %>% #' @importFrom pointblank interrogate #' @importFrom dplyr collect #' @export @@ -874,9 +873,9 @@ compare_datasets_from_yaml <- function(data_reference, # already carries __eq columns; col_vals_equal(na_pass = ...) alone cannot # express these semantics. if (!is_lazy && length(fail$eq) > 0) { - for (c in fail$eq) { - cmp_for_agent[[datadiff_eq_col(c)]] <- eq_col_bool( - cmp_for_agent, col = c, ref_suffix = ref_suffix, na_equal = na_equal + for (col_nm in fail$eq) { + cmp_for_agent[[datadiff_eq_col(col_nm)]] <- eq_col_bool( + cmp_for_agent, col = col_nm, ref_suffix = ref_suffix, na_equal = na_equal ) } } diff --git a/R/coverage.R b/R/coverage.R index 0807d09..5bfbfd9 100644 --- a/R/coverage.R +++ b/R/coverage.R @@ -9,7 +9,9 @@ # matching col_vals_equal(..., na_pass = FALSE). tol_col_counts <- function(tbl, col) { ok <- tol_col_bool(tbl, col = col) - list(n = length(ok), n_failed = sum(is.na(ok) | !ok)) + # length - sum(TRUE) counts FALSE and NA in one pass without allocating + # the two intermediate logical vectors of is.na(ok) | !ok + list(n = length(ok), n_failed = length(ok) - sum(ok, na.rm = TRUE)) } # n / n_failed for an equality column (resolved boolean, no NA). @@ -104,23 +106,23 @@ build_coverage <- function(tbl, tol_cols, eq_cols, # Existence checks: every common (non-type-mismatch) column gets a col_exists # check, distinct from its value check. These pass (the column is present in # both datasets by construction), mirroring a full per-column pointblank run. - for (c in c(tol_cols, eq_cols)) { - add(c, "col_exists", 1L, 0L) + for (col_nm in c(tol_cols, eq_cols)) { + add(col_nm, "col_exists", 1L, 0L) } - for (c in tol_cols) { - cnt <- counts[[c]] %||% tol_col_counts(tbl, col = c) - add(c, "tolerance", cnt$n, cnt$n_failed) + for (col_nm in tol_cols) { + cnt <- counts[[col_nm]] %||% tol_col_counts(tbl, col = col_nm) + add(col_nm, "tolerance", cnt$n, cnt$n_failed) } - for (c in eq_cols) { - cnt <- counts[[c]] %||% - eq_col_counts(tbl, col = c, ref_suffix = ref_suffix, na_equal = na_equal) - add(c, "equality", cnt$n, cnt$n_failed) + for (col_nm in eq_cols) { + cnt <- counts[[col_nm]] %||% + eq_col_counts(tbl, col = col_nm, ref_suffix = ref_suffix, na_equal = na_equal) + add(col_nm, "equality", cnt$n, cnt$n_failed) } - for (c in missing_in_candidate) { - add(c, "missing_column", 1L, 1L) + for (col_nm in missing_in_candidate) { + add(col_nm, "missing_column", 1L, 1L) } - for (c in type_mismatch_cols) { - add(c, "type_mismatch", 1L, 1L) + for (col_nm in type_mismatch_cols) { + add(col_nm, "type_mismatch", 1L, 1L) } if (isTRUE(row_validation_info$check_count)) { add("", "row_count", 1L, if (isTRUE(row_count_ok)) 0L else 1L) diff --git a/R/duplicate_keys.R b/R/duplicate_keys.R index 02f9708..7068315 100644 --- a/R/duplicate_keys.R +++ b/R/duplicate_keys.R @@ -10,14 +10,15 @@ # tables keep the SQL-native count()/group_by (cheap inside the database). format_key_examples <- function(uniq_keys, key) { - rows <- unname(apply(uniq_keys, 1, function(r) { + n_groups <- nrow(uniq_keys) + shown <- utils::head(uniq_keys, 3L) + rows <- unname(apply(shown, 1, function(r) { paste(key, "=", r, collapse = ", ") })) - if (length(rows) <= 3) { - rows - } else { - c(rows[1:3], "...") + if (n_groups > 3L) { + rows <- c(rows, "...") } + rows } #' @importFrom rlang .data diff --git a/R/pointblank_setup.R b/R/pointblank_setup.R index 2173113..7378450 100644 --- a/R/pointblank_setup.R +++ b/R/pointblank_setup.R @@ -71,8 +71,8 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ # Add dummy columns for missing columns BEFORE creating the agent # These columns are set to FALSE and we'll check they equal TRUE (will fail) - for (c in missing_in_candidate) { - dummy_col <- paste0(datadiff_prefix_missing_col, c) + for (col_nm in missing_in_candidate) { + dummy_col <- paste0(datadiff_prefix_missing_col, col_nm) if (is_non_local(cmp)) { cmp <- dplyr::mutate(cmp, !!dummy_col := FALSE) } else { @@ -82,8 +82,8 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ # Add dummy FALSE columns for type-mismatched columns. # These will generate a dedicated failing validation step per column. - for (c in type_mismatch_cols) { - dummy_col <- paste0(datadiff_prefix_type_mismatch, c) + for (col_nm in type_mismatch_cols) { + dummy_col <- paste0(datadiff_prefix_type_mismatch, col_nm) if (is_non_local(cmp)) { cmp <- dplyr::mutate(cmp, !!dummy_col := FALSE) } else { @@ -98,11 +98,11 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ # express the one-sided/two-sided NA distinction). eq_cols <- setdiff(x = common_cols, y = tol_cols) if (!is_non_local(cmp)) { - for (c in eq_cols) { - eq_col <- datadiff_eq_col(c) + for (col_nm in eq_cols) { + eq_col <- datadiff_eq_col(col_nm) if (!(eq_col %in% get_col_names(cmp))) { cmp[[eq_col]] <- eq_col_bool( - cmp, col = c, ref_suffix = ref_suffix, na_equal = na_equal + cmp, col = col_nm, ref_suffix = ref_suffix, na_equal = na_equal ) } } @@ -120,37 +120,37 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ # Skipped for the non-local path where cmp is a slim table containing only # boolean validation columns (the original data columns are not present). if (add_col_exists_steps) { - for (c in common_cols) { - agent <- agent %>% col_exists(columns = all_of(c)) + for (col_nm in common_cols) { + agent <- agent %>% col_exists(columns = all_of(col_nm)) } } # For missing columns, add a validation that will fail - for (c in missing_in_candidate) { - dummy_col <- paste0(datadiff_prefix_missing_col, c) + for (col_nm in missing_in_candidate) { + dummy_col <- paste0(datadiff_prefix_missing_col, col_nm) agent <- agent %>% col_vals_equal( columns = all_of(dummy_col), value = TRUE, na_pass = FALSE, - label = paste("col_exists:", c) + label = paste("col_exists:", col_nm) ) } # For type-mismatched columns, add a validation that will always fail. - for (c in type_mismatch_cols) { - dummy_col <- paste0(datadiff_prefix_type_mismatch, c) + for (col_nm in type_mismatch_cols) { + dummy_col <- paste0(datadiff_prefix_type_mismatch, col_nm) agent <- agent %>% col_vals_equal( columns = all_of(dummy_col), value = TRUE, na_pass = FALSE, - label = paste("type_mismatch:", c) + label = paste("type_mismatch:", col_nm) ) } - for (c in eq_cols) { - eq_col <- datadiff_eq_col(c) + for (col_nm in eq_cols) { + eq_col <- datadiff_eq_col(col_nm) if (!(eq_col %in% col_names)) { # Only reachable with a lazy cmp missing its precomputed boolean: # the documented contract requires it (deriving it here would need @@ -164,8 +164,8 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ col_vals_equal(columns = all_of(eq_col), value = TRUE, na_pass = FALSE) } - for (c in tol_cols) { - ok_col <- datadiff_ok_col(c) + for (col_nm in tol_cols) { + ok_col <- datadiff_ok_col(col_nm) agent <- agent %>% col_vals_equal(columns = all_of(ok_col), value = TRUE, na_pass = FALSE) } diff --git a/R/preprocessing.R b/R/preprocessing.R index 10e7c54..6a0dbfb 100644 --- a/R/preprocessing.R +++ b/R/preprocessing.R @@ -19,7 +19,7 @@ normalize_text <- function(x, case_insensitive = FALSE, trim = FALSE) { x <- trimws(x) } if (case_insensitive) { - x <- tolower(x = x) + x <- tolower(x) } x } @@ -63,6 +63,7 @@ preprocess_dataframe <- function(df, col_rules, schema = NULL) { } } } + lazy_exprs <- list() for (nm in names(col_rules)) { cr <- col_rules[[nm]] normalized <- identical(cr$equal_mode %||% "exact", "normalized") @@ -93,23 +94,26 @@ preprocess_dataframe <- function(df, col_rules, schema = NULL) { if (is_char && should_normalize) { if (is_non_local(out)) { - # Non-local path: build transformations via dplyr::mutate() - nm_sym <- dplyr::sym(nm) + # Non-local path: compose ONE expression per column and accumulate; + # a single mutate() at the end keeps the dbplyr query-construction + # cost O(1) instead of one or two mutate() layers per column + expr <- dplyr::sym(nm) if (trim) { if (is_arrow(out)) { # Arrow does not support trimws(); use stringr::str_trim() instead if (!requireNamespace("stringr", quietly = TRUE)) { stop("Package 'stringr' is required for trim on Arrow objects.") } - out <- dplyr::mutate(out, !!nm := stringr::str_trim(!!nm_sym)) + expr <- rlang::expr(stringr::str_trim(!!expr)) } else { # SQL path: trimws() -> SQL TRIM() (translated by dbplyr) - out <- dplyr::mutate(out, !!nm := trimws(!!nm_sym)) + expr <- rlang::expr(trimws(!!expr)) } } if (case_insensitive) { - out <- dplyr::mutate(out, !!nm := tolower(!!nm_sym)) + expr <- rlang::expr(tolower(!!expr)) } + lazy_exprs[[nm]] <- expr } else { out[[nm]] <- normalize_text( out[[nm]], @@ -119,5 +123,8 @@ preprocess_dataframe <- function(df, col_rules, schema = NULL) { } } } + if (length(lazy_exprs) > 0) { + out <- dplyr::mutate(out, !!!lazy_exprs) + } out } diff --git a/R/tolerance.R b/R/tolerance.R index d3ab891..edfba55 100644 --- a/R/tolerance.R +++ b/R/tolerance.R @@ -21,6 +21,10 @@ #' @param na_equal Logical; whether NA/NaN on both sides count as equal. #' @return A list with numeric vectors \code{absdiff}, \code{thresh} and the #' logical vector \code{ok}. +#' @note When both compared columns are integer, \code{cand - ref} is +#' integer subtraction: a difference beyond \code{.Machine$integer.max} +#' overflows to NA with a warning (and NA counts as a failure). Cast such +#' columns to double upstream if their differences can approach that range. #' @noRd compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal) { # is.na() covers NaN in R, so the NA masks subsume the NaN ones: NaN-specific @@ -35,9 +39,10 @@ compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equa absdiff <- abs(cand_vals - ref_vals) absdiff[same_inf] <- 0 - thresh <- abs_tol + rel_tol * abs(ref_vals) + abs_ref <- abs(ref_vals) + thresh <- abs_tol + rel_tol * abs_ref - fp_correction <- 8 * .Machine$double.eps * abs(ref_vals) + fp_correction <- 8 * .Machine$double.eps * abs_ref fp_correction[!is.finite(fp_correction)] <- 0 within_tol <- absdiff <= thresh + fp_correction @@ -72,8 +77,9 @@ compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equa compute_tolerance_ok <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal) { if (!anyNA(cand_vals) && !anyNA(ref_vals) && all(is.finite(cand_vals)) && all(is.finite(ref_vals))) { - thresh <- abs_tol + rel_tol * abs(ref_vals) - fp <- 8 * .Machine$double.eps * abs(ref_vals) + abs_ref <- abs(ref_vals) + thresh <- abs_tol + rel_tol * abs_ref + fp <- 8 * .Machine$double.eps * abs_ref return(abs(cand_vals - ref_vals) <= thresh + fp) } if (!any(is.infinite(cand_vals)) && !any(is.infinite(ref_vals))) { @@ -81,8 +87,9 @@ compute_tolerance_ok <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal # NaN in R, so NaN follows the NA rules identically here and in the # kernel. Only Inf diverges (abs(Inf - Inf) is NaN, but same-sign # infinities must PASS), hence the infinity-only fallback test. - thresh <- abs_tol + rel_tol * abs(ref_vals) - fp <- 8 * .Machine$double.eps * abs(ref_vals) + abs_ref <- abs(ref_vals) + thresh <- abs_tol + rel_tol * abs_ref + fp <- 8 * .Machine$double.eps * abs_ref ok <- abs(cand_vals - ref_vals) <= thresh + fp # NA wherever an NA/NaN was involved: a one-sided one is a difference... ok[is.na(ok)] <- FALSE @@ -97,12 +104,12 @@ compute_tolerance_ok <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal #' Add only the `__ok` tolerance columns (hot path) #' -#' Like [add_tolerance_columns()] but materialises only the boolean `__ok` +#' Like `add_tolerance_columns()` but materialises only the boolean `__ok` #' columns - which alone determine the verdict - in a single bind. The #' `__absdiff` / `__thresh` diagnostic columns are not produced here: #' no verdict logic reads them, and on the all-pass fast path there are no #' extracts to surface them in. On the failure path they are re-added for the -#' failing columns only by [add_diff_columns()], so the failing-row extracts +#' failing columns only by `add_diff_columns()`, so the failing-row extracts #' keep the explicit measured deviations at a cost proportional to the failing #' columns rather than the table width. #' @@ -115,11 +122,11 @@ add_ok_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { } ok_cols <- vector("list", length(tol_cols)) for (i in seq_along(tol_cols)) { - c <- tol_cols[i] + col_nm <- tol_cols[i] ok_cols[[i]] <- compute_tolerance_ok( - cand_vals = cmp[[c]], ref_vals = cmp[[paste0(c, ref_suffix)]], - abs_tol = col_rules[[c]][["abs"]] %||% 0, - rel_tol = col_rules[[c]][["rel"]] %||% 0, + cand_vals = cmp[[col_nm]], ref_vals = cmp[[paste0(col_nm, ref_suffix)]], + abs_tol = col_rules[[col_nm]][["abs"]] %||% 0, + rel_tol = col_rules[[col_nm]][["rel"]] %||% 0, na_equal = na_equal ) } @@ -134,7 +141,7 @@ add_ok_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { #' failing-row extracts - `pointblank::get_data_extracts()`, the HTML report and #' its CSV download - show the explicit gap next to the candidate and reference #' values, as they did up to 0.4.7. The `__ok` verdict columns are expected -#' to exist already (see [add_ok_columns()]) and are not touched. Cost is +#' to exist already (see `add_ok_columns()`) and are not touched. Cost is #' proportional to the number of columns passed, so the all-pass fast path and #' the passing majority of columns pay nothing. #' @@ -149,11 +156,11 @@ add_diff_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { absdiff_cols <- vector("list", m) thresh_cols <- vector("list", m) for (i in seq_len(m)) { - c <- tol_cols[i] + col_nm <- tol_cols[i] blocks <- compute_tolerance_col( - cand_vals = cmp[[c]], ref_vals = cmp[[paste0(c, ref_suffix)]], - abs_tol = col_rules[[c]][["abs"]] %||% 0, - rel_tol = col_rules[[c]][["rel"]] %||% 0, + cand_vals = cmp[[col_nm]], ref_vals = cmp[[paste0(col_nm, ref_suffix)]], + abs_tol = col_rules[[col_nm]][["abs"]] %||% 0, + rel_tol = col_rules[[col_nm]][["rel"]] %||% 0, na_equal = na_equal ) absdiff_cols[[i]] <- blocks$absdiff @@ -261,22 +268,22 @@ add_bool_cols_sql <- function(cmp, tol_cols, eq_cols, col_rules, ref_suffix, ) } - tol_exprs <- vapply(X = tol_cols, FUN = function(c) { - cc <- q(c) - rc <- q(paste0(c, ref_suffix)) - at <- col_rules[[c]][["abs"]] %||% 0 - rt <- col_rules[[c]][["rel"]] %||% 0 + tol_exprs <- vapply(X = tol_cols, FUN = function(col_nm) { + cc <- q(col_nm) + rc <- q(paste0(col_nm, ref_suffix)) + at <- col_rules[[col_nm]][["abs"]] %||% 0 + rt <- col_rules[[col_nm]][["rel"]] %||% 0 within <- sprintf("ABS(%s - %s) <= (%s + %s * ABS(%s)) + %s * ABS(%s)", cc, rc, num(at), num(rt), rc, num(fp_eps), rc) - sprintf("%s AS %s", case_tol(cc, rc, cond = within), q(datadiff_ok_col(c))) + sprintf("%s AS %s", case_tol(cc, rc, cond = within), q(datadiff_ok_col(col_nm))) }, FUN.VALUE = character(1), USE.NAMES = FALSE) - eq_exprs <- vapply(X = eq_cols, FUN = function(c) { - cc <- q(c) - rc <- q(paste0(c, ref_suffix)) + eq_exprs <- vapply(X = eq_cols, FUN = function(col_nm) { + cc <- q(col_nm) + rc <- q(paste0(col_nm, ref_suffix)) sprintf("%s AS %s", - case_eq(cc, rc, nan_aware = c %in% eq_num_cols), - q(datadiff_eq_col(c))) + case_eq(cc, rc, nan_aware = col_nm %in% eq_num_cols), + q(datadiff_eq_col(col_nm))) }, FUN.VALUE = character(1), USE.NAMES = FALSE) exprs <- c(tol_exprs, eq_exprs) @@ -309,32 +316,35 @@ add_tolerance_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal # (one for __absdiff + __thresh, one for __ok) regardless of how many columns # there are. Each individual mutate() adds one lazy_query node to the dbplyr # plan; with 100+ columns the nesting would exceed R's expression stack limit. - if (length(tol_cols) == 0) return(cmp) + if (length(tol_cols) == 0) { + return(cmp) + } + + # IEEE 754: floating-point subtraction introduces rounding errors + # proportional to the magnitude of the operands, not to the threshold. + # e.g. 100.01 - 100.00 = 0.0100000000000051 > 0.01 in double precision. + # Adding a few ULPs of the reference magnitude absorbs this error without + # meaningfully widening the user-specified tolerance. + fp_eps <- 8 * .Machine$double.eps exprs_diff <- list() exprs_ok <- list() - for (c in tol_cols) { - reference_c <- paste0(c, ref_suffix) - abs_tol <- col_rules[[c]][["abs"]] %||% 0 - rel_tol <- col_rules[[c]][["rel"]] %||% 0 - c_sym <- dplyr::sym(c) + for (col_nm in tol_cols) { + reference_c <- paste0(col_nm, ref_suffix) + abs_tol <- col_rules[[col_nm]][["abs"]] %||% 0 + rel_tol <- col_rules[[col_nm]][["rel"]] %||% 0 + c_sym <- dplyr::sym(col_nm) rc_sym <- dplyr::sym(reference_c) - absdiff_col <- paste0(c, "__absdiff") - thresh_col <- paste0(c, "__thresh") - ok_col <- datadiff_ok_col(c) + absdiff_col <- paste0(col_nm, "__absdiff") + thresh_col <- paste0(col_nm, "__thresh") + ok_col <- datadiff_ok_col(col_nm) absdiff_sym <- dplyr::sym(absdiff_col) thresh_sym <- dplyr::sym(thresh_col) exprs_diff[[absdiff_col]] <- rlang::expr(abs(!!c_sym - !!rc_sym)) exprs_diff[[thresh_col]] <- rlang::expr(!!abs_tol + !!rel_tol * abs(!!rc_sym)) - # IEEE 754: floating-point subtraction introduces rounding errors - # proportional to the magnitude of the operands, not to the threshold. - # e.g. 100.01 - 100.00 = 0.0100000000000051 > 0.01 in double precision. - # Adding a few ULPs of the reference magnitude absorbs this error without - # meaningfully widening the user-specified tolerance. - fp_eps <- 8 * .Machine$double.eps if (na_equal) { exprs_ok[[ok_col]] <- rlang::expr(dplyr::case_when( is.na(!!c_sym) & is.na(!!rc_sym) ~ TRUE, @@ -373,11 +383,11 @@ add_tolerance_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal ok_cols <- vector("list", m) for (i in seq_len(m)) { - c <- tol_cols[i] + col_nm <- tol_cols[i] blocks <- compute_tolerance_col( - cand_vals = cmp[[c]], ref_vals = cmp[[paste0(c, ref_suffix)]], - abs_tol = col_rules[[c]][["abs"]] %||% 0, - rel_tol = col_rules[[c]][["rel"]] %||% 0, + cand_vals = cmp[[col_nm]], ref_vals = cmp[[paste0(col_nm, ref_suffix)]], + abs_tol = col_rules[[col_nm]][["abs"]] %||% 0, + rel_tol = col_rules[[col_nm]][["rel"]] %||% 0, na_equal = na_equal ) absdiff_cols[[i]] <- blocks$absdiff diff --git a/R/utils.R b/R/utils.R index f38d9af..8bde131 100644 --- a/R/utils.R +++ b/R/utils.R @@ -13,8 +13,12 @@ get_col_names <- function(x) { if (length(nms) == 0) names(x) else nms } +is_arrow <- function(x) { + inherits(x, c("ArrowObject", "arrow_dplyr_query")) +} + is_non_local <- function(x) { - inherits(x, "tbl_lazy") || inherits(x, c("ArrowObject", "arrow_dplyr_query")) + inherits(x, "tbl_lazy") || is_arrow(x) } # Package-local mutable state (temp-table counter). @@ -29,10 +33,6 @@ datadiff_tmp_table_name <- function() { sprintf("datadiff_tmp_%d_%d", Sys.getpid(), counter) } -is_arrow <- function(x) { - inherits(x, c("ArrowObject", "arrow_dplyr_query")) -} - # Convert an Arrow dataset to a DuckDB tbl_lazy on `con`. # # For file-backed Parquet datasets, uses DuckDB's native read_parquet() so diff --git a/inst/templates/rules_template.yaml b/inst/templates/rules_template.yaml deleted file mode 100644 index 95a28f3..0000000 --- a/inst/templates/rules_template.yaml +++ /dev/null @@ -1,37 +0,0 @@ -version: 1 -defaults: - na_equal: yes - ignore_columns: [] # Columns to ignore during schema comparison - keys: [] - label: ~ -row_validation: - check_count: no - expected_count: ~ - tolerance: 0.0 -by_type: - numeric: - abs: 1.0e-09 - rel: 1.0e-09 - integer: - abs: 0 - character: - equal_mode: exact - case_insensitive: no - trim: no - date: - equal_mode: exact - datetime: - equal_mode: exact - logical: - equal_mode: exact -by_name: - # Column-specific rules override type rules - # Examples: - # amount: - # abs: 0.001 # Absolute tolerance - # rel: 0.001 # Relative tolerance - # category: - # case_insensitive: yes - # trim: yes - # name: - # equal_mode: normalized From fd192eb14f08a6e7452fead84e4ee3ec38a03c4b Mon Sep 17 00:00:00 2001 From: infra-bot Date: Fri, 3 Jul 2026 11:54:15 +0200 Subject: [PATCH 35/40] test: nettoyage de la suite (issue #31) - doublon octet pour octet supprime (test-key-parameter 249-293) - plus aucune ecriture dans le repertoire courant : tempfile() + on.exit(unlink()) partout (politique CRAN, deja revendiquee par le NEWS 0.4.4) - fixtures orphelines rules.yaml / test.yaml supprimees (referencees par aucun test) - test-huge-parquet.R inexecutable partout (skip + chemins Windows en dur) deplace vers dev/manual-tests/ - test-extraction-params : les expect_ conditionnels silencieux assertent d'abord leur precondition (un check de cap ne peut plus passer a vide) - tests unitaires directs des petits helpers (format_key_examples, get_col_names, is_non_local/is_arrow, branche file = NULL du rapport) Non fait sciemment (documente NEWS) : la compression des redondances (-30% vise par l'issue) - refactoring a fort churn de tests verts, passe dediee. --- NEWS.md | 17 ++++ .../manual-tests/huge-parquet.R | 0 tests/testthat/rules.yaml | 29 ------- tests/testthat/test-extraction-params.R | 87 +++++++++---------- tests/testthat/test-input-validation.R | 15 ++-- tests/testthat/test-internal-helpers.R | 47 ++++++++++ tests/testthat/test-key-parameter.R | 76 ++++------------ tests/testthat/test.yaml | 29 ------- 8 files changed, 129 insertions(+), 171 deletions(-) rename tests/testthat/test-huge-parquet.R => dev/manual-tests/huge-parquet.R (100%) delete mode 100644 tests/testthat/rules.yaml create mode 100644 tests/testthat/test-internal-helpers.R delete mode 100644 tests/testthat/test.yaml diff --git a/NEWS.md b/NEWS.md index 7fa53e5..8c86fc3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -88,6 +88,23 @@ ## Chores +* Test-suite cleanup (issue #31): the byte-for-byte duplicated + "uses key parameter over YAML rules" block is gone; every test that wrote + a fixed-name YAML into the working directory now uses `tempfile()` + + `on.exit(unlink())` (CRAN policy, and what NEWS 0.4.4 already claimed); + the orphan committed fixtures `tests/testthat/rules.yaml` and `test.yaml` + (referenced by no test) are removed; the unrunnable + `test-huge-parquet.R` (skip-everywhere + hardcoded Windows paths) moves to + `dev/manual-tests/`; the silent conditional assertions of + `test-extraction-params.R` (`if (length(extracts) > 0) expect_...`) assert + their precondition first, so a cap check can no longer pass vacuously; and + the small internal helpers (`format_key_examples()`, `get_col_names()`, + `is_non_local()`/`is_arrow()`, the `file = NULL` branch of + `datadiff_report_html()`) get direct unit tests. The larger redundancy + compression the issue sketches (IEEE 754 in 7 copies, duplicate-key tests + in 3 files) is deliberately left for a dedicated pass: it is high-churn + refactoring of green tests. + * Packaging and code cleanup (issue #34): `dev/` is excluded from the tarball (`.Rbuildignore`); the orphan `inst/templates/rules_template.yaml` (referenced nowhere) is gone; unused `@importFrom` entries dropped diff --git a/tests/testthat/test-huge-parquet.R b/dev/manual-tests/huge-parquet.R similarity index 100% rename from tests/testthat/test-huge-parquet.R rename to dev/manual-tests/huge-parquet.R diff --git a/tests/testthat/rules.yaml b/tests/testthat/rules.yaml deleted file mode 100644 index 9fb092c..0000000 --- a/tests/testthat/rules.yaml +++ /dev/null @@ -1,29 +0,0 @@ -version: 1 -defaults: - na_equal: yes - ignore_columns: [] - keys: ~ - label: comparaison df -row_validation: - check_count: no - expected_count: ~ - tolerance: 0.0 -by_type: - numeric: - abs: 1.0e-09 - rel: 1.0e-09 - integer: - abs: 0 - character: - equal_mode: exact - case_insensitive: no - trim: no - date: - equal_mode: exact - datetime: - equal_mode: exact - logical: - equal_mode: exact -by_name: - id: [] - value: [] diff --git a/tests/testthat/test-extraction-params.R b/tests/testthat/test-extraction-params.R index 2156e11..a135b29 100644 --- a/tests/testthat/test-extraction-params.R +++ b/tests/testthat/test-extraction-params.R @@ -41,11 +41,10 @@ test_that("extract_failed = FALSE prevents row extraction", { expect_false(result$all_passed) # With extract_failed = FALSE, no rows should be extracted extracts <- pointblank::get_data_extracts(result$reponse) - # Extracts should be empty or contain empty data frames - if (length(extracts) > 0) { - total_rows <- sum(vapply(extracts, nrow, integer(1))) - expect_equal(total_rows, 0) - } + # Extracts must be empty or contain only empty data frames: computing the + # total over an empty list gives 0, so the assertion never silently skips + total_rows <- sum(vapply(extracts, nrow, integer(1))) + expect_equal(total_rows, 0) }) test_that("extract_failed accepts logical values only in practice", { @@ -86,12 +85,12 @@ test_that("get_first_n limits extracted rows", { extracts <- pointblank::get_data_extracts(result$reponse) # Each extract should have at most 5 rows - if (length(extracts) > 0) { - for (ext in extracts) { - if (is.data.frame(ext) && nrow(ext) > 0) { - expect_lte(nrow(ext), 5) - } - } + # Assert the precondition first: a failing comparison with extraction + # enabled must produce at least one extract (a silent empty list would + # make the cap check vacuous) + expect_gt(length(extracts), 0) + for (ext in extracts) { + expect_lte(nrow(as.data.frame(ext)), 5) } }) @@ -106,12 +105,12 @@ test_that("get_first_n = 1 extracts only first failure", { expect_false(result$all_passed) extracts <- pointblank::get_data_extracts(result$reponse) - if (length(extracts) > 0) { - for (ext in extracts) { - if (is.data.frame(ext) && nrow(ext) > 0) { - expect_lte(nrow(ext), 1) - } - } + # Assert the precondition first: a failing comparison with extraction + # enabled must produce at least one extract (a silent empty list would + # make the cap check vacuous) + expect_gt(length(extracts), 0) + for (ext in extracts) { + expect_lte(nrow(as.data.frame(ext)), 1) } }) @@ -160,12 +159,12 @@ test_that("sample_n limits extracted rows via random sampling", { expect_false(result$all_passed) extracts <- pointblank::get_data_extracts(result$reponse) - if (length(extracts) > 0) { - for (ext in extracts) { - if (is.data.frame(ext) && nrow(ext) > 0) { - expect_lte(nrow(ext), 10) - } - } + # Assert the precondition first: a failing comparison with extraction + # enabled must produce at least one extract (a silent empty list would + # make the cap check vacuous) + expect_gt(length(extracts), 0) + for (ext in extracts) { + expect_lte(nrow(as.data.frame(ext)), 10) } }) @@ -180,12 +179,12 @@ test_that("sample_n = 1 extracts only one random failure", { expect_false(result$all_passed) extracts <- pointblank::get_data_extracts(result$reponse) - if (length(extracts) > 0) { - for (ext in extracts) { - if (is.data.frame(ext) && nrow(ext) > 0) { - expect_lte(nrow(ext), 1) - } - } + # Assert the precondition first: a failing comparison with extraction + # enabled must produce at least one extract (a silent empty list would + # make the cap check vacuous) + expect_gt(length(extracts), 0) + for (ext in extracts) { + expect_lte(nrow(as.data.frame(ext)), 1) } }) @@ -279,12 +278,12 @@ test_that("sample_limit caps sample_frac results", { expect_false(result$all_passed) extracts <- pointblank::get_data_extracts(result$reponse) - if (length(extracts) > 0) { - for (ext in extracts) { - if (is.data.frame(ext) && nrow(ext) > 0) { - expect_lte(nrow(ext), 5) - } - } + # Assert the precondition first: a failing comparison with extraction + # enabled must produce at least one extract (a silent empty list would + # make the cap check vacuous) + expect_gt(length(extracts), 0) + for (ext in extracts) { + expect_lte(nrow(as.data.frame(ext)), 5) } }) @@ -300,12 +299,12 @@ test_that("sample_limit = 1 limits to single row", { expect_false(result$all_passed) extracts <- pointblank::get_data_extracts(result$reponse) - if (length(extracts) > 0) { - for (ext in extracts) { - if (is.data.frame(ext) && nrow(ext) > 0) { - expect_lte(nrow(ext), 1) - } - } + # Assert the precondition first: a failing comparison with extraction + # enabled must produce at least one extract (a silent empty list would + # make the cap check vacuous) + expect_gt(length(extracts), 0) + for (ext in extracts) { + expect_lte(nrow(as.data.frame(ext)), 1) } }) @@ -339,10 +338,8 @@ test_that("extract_failed = FALSE overrides other extraction params", { extracts <- pointblank::get_data_extracts(result$reponse) # With extract_failed = FALSE, should have no extracted rows - if (length(extracts) > 0) { - total_rows <- sum(vapply(extracts, nrow, integer(1))) - expect_equal(total_rows, 0) - } + total_rows <- sum(vapply(extracts, nrow, integer(1))) + expect_equal(total_rows, 0) }) test_that("get_first_n and sample_n are mutually exclusive (last one wins or pointblank handles)", { diff --git a/tests/testthat/test-input-validation.R b/tests/testthat/test-input-validation.R index 3f1d49f..9399278 100644 --- a/tests/testthat/test-input-validation.R +++ b/tests/testthat/test-input-validation.R @@ -9,13 +9,13 @@ test_that("write_rules_template handles invalid key parameter", { unlink(temp_path) # Test empty key vector (should error) - expect_error(write_rules_template(df, key = character(0), path = "test.yaml")) + expect_error(write_rules_template(df, key = character(0), path = tempfile(fileext = ".yaml"))) # Test non-character key (should error) - expect_error(write_rules_template(df, key = 123, path = "test.yaml")) + expect_error(write_rules_template(df, key = 123, path = tempfile(fileext = ".yaml"))) # Test non-existent column as key (should error) - expect_error(write_rules_template(df, key = "nonexistent", path = "test.yaml")) + expect_error(write_rules_template(df, key = "nonexistent", path = tempfile(fileext = ".yaml"))) }) test_that("write_rules_template handles invalid ignore_columns_default", { @@ -96,16 +96,17 @@ test_that("write_rules_template handles extreme values", { test_that("write_rules_template handles invalid data types", { # Test with non-dataframe input (should error) - expect_error(write_rules_template("not_a_dataframe", key = "id", path = "test.yaml")) + expect_error(write_rules_template("not_a_dataframe", key = "id", path = tempfile(fileext = ".yaml"))) # Test with empty dataframe (should error on key) empty_df <- data.frame() - expect_error(write_rules_template(empty_df, key = "id", path = "test.yaml")) + expect_error(write_rules_template(empty_df, key = "id", path = tempfile(fileext = ".yaml"))) # Test with dataframe having no rows (should work - creates template with empty by_name) no_rows_df <- data.frame(id = integer(0), value = numeric(0)) - expect_no_error(write_rules_template(no_rows_df, key = "id", path = "test_no_rows.yaml")) - if (file.exists("test_no_rows.yaml")) unlink("test_no_rows.yaml") + no_rows_path <- tempfile(fileext = ".yaml") + on.exit(unlink(no_rows_path), add = TRUE) + expect_no_error(write_rules_template(no_rows_df, key = "id", path = no_rows_path)) }) test_that("write_rules_template handles file path issues", { diff --git a/tests/testthat/test-internal-helpers.R b/tests/testthat/test-internal-helpers.R new file mode 100644 index 0000000..b8c38e3 --- /dev/null +++ b/tests/testthat/test-internal-helpers.R @@ -0,0 +1,47 @@ +# Direct unit tests for small internal helpers that previously had no +# dedicated coverage. + +test_that("format_key_examples formats and truncates", { + keys <- data.frame(id = c(1, 2, 3, 4), grp = c("a", "b", "c", "d")) + out <- format_key_examples(keys, c("id", "grp")) + expect_length(out, 4L) # 3 examples + "..." + expect_identical(out[4], "...") + expect_match(out[1], "id = 1") + expect_match(out[1], "grp = a") + + out2 <- format_key_examples(keys[1:2, , drop = FALSE], c("id", "grp")) + expect_length(out2, 2L) # below threshold: no marker +}) + +test_that("get_col_names works on data.frames and 0-column inputs", { + expect_identical(get_col_names(data.frame(a = 1, b = 2)), c("a", "b")) + expect_length(get_col_names(data.frame()), 0L) +}) + +test_that("is_non_local and is_arrow classify inputs", { + df <- data.frame(a = 1) + expect_false(is_non_local(df)) + expect_false(is_arrow(df)) + + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + duckdb::dbWriteTable(con, "t", df) + lazy <- dplyr::tbl(con, "t") + expect_true(is_non_local(lazy)) + expect_false(is_arrow(lazy)) + + skip_if_not_installed("arrow") + at <- arrow::arrow_table(df) + expect_true(is_non_local(at)) + expect_true(is_arrow(at)) +}) + +test_that("datadiff_report_html with file = NULL returns the report invisibly", { + ref <- data.frame(id = 1:2, x = c(1, 2)) + res <- suppressMessages(compare_datasets_from_yaml(ref, ref, key = "id")) + vis <- withVisible(datadiff_report_html(res, file = NULL)) + expect_false(vis$visible) + expect_false(is.null(vis$value)) +}) diff --git a/tests/testthat/test-key-parameter.R b/tests/testthat/test-key-parameter.R index 86fb923..9968132 100644 --- a/tests/testthat/test-key-parameter.R +++ b/tests/testthat/test-key-parameter.R @@ -26,22 +26,22 @@ by_type: abs: 0.5 ' - writeLines(yaml_content, 'test_key_param.yaml') + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(yaml_content, con = yaml_path) # Test 1: Without key parameter, should use YAML key (customer_id) - result_yaml <- compare_datasets_from_yaml(ref, cand, path = 'test_key_param.yaml') + result_yaml <- compare_datasets_from_yaml(ref, cand, path = yaml_path) expect_true(result_yaml$all_passed) # Test 2: With key parameter "id", should override YAML and use "id" - result_param <- compare_datasets_from_yaml(ref, cand, key = "id", path = 'test_key_param.yaml') + result_param <- compare_datasets_from_yaml(ref, cand, key = "id", path = yaml_path) expect_true(result_param$all_passed) # Test 3: With multiple keys parameter, should work - result_multi <- compare_datasets_from_yaml(ref, cand, key = c("id", "customer_id"), path = 'test_key_param.yaml') + result_multi <- compare_datasets_from_yaml(ref, cand, key = c("id", "customer_id"), path = yaml_path) expect_true(result_multi$all_passed) - # Clean up - unlink('test_key_param.yaml') }) test_that("compare_datasets_from_yaml key parameter handles NULL and missing keys", { @@ -58,18 +58,18 @@ by_type: abs: 0.5 ' - writeLines(yaml_content, 'test_key_null.yaml') + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(yaml_content, con = yaml_path) # Test 1: NULL key parameter with no keys in YAML should work (no key comparison) - result_null <- compare_datasets_from_yaml(ref, cand, key = NULL, path = 'test_key_null.yaml') + result_null <- compare_datasets_from_yaml(ref, cand, key = NULL, path = yaml_path) expect_true(result_null$all_passed) # Test 2: No key parameter (default NULL) with no keys in YAML - result_default <- compare_datasets_from_yaml(ref, cand, path = 'test_key_null.yaml') + result_default <- compare_datasets_from_yaml(ref, cand, path = yaml_path) expect_true(result_default$all_passed) - # Clean up - unlink('test_key_null.yaml') }) test_that("compare_datasets_from_yaml errors when key columns are absent", { @@ -198,15 +198,15 @@ by_type: abs: 0.1 ' - writeLines(yaml_content, 'test_key_precedence.yaml') + yaml_path <- tempfile(fileext = ".yaml") + on.exit(unlink(yaml_path), add = TRUE) + writeLines(yaml_content, con = yaml_path) # Using YAML key (category) would compare X->X, Y->Z, Z->Y which fails # But we override with id parameter - result <- compare_datasets_from_yaml(ref, cand, key = "id", path = 'test_key_precedence.yaml') + result <- compare_datasets_from_yaml(ref, cand, key = "id", path = yaml_path) expect_true(result$all_passed) - # Clean up - unlink('test_key_precedence.yaml') }) # --- Duplicate key detection tests --- @@ -321,52 +321,6 @@ test_that("no warning when keys are unique", { ) }) -test_that("compare_datasets_from_yaml uses key parameter over YAML rules", { - # Create test data with multiple potential key columns - ref <- data.frame( - id = 1:3, - customer_id = c("A", "B", "C"), - value = c(10.0, 20.0, 30.0), - name = c("Alice", "Bob", "Charlie") - ) - - # Candidate data - same values but different order to test key-based sorting - cand <- data.frame( - id = c(2, 1, 3), - customer_id = c("B", "A", "C"), - value = c(20.1, 10.1, 30.1), - name = c("Bob", "Alice", "Charlie") - ) - - # YAML with customer_id as key - yaml_content <- ' -version: 1 -defaults: - na_equal: yes - keys: ["customer_id"] -by_type: - numeric: - abs: 0.5 -' - - writeLines(yaml_content, 'test_key_param.yaml') - - # Test 1: Without key parameter, should use YAML key (customer_id) - result_yaml <- compare_datasets_from_yaml(ref, cand, path = 'test_key_param.yaml') - expect_true(result_yaml$all_passed) - - # Test 2: With key parameter "id", should override YAML and use "id" - result_param <- compare_datasets_from_yaml(ref, cand, key = "id", path = 'test_key_param.yaml') - expect_true(result_param$all_passed) - - # Test 3: With multiple keys parameter, should work - result_multi <- compare_datasets_from_yaml(ref, cand, key = c("id", "customer_id"), path = 'test_key_param.yaml') - expect_true(result_multi$all_passed) - - # Clean up - unlink('test_key_param.yaml') -}) - test_that("warning truncates to 3 examples when more than 3 duplicate key values in reference", { # 4 unique duplicate key values -> takes the else branch (n_dup_keys > 3) diff --git a/tests/testthat/test.yaml b/tests/testthat/test.yaml deleted file mode 100644 index 9fb092c..0000000 --- a/tests/testthat/test.yaml +++ /dev/null @@ -1,29 +0,0 @@ -version: 1 -defaults: - na_equal: yes - ignore_columns: [] - keys: ~ - label: comparaison df -row_validation: - check_count: no - expected_count: ~ - tolerance: 0.0 -by_type: - numeric: - abs: 1.0e-09 - rel: 1.0e-09 - integer: - abs: 0 - character: - equal_mode: exact - case_insensitive: no - trim: no - date: - equal_mode: exact - datetime: - equal_mode: exact - logical: - equal_mode: exact -by_name: - id: [] - value: [] From 3c433029d3da48fa41db448f06baadbdebbe349c Mon Sep 17 00:00:00 2001 From: vincent Date: Sun, 12 Jul 2026 10:04:39 +0200 Subject: [PATCH 36/40] feat!: renommer le champ resultat reponse en response avec alias deprecie Le resultat de compare_datasets_from_yaml() porte la classe datadiff_result : ses methodes $ et [[ redirigent l'ancien nom reponse vers response avec un warning de depreciation (une fois par session), suppression prevue dans une release future. datadiff_report_html() accepte encore un resultat sauvegarde par une ancienne version. Le code qui inspecte names(result) doit migrer. BREAKING CHANGE: le champ reponse devient response ; l'alias ne figure pas dans names(result). --- NAMESPACE | 3 + R/compare_datasets_from_yaml.R | 29 +++++---- R/datadiff-result.R | 51 +++++++++++++++ R/report.R | 46 +++++++------- README.Rmd | 4 +- README.md | 4 +- dev/RFC-api-0.5.md | 20 +++--- man/compare_datasets_from_yaml.Rd | 13 ++-- man/print.datadiff_report.Rd | 2 +- tests/testthat/test-avoidable-scans.R | 2 +- tests/testthat/test-default-rules.R | 6 +- tests/testthat/test-extract-diagnostics.R | 6 +- tests/testthat/test-extraction-params.R | 24 +++---- tests/testthat/test-internal-coverage.R | 4 +- tests/testthat/test-lazy-aggregates.R | 8 +-- tests/testthat/test-report-extracts-dir.R | 4 +- tests/testthat/test-report-robustness.R | 14 ++--- tests/testthat/test-report.R | 46 +++++++------- tests/testthat/test-response-field.R | 76 +++++++++++++++++++++++ vignettes/datadiff.Rmd | 22 ++++--- 20 files changed, 268 insertions(+), 116 deletions(-) create mode 100644 R/datadiff-result.R create mode 100644 tests/testthat/test-response-field.R diff --git a/NAMESPACE b/NAMESPACE index 1d82dd0..678b00e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,7 +1,10 @@ # Generated by roxygen2: do not edit by hand +S3method("$",datadiff_result) +S3method("[[",datadiff_result) S3method(print,datadiff_coverage) S3method(print,datadiff_report) +S3method(print,datadiff_result) export(add_tolerance_columns) export(analyze_columns) export(compare_datasets_from_yaml) diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 0954a26..41503e0 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -343,16 +343,19 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' literal (an invalid one is an error on every path), but it only takes #' effect when Arrow datasets are used: plain `data.frame`s or `tbl_lazy` #' inputs ignore a valid value. -#' @return A list containing: +#' @return A list of class \code{datadiff_result} containing: #' \item{all_passed}{Logical; \code{TRUE} when every check passed. The #' first element of the returned list and the single verdict consumers #' should read.} #' \item{agent}{The configured pointblank agent as built (NOT interrogated: -#' the verdict lives in \code{reponse}, which is the interrogated one)} -#' \item{reponse}{Interrogated pointblank agent (class \code{datadiff_report}): +#' the verdict lives in \code{response}, which is the interrogated one)} +#' \item{response}{Interrogated pointblank agent (class \code{datadiff_report}): #' usable by \code{pointblank::all_passed()} / \code{get_data_extracts()}; #' printing it lazily renders a full pointblank-style report from -#' \code{coverage} (built on demand, memoized).} +#' \code{coverage} (built on demand, memoized). Reading it under its +#' historical name \code{reponse} still works but is deprecated and +#' emits a warning (once per session); the alias will be removed in a +#' future release.} #' \item{missing_in_candidate}{Columns missing in candidate data} #' \item{extra_in_candidate}{Extra columns in candidate data} #' \item{applied_rules}{Final column-specific rules applied} @@ -381,7 +384,7 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' tmp <- tempfile(fileext = ".yaml") #' write_rules_template(ref, key = "id", path = tmp) #' result <- compare_datasets_from_yaml(ref, cand, key = "id", path = tmp) -#' result$reponse +#' result$response compare_datasets_from_yaml <- function(data_reference, data_candidate, key = NULL, @@ -934,7 +937,7 @@ compare_datasets_from_yaml <- function(data_reference, ) } - reponse <- interrogate( + response <- interrogate( agent, extract_failed = extract_failed, get_first_n = get_first_n, @@ -943,24 +946,24 @@ compare_datasets_from_yaml <- function(data_reference, sample_limit = sample_limit ) - all_passed <- pointblank::all_passed(reponse) + all_passed <- pointblank::all_passed(response) - # Make reponse render the full pointblank report lazily (on print) from the + # Make response render the full pointblank report lazily (on print) from the # coverage, while remaining a real interrogated agent for all_passed() and # get_data_extracts(). - reponse <- as_datadiff_report( - reponse, coverage = coverage, label = label, lang = lang, locale = locale, + response <- as_datadiff_report( + response, coverage = coverage, label = label, lang = lang, locale = locale, warn_at = warn_at, stop_at = stop_at ) - list( + new_datadiff_result(list( all_passed = all_passed, agent = agent, - reponse = reponse, + response = response, missing_in_candidate = missing_in_candidate, extra_in_candidate = extra_in_candidate, applied_rules = col_rules, coverage = coverage, summary = summarize_coverage(coverage) - ) + )) } diff --git a/R/datadiff-result.R b/R/datadiff-result.R new file mode 100644 index 0000000..7e9d787 --- /dev/null +++ b/R/datadiff-result.R @@ -0,0 +1,51 @@ +# Accessors for the result of compare_datasets_from_yaml(). +# +# The result is a plain named list of class `datadiff_result`. The class only +# exists to keep the historical `reponse` field name readable while user code +# migrates to `response`: `$` and `[[` redirect the old name to the new one +# with a deprecation warning, and behave like plain list access otherwise. + +new_datadiff_result <- function(fields) { + structure(fields, class = "datadiff_result") +} + +warn_reponse_deprecated <- function() { + rlang::warn( + message = paste( + "The `reponse` field is deprecated as of {datadiff} 0.6.0:", + "use `response` instead.", + "`reponse` will be removed in a future release." + ), + .frequency = "once", + .frequency_id = "datadiff_reponse_deprecated", + class = "datadiff_deprecated_reponse_warning" + ) +} + +#' @export +#' @noRd +`$.datadiff_result` <- function(x, name) { + if (identical(name, "reponse")) { + warn_reponse_deprecated() + name <- "response" + } + # exact = FALSE replicates the silent partial matching of `$` on plain lists + unclass(x)[[name, exact = FALSE]] +} + +#' @export +#' @noRd +`[[.datadiff_result` <- function(x, i, ...) { + if (identical(i, "reponse")) { + warn_reponse_deprecated() + i <- "response" + } + unclass(x)[[i, ...]] +} + +#' @export +#' @noRd +print.datadiff_result <- function(x, ...) { + print(unclass(x), ...) + invisible(x) +} diff --git a/R/report.R b/R/report.R index 75136ac..ce9f0b0 100644 --- a/R/report.R +++ b/R/report.R @@ -1,9 +1,9 @@ # Lazy pointblank-style report. # -# The fast path keeps res$reponse a real interrogated agent (minimal when green, +# The fast path keeps res$response a real interrogated agent (minimal when green, # targeted when red) so pointblank::all_passed() and get_data_extracts() keep # working. We prefix the class "datadiff_report" and attach the coverage so that -# PRINTING res$reponse builds, on demand, a full pointblank agent report (one +# PRINTING res$response builds, on demand, a full pointblank agent report (one # step per column) from the pre-computed coverage - without re-running the slow # per-column interrogation. The build cost is paid only when the report is # actually displayed, and memoized so repeated prints are instant. @@ -180,17 +180,17 @@ mark_agent_interrogated <- function(agent) { # Attach the lazy-report capability to a real interrogated agent: prefix the # class (print dispatches to print.datadiff_report) and stash what is needed to # render the full report, plus a (reference-semantics) environment for caching. -as_datadiff_report <- function(reponse, coverage, label, lang, locale, +as_datadiff_report <- function(response, coverage, label, lang, locale, warn_at = datadiff_default_warn_at, stop_at = datadiff_default_stop_at) { - attr(reponse, "datadiff_coverage") <- coverage - attr(reponse, "datadiff_label") <- label - attr(reponse, "datadiff_lang") <- lang - attr(reponse, "datadiff_locale") <- locale - attr(reponse, "datadiff_warn_at") <- warn_at - attr(reponse, "datadiff_stop_at") <- stop_at - attr(reponse, "datadiff_render") <- new.env(parent = emptyenv()) - class(reponse) <- c("datadiff_report", class(reponse)) - reponse + attr(response, "datadiff_coverage") <- coverage + attr(response, "datadiff_label") <- label + attr(response, "datadiff_lang") <- lang + attr(response, "datadiff_locale") <- locale + attr(response, "datadiff_warn_at") <- warn_at + attr(response, "datadiff_stop_at") <- stop_at + attr(response, "datadiff_render") <- new.env(parent = emptyenv()) + class(response) <- c("datadiff_report", class(response)) + response } # Build (once) and memoize the pointblank agent report for a datadiff_report. @@ -221,7 +221,7 @@ datadiff_render_report <- function(x) { #' pointblank-style report (HTML in interactive sessions / viewer). The report #' is built on first print and memoized. #' -#' @param x A `datadiff_report` (the `reponse` element of a comparison result). +#' @param x A `datadiff_report` (the `response` element of a comparison result). #' @param ... Unused. #' @return `x`, invisibly. #' @exportS3Method print datadiff_report @@ -260,33 +260,37 @@ datadiff_report_html <- function(res, file = NULL, extracts_dir = NULL) { if (is.null(coverage)) { stop("`res` has no `coverage`; was it produced by compare_datasets_from_yaml()?") } - reponse <- res$reponse - report <- if (inherits(reponse, "datadiff_report")) { + response <- res[["response"]] + if (is.null(response)) { + # A result saved by a version where the field was still named `reponse` + response <- res[["reponse"]] + } + report <- if (inherits(response, "datadiff_report")) { # Share the print() memoization: read the cached report when a print # already built it, and feed the cache otherwise - datadiff_render_report(reponse) + datadiff_render_report(response) } else { # No lazy-report wrapper (hand-assembled res): one-shot synthetic build pointblank::get_agent_report(build_report_agent( coverage = coverage, label = "datadiff report", - real_agent = reponse + real_agent = response )) } if (!is.null(file)) { pointblank::export_report(report, filename = file, quiet = TRUE) } if (!is.null(extracts_dir)) { - write_extract_csvs(reponse, extracts_dir = extracts_dir) + write_extract_csvs(response, extracts_dir = extracts_dir) } invisible(report) } # Write each failing-step extract of an interrogated agent as a CSV file. # Returns the written paths, invisibly. -write_extract_csvs <- function(reponse, extracts_dir) { +write_extract_csvs <- function(response, extracts_dir) { extracts <- tryCatch( - pointblank::get_data_extracts(reponse), + pointblank::get_data_extracts(response), error = function(e) { warning(sprintf( "extracts_dir: could not read the data extracts (%s); no CSV written.", @@ -301,7 +305,7 @@ write_extract_csvs <- function(reponse, extracts_dir) { if (!dir.exists(extracts_dir)) { dir.create(extracts_dir, recursive = TRUE) } - vs <- reponse$validation_set + vs <- response$validation_set paths <- vapply(X = names(extracts), FUN = function(nm) { i <- as.integer(nm) # Look the step up by its id (vs$i), not by row position: contiguity of diff --git a/README.Rmd b/README.Rmd index 8a77056..0c8d68d 100644 --- a/README.Rmd +++ b/README.Rmd @@ -67,7 +67,7 @@ result$summary # aggregate counts (n_checks, n_pass, n_fail, ...) # Printing the response lazily renders the full pointblank-style report # (built on demand, only when displayed): -print(result$reponse) +print(result$response) # ...or export that report to a standalone HTML file: datadiff_report_html(result, file = "report.html") @@ -84,7 +84,7 @@ datadiff_report_html(result, file = "report.html") - **Fast on wide tables**: an all-pass comparison short-circuits the per-column pointblank agent, so validating hundreds/thousands of columns stays quick - **Faithful coverage**: `result$coverage` always lists every check performed - (even when everything passes), and `result$reponse` renders the full + (even when everything passes), and `result$response` renders the full pointblank HTML report lazily, only when printed ## Dependencies diff --git a/README.md b/README.md index 814bffd..fe81665 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ result$summary # aggregate counts (n_checks, n_pass, n_fail, ...) # Printing the response lazily renders the full pointblank-style report # (built on demand, only when displayed): -print(result$reponse) +print(result$response) # ...or export that report to a standalone HTML file: datadiff_report_html(result, file = "report.html") @@ -81,7 +81,7 @@ datadiff_report_html(result, file = "report.html") per-column pointblank agent, so validating hundreds/thousands of columns stays quick - **Faithful coverage**: `result$coverage` always lists every check - performed (even when everything passes), and `result$reponse` renders + performed (even when everything passes), and `result$response` renders the full pointblank HTML report lazily, only when printed ## Dependencies diff --git a/dev/RFC-api-0.5.md b/dev/RFC-api-0.5.md index 5fa6787..1cd4404 100644 --- a/dev/RFC-api-0.5.md +++ b/dev/RFC-api-0.5.md @@ -9,8 +9,9 @@ leur issue de rattachement. La fonction centrale `compare_datasets_from_yaml()` a grossi par accretion : 17 parametres a plat, un nom qui ne reflete plus l'usage (le YAML est optionnel depuis 0.1.5), un retour a 8 champs partiellement redondants, et un -melange de langues (arguments anglais, champ `$reponse` en francais, rapport -par defaut en francais, messages en anglais). +melange de langues (arguments anglais, champ historique `$reponse` en +francais, renomme `$response` en 0.6.0 avec alias deprecie, rapport par +defaut en francais, messages en anglais). ## Proposition @@ -51,9 +52,11 @@ compare_datasets( ### 3. Retour : classe `datadiff_result` Etat actuel : `all_passed` existe en 3 exemplaires (`$all_passed`, -`$summary$all_passed`, `pointblank::all_passed($reponse)`) ; `$agent` n'est +`$summary$all_passed`, `pointblank::all_passed($response)`) ; `$agent` n'est pas interroge (et il est factice sur le fast-path all-pass) sans usage -utilisateur identifie ; `$reponse` est du franglais. +utilisateur identifie. Depuis 0.6.0, le champ s'appelle `$response` (la +classe `datadiff_result` et la mecanique de depreciation `$`/`[[` proposees +ici existent deja ; `$reponse` reste lisible avec warning). Cible : @@ -61,12 +64,15 @@ Cible : res$passed # le verdict, une seule fois res$coverage # inchange res$summary # inchange (sans all_passed duplique) -res$report # l'agent interroge (ex-$reponse), print() paresseux inchange +res$report # l'agent interroge (ex-$response), print() paresseux inchange + # NB : renommer response -> report imposerait une 2e migration + # aux utilisateurs ; a arbitrer sur l'issue #33 res$applied_rules, res$missing_in_candidate, res$extra_in_candidate ``` -- `$reponse` et `$all_passed` restent presents une version avec un warning de - depreciation a l'acces (active binding ou methode `$.datadiff_result`). +- `$response` (et l'alias historique `$reponse`) et `$all_passed` restent + presents une version avec un warning de depreciation a l'acces (la methode + `$.datadiff_result` de 0.6.0 fournit deja la mecanique). - `$agent` est retire du contrat documente (garde interne si necessaire). ### 4. warn_at / stop_at diff --git a/man/compare_datasets_from_yaml.Rd b/man/compare_datasets_from_yaml.Rd index e550b56..628a436 100644 --- a/man/compare_datasets_from_yaml.Rd +++ b/man/compare_datasets_from_yaml.Rd @@ -88,16 +88,19 @@ effect when Arrow datasets are used: plain \code{data.frame}s or \code{tbl_lazy} inputs ignore a valid value.} } \value{ -A list containing: +A list of class \code{datadiff_result} containing: \item{all_passed}{Logical; \code{TRUE} when every check passed. The first element of the returned list and the single verdict consumers should read.} \item{agent}{The configured pointblank agent as built (NOT interrogated: -the verdict lives in \code{reponse}, which is the interrogated one)} -\item{reponse}{Interrogated pointblank agent (class \code{datadiff_report}): +the verdict lives in \code{response}, which is the interrogated one)} +\item{response}{Interrogated pointblank agent (class \code{datadiff_report}): usable by \code{pointblank::all_passed()} / \code{get_data_extracts()}; printing it lazily renders a full pointblank-style report from -\code{coverage} (built on demand, memoized).} +\code{coverage} (built on demand, memoized). Reading it under its +historical name \code{reponse} still works but is deprecated and +emits a warning (once per session); the alias will be removed in a +future release.} \item{missing_in_candidate}{Columns missing in candidate data} \item{extra_in_candidate}{Extra columns in candidate data} \item{applied_rules}{Final column-specific rules applied} @@ -153,5 +156,5 @@ result <- compare_datasets_from_yaml(ref, cand, key = "id") tmp <- tempfile(fileext = ".yaml") write_rules_template(ref, key = "id", path = tmp) result <- compare_datasets_from_yaml(ref, cand, key = "id", path = tmp) -result$reponse +result$response } diff --git a/man/print.datadiff_report.Rd b/man/print.datadiff_report.Rd index ec81399..dfc3f47 100644 --- a/man/print.datadiff_report.Rd +++ b/man/print.datadiff_report.Rd @@ -7,7 +7,7 @@ \method{print}{datadiff_report}(x, ...) } \arguments{ -\item{x}{A \code{datadiff_report} (the \code{reponse} element of a comparison result).} +\item{x}{A \code{datadiff_report} (the \code{response} element of a comparison result).} \item{...}{Unused.} } diff --git a/tests/testthat/test-avoidable-scans.R b/tests/testthat/test-avoidable-scans.R index cce2882..2f5c0b1 100644 --- a/tests/testthat/test-avoidable-scans.R +++ b/tests/testthat/test-avoidable-scans.R @@ -36,7 +36,7 @@ test_that("ignored and extra columns stay out of the joined comparison", { # The failing-row extract carries the key and the compared columns, not the # ignored column nor the candidate-only extra column - ex <- pointblank::get_data_extracts(res$reponse) + ex <- pointblank::get_data_extracts(res$response) ex1 <- as.data.frame(ex[[1]]) expect_true("id" %in% names(ex1)) expect_false("noise" %in% names(ex1)) diff --git a/tests/testthat/test-default-rules.R b/tests/testthat/test-default-rules.R index 2893dcd..cfe142d 100644 --- a/tests/testthat/test-default-rules.R +++ b/tests/testthat/test-default-rules.R @@ -7,7 +7,7 @@ test_that("compare_datasets_from_yaml works without path (default rules)", { result <- compare_datasets_from_yaml(ref, cand) expect_true(result$all_passed) - expect_s3_class(result$reponse, "ptblank_agent") + expect_s3_class(result$response, "ptblank_agent") }) test_that("compare_datasets_from_yaml with key but no path works", { @@ -17,7 +17,7 @@ test_that("compare_datasets_from_yaml with key but no path works", { result <- compare_datasets_from_yaml(ref, cand, key = "id") expect_true(result$all_passed) - expect_s3_class(result$reponse, "ptblank_agent") + expect_s3_class(result$response, "ptblank_agent") }) test_that("compare_datasets_from_yaml detects differences without path", { @@ -204,7 +204,7 @@ test_that("custom label is used when path is NULL", { result <- compare_datasets_from_yaml(ref, cand, label = "Custom Test Label") # The label should be set in the agent - expect_s3_class(result$reponse, "ptblank_agent") + expect_s3_class(result$response, "ptblank_agent") }) test_that("comparison without path handles missing columns", { diff --git a/tests/testthat/test-extract-diagnostics.R b/tests/testthat/test-extract-diagnostics.R index 55baed3..f0b93f6 100644 --- a/tests/testthat/test-extract-diagnostics.R +++ b/tests/testthat/test-extract-diagnostics.R @@ -22,7 +22,7 @@ test_that("failing extracts expose the measured deviation and threshold", { expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) expect_true(length(extracts) >= 1) ex <- extracts[[1]] @@ -53,7 +53,7 @@ test_that("diagnostics are computed for failing tolerance columns only", { expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) ex <- extracts[[1]] expect_true(all(c("bad__absdiff", "bad__thresh") %in% names(ex))) @@ -72,7 +72,7 @@ test_that("all-pass fast path materialises no diagnostic columns", { expect_true(result$all_passed) expect_false(any(grepl("__absdiff$|__thresh$", colnames(result$agent$tbl)))) - expect_length(pointblank::get_data_extracts(result$reponse), 0L) + expect_length(pointblank::get_data_extracts(result$response), 0L) }) test_that("add_diff_columns appends only __absdiff/__thresh, matching the kernel", { diff --git a/tests/testthat/test-extraction-params.R b/tests/testthat/test-extraction-params.R index a135b29..18bd66c 100644 --- a/tests/testthat/test-extraction-params.R +++ b/tests/testthat/test-extraction-params.R @@ -26,7 +26,7 @@ test_that("extract_failed defaults to TRUE", { expect_false(result$all_passed) # With extract_failed = TRUE (default), extracts should contain data - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) expect_true(length(extracts) > 0 || is.list(extracts)) }) @@ -40,7 +40,7 @@ test_that("extract_failed = FALSE prevents row extraction", { expect_false(result$all_passed) # With extract_failed = FALSE, no rows should be extracted - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) # Extracts must be empty or contain only empty data frames: computing the # total over an empty list gives 0, so the assertion never silently skips total_rows <- sum(vapply(extracts, nrow, integer(1))) @@ -82,7 +82,7 @@ test_that("get_first_n limits extracted rows", { ) expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) # Each extract should have at most 5 rows # Assert the precondition first: a failing comparison with extraction @@ -103,7 +103,7 @@ test_that("get_first_n = 1 extracts only first failure", { ) expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) # Assert the precondition first: a failing comparison with extraction # enabled must produce at least one extract (a silent empty list would @@ -157,7 +157,7 @@ test_that("sample_n limits extracted rows via random sampling", { ) expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) # Assert the precondition first: a failing comparison with extraction # enabled must produce at least one extract (a silent empty list would @@ -177,7 +177,7 @@ test_that("sample_n = 1 extracts only one random failure", { ) expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) # Assert the precondition first: a failing comparison with extraction # enabled must produce at least one extract (a silent empty list would @@ -276,7 +276,7 @@ test_that("sample_limit caps sample_frac results", { ) expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) # Assert the precondition first: a failing comparison with extraction # enabled must produce at least one extract (a silent empty list would @@ -297,7 +297,7 @@ test_that("sample_limit = 1 limits to single row", { ) expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) # Assert the precondition first: a failing comparison with extraction # enabled must produce at least one extract (a silent empty list would @@ -335,7 +335,7 @@ test_that("extract_failed = FALSE overrides other extraction params", { ) expect_false(result$all_passed) - extracts <- pointblank::get_data_extracts(result$reponse) + extracts <- pointblank::get_data_extracts(result$response) # With extract_failed = FALSE, should have no extracted rows total_rows <- sum(vapply(extracts, nrow, integer(1))) @@ -433,7 +433,7 @@ test_that("lightweight mode (extract_failed = FALSE) produces smaller output", { expect_equal(result_full$all_passed, result_light$all_passed) # Light version should have no extracts - extracts_light <- pointblank::get_data_extracts(result_light$reponse) + extracts_light <- pointblank::get_data_extracts(result_light$response) if (length(extracts_light) > 0) { total_rows_light <- sum(vapply(extracts_light, nrow, integer(1))) expect_equal(total_rows_light, 0) @@ -454,8 +454,8 @@ test_that("get_first_n reduces extract size compared to no limit", { get_first_n = 5 ) - extracts_unlimited <- pointblank::get_data_extracts(result_unlimited$reponse) - extracts_limited <- pointblank::get_data_extracts(result_limited$reponse) + extracts_unlimited <- pointblank::get_data_extracts(result_unlimited$response) + extracts_limited <- pointblank::get_data_extracts(result_limited$response) # Count total rows in extracts count_rows <- function(extracts) { diff --git a/tests/testthat/test-internal-coverage.R b/tests/testthat/test-internal-coverage.R index d2b6fa9..e9c55f2 100644 --- a/tests/testthat/test-internal-coverage.R +++ b/tests/testthat/test-internal-coverage.R @@ -21,7 +21,7 @@ test_that("build_report_agent returns an agent for empty coverage (no real agent }) test_that("datadiff_report_html errors when the result has no coverage", { - expect_error(datadiff_report_html(list(reponse = NULL)), "coverage") + expect_error(datadiff_report_html(list(response = NULL)), "coverage") }) test_that("datadiff_render_report tolerates a missing cache environment", { @@ -33,7 +33,7 @@ test_that("datadiff_render_report tolerates a missing cache environment", { res <- suppressMessages( compare_datasets_from_yaml(ref, ref, key = c("id", ".row"), path = tmp) ) - x <- res$reponse + x <- res$response attr(x, "datadiff_render") <- NULL # force the fallback branch rep <- datadiff_render_report(x) expect_s3_class(rep, "gt_tbl") diff --git a/tests/testthat/test-lazy-aggregates.R b/tests/testthat/test-lazy-aggregates.R index d385e37..e8618c7 100644 --- a/tests/testthat/test-lazy-aggregates.R +++ b/tests/testthat/test-lazy-aggregates.R @@ -24,7 +24,7 @@ test_that("a green lazy comparison does not collect the boolean table", { expect_identical(as.integer(cov$n[cov$column == "a" & cov$check == "tolerance"]), as.integer(n)) expect_identical(as.integer(sum(cov$n_failed)), 0L) # The agent is the constant-size pass placeholder, not N collected rows - expect_lte(nrow(res$reponse$tbl), 1L) + expect_lte(nrow(res$response$tbl), 1L) }) test_that("a red lazy comparison collects only the failing boolean columns", { @@ -49,7 +49,7 @@ test_that("a red lazy comparison collects only the failing boolean columns", { # Only the failing tolerance column (plus the row-count flag consumed by # its validation step) reaches the local agent: b__ok and s__eq stay in # the database - agent_cols <- names(res$reponse$tbl) + agent_cols <- names(res$response$tbl) expect_true("a__ok" %in% agent_cols) expect_false("b__ok" %in% agent_cols) expect_false("s__eq" %in% agent_cols) @@ -62,7 +62,7 @@ test_that("a red lazy comparison collects only the failing boolean columns", { expect_identical(as.integer(cov$n_failed[cov$column == "s" & cov$check == "equality"]), 0L) # And the extracts still surface the failing rows - ex <- pointblank::get_data_extracts(res$reponse) + ex <- pointblank::get_data_extracts(res$response) expect_gte(length(ex), 1L) }) @@ -109,7 +109,7 @@ test_that("structural-only lazy failure keeps a failing verdict (no value checks # The missing column is the only failure: the agent verdict must agree # with the coverage, not pass on a 0-unit dummy step expect_false(res$all_passed) - expect_false(pointblank::all_passed(res$reponse)) + expect_false(pointblank::all_passed(res$response)) expect_identical(res$missing_in_candidate, "extra_ref") }) diff --git a/tests/testthat/test-report-extracts-dir.R b/tests/testthat/test-report-extracts-dir.R index ca649f6..3dfe31c 100644 --- a/tests/testthat/test-report-extracts-dir.R +++ b/tests/testthat/test-report-extracts-dir.R @@ -53,14 +53,14 @@ test_that("column names are sanitized in extract file names", { expect_false(grepl("[ ()/]", csvs)) }) -test_that("an unreadable reponse warns explicitly and writes nothing", { +test_that("an unreadable response warns explicitly and writes nothing", { res_broken <- list( coverage = structure( data.frame(column = "x", check = "tolerance", n = 1L, n_failed = 1L, status = "FAIL", stringsAsFactors = FALSE), class = c("datadiff_coverage", "data.frame") ), - reponse = list() + response = list() ) out_dir <- tempfile(pattern = "datadiff_extracts_") on.exit(unlink(out_dir, recursive = TRUE), add = TRUE) diff --git a/tests/testthat/test-report-robustness.R b/tests/testthat/test-report-robustness.R index cc690a7..123bfa7 100644 --- a/tests/testthat/test-report-robustness.R +++ b/tests/testthat/test-report-robustness.R @@ -7,7 +7,7 @@ test_that("datadiff_report_html reads and feeds the print() memoization", { cand <- data.frame(id = 1:2, x = c(1, 3)) res <- suppressMessages(compare_datasets_from_yaml(ref, cand, key = "id")) - cache <- attr(res$reponse, "datadiff_render") + cache <- attr(res$response, "datadiff_render") expect_true(is.environment(cache)) expect_null(cache$report) @@ -28,18 +28,18 @@ test_that("an eval_error on the real agent reaches the report", { # Simulate a step whose interrogation failed to evaluate: pointblank stores # eval_error = TRUE and n_failed = NA for such steps. Target the x__ok value # step (the one the report maps back to the "x" coverage row). - vs <- res$reponse$validation_set + vs <- res$response$validation_set err_idx <- which(vapply(vs$column, function(cc) { identical(cc[1], "x__ok") }, logical(1)))[1] vs$eval_error[err_idx] <- TRUE vs$n_failed[err_idx] <- NA_real_ - res$reponse$validation_set <- vs + res$response$validation_set <- vs agent <- build_report_agent( coverage = res$coverage, label = "eval error propagation", - real_agent = res$reponse + real_agent = res$response ) # The synthetic branch must not swallow the failure: the rebuilt validation @@ -53,15 +53,15 @@ test_that("an all-NA n_failed agent (pure eval_error) is not treated as all-pass res <- suppressMessages(compare_datasets_from_yaml(ref, cand, key = "id")) # Every real step errored: n_failed is NA everywhere, eval_error TRUE - vs <- res$reponse$validation_set + vs <- res$response$validation_set vs$eval_error <- rep(TRUE, nrow(vs)) vs$n_failed <- rep(NA_real_, nrow(vs)) - res$reponse$validation_set <- vs + res$response$validation_set <- vs agent <- build_report_agent( coverage = res$coverage, label = "pure eval error", - real_agent = res$reponse + real_agent = res$response ) # Pre-fix, the any(n_failed > 0, na.rm = TRUE) guard was FALSE and the diff --git a/tests/testthat/test-report.R b/tests/testthat/test-report.R index 9b34e99..1f38e2d 100644 --- a/tests/testthat/test-report.R +++ b/tests/testthat/test-report.R @@ -1,8 +1,8 @@ # TDD spec for the lazy pointblank-style HTML report. # -# The fast path keeps res$reponse a real interrogated agent (so all_passed() and +# The fast path keeps res$response a real interrogated agent (so all_passed() and # get_data_extracts() keep working), but prefixes class "datadiff_report" and -# attaches the coverage so that PRINTING res$reponse lazily builds a full +# attaches the coverage so that PRINTING res$response lazily builds a full # pointblank report (one step per column) on demand - the cost is paid only when # the report is displayed, never on the compute path. @@ -86,7 +86,7 @@ test_that("failing report merges the real extract and lists every column", { expect_false(res$all_passed) ag <- build_report_agent(res$coverage, label = "L", lang = "en", - locale = "en_US", real_agent = res$reponse) + locale = "en_US", real_agent = res$response) vs <- ag$validation_set # every coverage column is represented (full "X tests" overview) @@ -116,7 +116,7 @@ test_that("col_exists rows stay PASS even when the column's value check fails", res <- run(ref, cand) expect_false(res$all_passed) ag <- build_report_agent(res$coverage, label = "L", lang = "en", - locale = "en_US", real_agent = res$reponse) + locale = "en_US", real_agent = res$response) vs <- ag$validation_set is_a <- vapply(vs$column, identical, logical(1), "a") @@ -142,7 +142,7 @@ test_that("report counts match coverage for structural checks (augment path)", { expect_false(res$all_passed) ag <- build_report_agent(res$coverage, label = "L", lang = "en", - locale = "en_US", real_agent = res$reponse) + locale = "en_US", real_agent = res$response) vs <- ag$validation_set # the missing_column row must report coverage's counts (1/1), not nrow @@ -185,7 +185,7 @@ test_that("report presents col_exists AND col_vals_equal as distinct checks", { res <- run(mk_ref(), mk_ref()) # all green expect_true(res$all_passed) ag <- build_report_agent(res$coverage, label = "L", lang = "en", - locale = "en_US", real_agent = res$reponse) + locale = "en_US", real_agent = res$response) types <- ag$validation_set$assertion_type # both kinds of check must appear, not collapsed into one expect_true("col_exists" %in% types) @@ -326,47 +326,47 @@ test_that("end-to-end: all-pass comparison HTML report shows row counts (both pa expect_false(has_banner(res_lazy)) }) -# --- API compatibility: res$reponse stays a usable agent -------------------- +# --- API compatibility: res$response stays a usable agent -------------------- -test_that("green res$reponse is a datadiff_report that is still a usable agent", { +test_that("green res$response is a datadiff_report that is still a usable agent", { res <- run(mk_ref(), mk_ref()) expect_true(res$all_passed) - expect_identical(class(res$reponse)[1], "datadiff_report") - expect_true(inherits(res$reponse, "ptblank_agent")) - expect_true(inherits(res$reponse, "has_intel")) - expect_true(pointblank::all_passed(res$reponse)) - expect_length(pointblank::get_data_extracts(res$reponse), 0L) + expect_identical(class(res$response)[1], "datadiff_report") + expect_true(inherits(res$response, "ptblank_agent")) + expect_true(inherits(res$response, "has_intel")) + expect_true(pointblank::all_passed(res$response)) + expect_length(pointblank::get_data_extracts(res$response), 0L) }) -test_that("red res$reponse keeps failing cells extractable", { +test_that("red res$response keeps failing cells extractable", { ref <- mk_ref(); cand <- ref cand$a[2] <- 99 res <- run(ref, cand) expect_false(res$all_passed) - expect_identical(class(res$reponse)[1], "datadiff_report") - expect_true(inherits(res$reponse, "ptblank_agent")) + expect_identical(class(res$response)[1], "datadiff_report") + expect_true(inherits(res$response, "ptblank_agent")) expect_equal(failing_cells(res, key = KEYR), "a@2|1") }) # --- print renders, in green and red ---------------------------------------- -test_that("printing res$reponse renders without error (green and red)", { +test_that("printing res$response renders without error (green and red)", { res_g <- run(mk_ref(), mk_ref()) - expect_output(print(res_g$reponse)) + expect_output(print(res_g$response)) ref <- mk_ref(); cand <- ref; cand$txt[3] <- "Z" res_r <- run(ref, cand) - expect_output(print(res_r$reponse)) + expect_output(print(res_r$response)) }) # --- memoization: second render reuses the cache ---------------------------- test_that("the rendered report is memoized after first print", { res <- run(mk_ref(), mk_ref()) - cache <- attr(res$reponse, "datadiff_render") + cache <- attr(res$response, "datadiff_render") expect_true(is.environment(cache)) expect_null(cache$report) - invisible(capture.output(print(res$reponse))) + invisible(capture.output(print(res$response))) expect_false(is.null(cache$report)) # populated by the first print }) @@ -399,6 +399,6 @@ test_that("lazy comparison also yields a printable datadiff_report", { dplyr::tbl(con, "ref"), dplyr::tbl(con, "cand"), key = KEYR, path = tmp )) expect_true(res$all_passed) - expect_identical(class(res$reponse)[1], "datadiff_report") - expect_output(print(res$reponse)) + expect_identical(class(res$response)[1], "datadiff_report") + expect_output(print(res$response)) }) diff --git a/tests/testthat/test-response-field.R b/tests/testthat/test-response-field.R new file mode 100644 index 0000000..b1d9f5b --- /dev/null +++ b/tests/testthat/test-response-field.R @@ -0,0 +1,76 @@ +# The comparison result exposes the interrogated agent under `response`. +# The historical `reponse` name stays readable for one release through the +# `datadiff_result` accessors, with a deprecation warning, so existing +# pipelines keep working while they migrate. + +make_pair <- function() { + ref <- data.frame(id = 1:3, value = c(1.0, 2.0, 3.0)) + list(ref = ref, cand = ref) +} + +test_that("the result carries the interrogated agent under `response`", { + d <- make_pair() + res <- suppressMessages( + compare_datasets_from_yaml(d$ref, d$cand, key = "id") + ) + expect_s3_class(res, "datadiff_result") + expect_true("response" %in% names(res)) + expect_false("reponse" %in% names(res)) + expect_s3_class(res$response, "ptblank_agent") + expect_s3_class(res$response, "datadiff_report") + expect_true(pointblank::all_passed(res$response)) +}) + +test_that("`$reponse` still resolves, with a deprecation warning", { + rlang::local_options(rlib_warning_verbosity = "verbose") + d <- make_pair() + res <- suppressMessages( + compare_datasets_from_yaml(d$ref, d$cand, key = "id") + ) + expect_warning(via_dollar <- res$reponse, "reponse") + expect_identical(via_dollar, res$response) +}) + +test_that("`[[\"reponse\"]]` still resolves, with a deprecation warning", { + rlang::local_options(rlib_warning_verbosity = "verbose") + d <- make_pair() + res <- suppressMessages( + compare_datasets_from_yaml(d$ref, d$cand, key = "id") + ) + expect_warning(via_brackets <- res[["reponse"]], "reponse") + expect_identical(via_brackets, res$response) +}) + +test_that("the other result fields keep plain list access semantics", { + d <- make_pair() + res <- suppressMessages( + compare_datasets_from_yaml(d$ref, d$cand, key = "id") + ) + expect_identical(res$all_passed, res[["all_passed"]]) + expect_true(res$all_passed) + expect_null(res$does_not_exist) + expect_identical(res[[1L]], res$all_passed) + expect_s3_class(res$coverage, "data.frame") +}) + +test_that("printing the result does not leak the class attribute", { + d <- make_pair() + res <- suppressMessages( + compare_datasets_from_yaml(d$ref, d$cand, key = "id") + ) + printed <- capture.output(print(res)) + expect_false(any(grepl("datadiff_result", printed, fixed = TRUE))) +}) + +test_that("datadiff_report_html() accepts a result saved by an older version", { + d <- make_pair() + res <- suppressMessages( + compare_datasets_from_yaml(d$ref, d$cand, key = "id") + ) + legacy <- list( + all_passed = res$all_passed, + coverage = res$coverage, + reponse = res$response + ) + expect_no_error(report <- datadiff_report_html(legacy)) +}) diff --git a/vignettes/datadiff.Rmd b/vignettes/datadiff.Rmd index d30a3f8..423acdd 100644 --- a/vignettes/datadiff.Rmd +++ b/vignettes/datadiff.Rmd @@ -86,11 +86,15 @@ names(result) | `coverage` | One row per check actually performed (see below) | | `summary` | Aggregate counts derived from `coverage` | | `agent` | The configured pointblank agent (before interrogation) | -| `reponse` | A `datadiff_report`: a real pointblank agent that renders the full report lazily when printed (see below) | +| `response` | A `datadiff_report`: a real pointblank agent that renders the full report lazily when printed (see below) | | `missing_in_candidate` | Columns present in reference but absent from candidate | | `extra_in_candidate` | Columns present in candidate but absent from reference | | `applied_rules` | The effective per-column rules that were applied | +The agent used to be exposed under the French name `reponse`; that alias +still resolves (with a once-per-session deprecation warning) and will be +removed in a future release: switch to `response`. + ### Seeing exactly what was checked `result$coverage` is the first thing to look at: it lists **every check that @@ -118,14 +122,14 @@ result$coverage[result$coverage$status == "FAIL", ] ### The full pointblank report -`result$reponse` stays a genuine pointblank agent - `pointblank::all_passed()` +`result$response` stays a genuine pointblank agent - `pointblank::all_passed()` and `pointblank::get_data_extracts()` work on it as usual. **Printing** it renders the full per-column pointblank report (HTML in interactive sessions). That report is built only when you display it, so a comparison itself stays fast even on very wide tables: ```{r print-report, eval = FALSE} -print(result$reponse) # renders the report on demand +print(result$response) # renders the report on demand datadiff_report_html(result, file = "report.html") # ...or save it to a file ``` @@ -159,13 +163,13 @@ result_fail <- compare_datasets_from_yaml(ref_fail, cand_fail, key = "id") result_fail$all_passed # Rows that failed at least one step -failed_rows <- pointblank::get_sundered_data(result_fail$reponse, type = "fail") +failed_rows <- pointblank::get_sundered_data(result_fail$response, type = "fail") failed_rows ``` The `type = "pass"` variant returns rows that passed all steps. This is useful to understand the scope of the problem before investigating further. -Note: on an all-pass LAZY comparison, `reponse` carries a constant-size +Note: on an all-pass LAZY comparison, `response` carries a constant-size placeholder rather than the source rows (the verdict is computed by SQL aggregates without loading the data into R), so sundering is only meaningful on a failing comparison. @@ -178,7 +182,7 @@ threshold that was applied. These are computed for the failing columns only, so wide all-pass comparisons pay no extra cost. ```{r failing-extracts} -extracts <- pointblank::get_data_extracts(result_fail$reponse) +extracts <- pointblank::get_data_extracts(result_fail$response) extracts[[1]] ``` @@ -188,7 +192,9 @@ On the in-memory (data.frame) path, the extracts of a failing tolerance column also carry the measured deviation and the applied threshold (`__absdiff`, `__thresh`). The lazy path never had these diagnostics: its agent works on a slim boolean table and its extracts only -show the boolean columns. +show the boolean columns - no key columns and no data values. To inspect the +failing rows of a lazy comparison with their keys and values, re-run the +comparison locally on the failing subset, or query the source directly. ### Controlling how many failing rows are extracted @@ -890,7 +896,7 @@ number of failing columns, not the total. This optimisation is invisible to you: `all_passed`, `coverage`, and the extracted failing cells are exactly what a full per-column run would produce. The only thing deferred is the **rendered** report, which is built on demand -when you print `result$reponse` (section 2). +when you print `result$response` (section 2). ### Lazy tables (dbplyr) From 0e23a324341b53d49c33e38023cb60ea037cdcb5 Mon Sep 17 00:00:00 2001 From: vincent Date: Sun, 12 Jul 2026 10:05:06 +0200 Subject: [PATCH 37/40] fix: retirer le message de debug "key is missing" du chemin positionnel Vestige emis sur chaque comparaison positionnelle par defaut ; la sortie utilisateur du cas nominal redevient silencieuse. Aucun test n'en dependait. --- R/compare_datasets_from_yaml.R | 2 -- tests/testthat/test-edge-cases.R | 13 +++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 41503e0..1ec7c97 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -549,8 +549,6 @@ compare_datasets_from_yaml <- function(data_reference, } } - if (is.null(key)) {message("key is missing")} - # Check for duplicate keys (only if key exists in both datasets) if (!is.null(key) && all(key %in% get_col_names(data_reference)) && all(key %in% get_col_names(data_candidate))) { # Detect duplicate key values. Local data.frames use a fast diff --git a/tests/testthat/test-edge-cases.R b/tests/testthat/test-edge-cases.R index cc984eb..b8e3a77 100644 --- a/tests/testthat/test-edge-cases.R +++ b/tests/testthat/test-edge-cases.R @@ -792,8 +792,8 @@ test_that("no crash and no spurious warning when all column types match", { }) # Helper for the positional-path tests: run expr, return the error message -# and every message emitted (the "key is missing" design note is expected; -# the error text must never leak on the message stream). +# and every message emitted (the error text must never leak on the message +# stream). catch_error_and_messages <- function(expr) { msgs <- character(0) err <- tryCatch( @@ -811,6 +811,15 @@ catch_error_and_messages <- function(expr) { list(error = err, messages = msgs) } +test_that("positional comparison emits no debug message about the missing key", { + ref <- data.frame(value = c(1.0, 2.0, 3.0)) + cand <- ref + expect_no_message( + compare_datasets_from_yaml(ref, cand), + message = "key is missing" + ) +}) + test_that("positional comparison with unequal row counts raises a clear error", { ref <- data.frame(value = 1:5) cand <- data.frame(value = 1:3) From d30d5325c2985c34ff33233709d94fa4b6da90aa Mon Sep 17 00:00:00 2001 From: vincent Date: Sun, 12 Jul 2026 10:05:06 +0200 Subject: [PATCH 38/40] test: executer la grille d'equivalence sur les deux backends run_compare() rejoue chaque cas de la grille sur DuckDB lazy et exige l'accord sur le verdict, les colonnes en echec et le nombre de lignes en echec par colonne. Le pinning par cellule (tuples de cle) reste local-only : les extraits lazy ne portent que les colonnes booleennes, sans cle ni valeur (etat identique sur CRAN 0.5.0, documente dans la vignette). Lookup des steps par id (vs$i) plutot que par position, et nettoyage du YAML temporaire en sortie de run_compare(). --- tests/testthat/helper-equivalence.R | 74 +++++++++++++++++++++-- tests/testthat/test-equivalence-guard.R | 8 ++- tests/testthat/test-yaml-arg-precedence.R | 8 +-- 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/tests/testthat/helper-equivalence.R b/tests/testthat/helper-equivalence.R index 58dc798..4f267e6 100644 --- a/tests/testthat/helper-equivalence.R +++ b/tests/testthat/helper-equivalence.R @@ -12,10 +12,36 @@ underlying_col <- function(col) { sub("__(ok|eq)$", "", col) } +# Named integer vector: for every failing validation step, the underlying +# data column mapped to its number of failing rows. This is the shape both +# backends can promise: lazy extracts carry the boolean check columns only +# (no key columns, no data values), so per-cell identification stays a +# local-path capability. +failing_row_counts <- function(res) { + rep <- res$response + if (is.null(rep)) { + return(integer(0)) + } + ex <- pointblank::get_data_extracts(rep) + if (length(ex) == 0) { + return(integer(0)) + } + vs <- rep$validation_set + cols <- vapply(X = names(ex), FUN = function(nm) { + i <- as.integer(nm) + underlying_col(vs$column[[match(i, vs$i)]][1]) + }, FUN.VALUE = character(1)) + counts <- vapply(X = ex, FUN = function(df) { + nrow(as.data.frame(df)) + }, FUN.VALUE = integer(1)) + names(counts) <- unname(cols) + counts[order(names(counts))] +} + # Sorted character vector of "@" for every failing cell # surfaced by get_data_extracts(). failing_cells <- function(res, key) { - rep <- res$reponse + rep <- res$response if (is.null(rep)) { return(character(0)) } @@ -27,7 +53,7 @@ failing_cells <- function(res, key) { out <- character(0) for (nm in names(ex)) { i <- as.integer(nm) - col <- underlying_col(vs$column[[i]]) + col <- underlying_col(vs$column[[match(i, vs$i)]][1]) df <- as.data.frame(ex[[nm]]) if (nrow(df) == 0) { next @@ -41,10 +67,16 @@ failing_cells <- function(res, key) { } # Run a comparison with explicit tolerances, returning the captured verdict. +# The same comparison also runs on the lazy backend (DuckDB) and both backends +# must agree on the verdict, the exact failing cells and the column +# bookkeeping; set compare_lazy = FALSE for a case that only makes sense +# locally. The returned verdict is the local one. run_compare <- function(ref, cand, key, numeric_abs = 0.101, integer_abs = 0L, - na_equal_default = TRUE, check_count_default = TRUE) { + na_equal_default = TRUE, check_count_default = TRUE, + compare_lazy = TRUE) { tmp <- tempfile(fileext = ".yml") + on.exit(unlink(tmp), add = TRUE) write_rules_template( ref, key = key, @@ -57,11 +89,45 @@ run_compare <- function(ref, cand, key, res <- suppressWarnings(suppressMessages( compare_datasets_from_yaml(ref, cand, key = key, path = tmp) )) - list( + out <- list( all_passed = res$all_passed, cells = failing_cells(res, key = key), missing = res$missing_in_candidate, extra = res$extra_in_candidate, res = res ) + if (isTRUE(compare_lazy) && + requireNamespace("DBI", quietly = TRUE) && + requireNamespace("duckdb", quietly = TRUE) && + requireNamespace("dbplyr", quietly = TRUE)) { + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + duckdb::duckdb_register(con, "ref_equiv", ref) + duckdb::duckdb_register(con, "cand_equiv", cand) + res_lazy <- suppressWarnings(suppressMessages( + compare_datasets_from_yaml( + dplyr::tbl(con, "ref_equiv"), + dplyr::tbl(con, "cand_equiv"), + key = key, + path = tmp + ) + )) + testthat::expect_identical( + res_lazy$all_passed, out$all_passed, + info = "lazy backend must agree on the verdict" + ) + testthat::expect_identical( + failing_row_counts(res_lazy), failing_row_counts(res), + info = "lazy backend must agree on the failing columns and their failing-row counts" + ) + testthat::expect_identical( + res_lazy$missing_in_candidate, out$missing, + info = "lazy backend must agree on the missing columns" + ) + testthat::expect_identical( + res_lazy$extra_in_candidate, out$extra, + info = "lazy backend must agree on the extra columns" + ) + } + out } diff --git a/tests/testthat/test-equivalence-guard.R b/tests/testthat/test-equivalence-guard.R index 4a80f2f..5b4fd9b 100644 --- a/tests/testthat/test-equivalence-guard.R +++ b/tests/testthat/test-equivalence-guard.R @@ -3,8 +3,12 @@ # These tests pin BOTH the verdict (all_passed) AND the exact set of failing # cells (as recovered through pointblank::get_data_extracts, the way the # enc.mco oracle consumes the result). Optimizations may change HOW the -# verdict is produced, never WHAT it produces; local and lazy backends must -# agree. A documented semantic fix (with a NEWS "Breaking changes" entry) MAY +# verdict is produced, never WHAT it produces. Every case also runs on the +# lazy backend (see run_compare in helper-equivalence.R): both backends must +# agree on the verdict, the failing columns and their failing-row counts. +# The per-cell pinning (key tuples) is local-only, because lazy extracts +# carry the boolean check columns without the key columns or data values. +# A documented semantic fix (with a NEWS "Breaking changes" entry) MAY # re-pin a verdict here, deliberately and never silently. KEY <- c("id", ".row") diff --git a/tests/testthat/test-yaml-arg-precedence.R b/tests/testthat/test-yaml-arg-precedence.R index 41d7a5d..669db65 100644 --- a/tests/testthat/test-yaml-arg-precedence.R +++ b/tests/testthat/test-yaml-arg-precedence.R @@ -1,4 +1,4 @@ -# Precedence between explicit arguments and YAML rules (issue #20): +# Precedence between explicit arguments and YAML rules: # explicit argument > YAML defaults > built-in default. test_that("explicit label argument wins over the YAML label", { @@ -12,16 +12,16 @@ test_that("explicit label argument wins over the YAML label", { ref, ref, key = "id", path = yaml_path, label = "explicit label" ) - expect_identical(res$reponse$label, "explicit label") + expect_identical(res$response$label, "explicit label") # Without an explicit argument, the YAML label applies res_yaml <- compare_datasets_from_yaml(ref, ref, key = "id", path = yaml_path) - expect_identical(res_yaml$reponse$label, "yaml label") + expect_identical(res_yaml$response$label, "yaml label") # An empty string means "no explicit label": the YAML label still applies res_empty <- compare_datasets_from_yaml(ref, ref, key = "id", path = yaml_path, label = "") - expect_identical(res_empty$reponse$label, "yaml label") + expect_identical(res_empty$response$label, "yaml label") }) test_that("YAML 'keys' field is read without $ partial matching", { From b314d1ed889783a9fa9b442df3e08d91a8ca97aa Mon Sep 17 00:00:00 2001 From: vincent Date: Sun, 12 Jul 2026 10:05:07 +0200 Subject: [PATCH 39/40] chore(release): version 0.6.0, lignee NEWS reconciliee avec CRAN 0.5.0 Le CRAN a recu un 0.5.0 (consolidation, publie le 2026-06-18) sans bump dans le depot. Le NEWS reinsere la section 0.5.0, cale la baseline des breaking changes sur 0.5.0 et reclasse le retrait de l'export %||% en breaking change. Version 0.6.0 : superieure au CRAN et bump mineur du fait des retraits d'API (reponse, %||%). --- DESCRIPTION | 2 +- NEWS.md | 31 ++++++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 006a831..d89754d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: datadiff Title: Data Validation Based on 'YAML' Rules -Version: 0.4.9.9000 +Version: 0.6.0 Authors@R: c( person("Vincent", "Guyader", , "vincent@thinkr.fr", role = c("cre", "aut"), comment = c(ORCID = "0000-0003-0671-9270")), diff --git a/NEWS.md b/NEWS.md index 8c86fc3..1408064 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -# datadiff (development version) +# datadiff 0.6.0 ## New features @@ -17,7 +17,7 @@ columns' booleans are collected for the pointblank agent. A green lazy comparison previously collected N x columns logicals (~2 GB of R memory for 4M rows x 125 columns, 32x the figure the code comment promised); it - now keeps O(columns) in R, and `res$reponse` carries a constant-size + now keeps O(columns) in R, and `res$response` carries a constant-size placeholder instead of N collected rows (issue #22). * Four avoidable scans and transfers are gone from the lazy path (issue #24): @@ -128,7 +128,7 @@ * Doc-vs-code drift resolved (issue #32): `@return` of `compare_datasets_from_yaml()` now lists all 8 elements including `all_passed` (the first one) and describes `agent` truthfully (configured, - NOT interrogated - `reponse` is); the README Quick Start shows the + NOT interrogated - `response` is); the README Quick Start shows the zero-configuration mode first and passes `key` explicitly everywhere; the README "Dev part" no longer embeds check/coverage outputs that go stale (it froze results from 0.4.2); the vignette documents the local-only @@ -254,6 +254,24 @@ ## Breaking changes +The baseline for these entries is datadiff 0.5.0, the version on CRAN +(functionally identical to 0.4.9: 0.5.0 was a consolidation release with no +behaviour change). + +* The comparison result now exposes the interrogated agent under `response`; + the historical French name `reponse` is deprecated. The result gains the + class `datadiff_result`, whose `$` and `[[` accessors keep `reponse` + readable (with a once-per-session warning) until its removal in a future + release. Code that iterates over `names(result)` or tests + `"reponse" %in% names(result)` must switch to `response` now; + `datadiff_report_html()` still accepts results saved by older versions. + +* `%||%` is no longer exported (its export, accidental since 0.4.5, masked + the operator from base R >= 4.4 and {rlang} at load time). Code calling + `datadiff::%||%` explicitly must switch to the base R or {rlang} operator; + code that merely had {datadiff} attached keeps working. See the + corresponding entry under "Bug fixes" (issue #26). + * The local (data.frame) path now applies the same NA semantics as the lazy path and the numeric tolerance kernel to **equality** columns: a one-sided NA (a value facing a missing value, including candidate rows with no @@ -280,6 +298,13 @@ to a keyless cross join (deprecated dplyr behavior producing a cartesian product). +# datadiff 0.5.0 + +* Release consolidating the 0.4.x maintenance series (wide-table performance, + lazy / Arrow / Parquet support, accurate HTML reports and IEEE 754 tolerance + handling). No user-facing behaviour changes since 0.4.9; see the entries + below for the details. + # datadiff 0.4.9 ## Bug fixes From bb0fe6a0b16bf5971bb7b27f3f5a435aca2c45b7 Mon Sep 17 00:00:00 2001 From: vincent Date: Sun, 12 Jul 2026 10:05:07 +0200 Subject: [PATCH 40/40] chore(dev): harnais de bench CRAN vs branche (hors tarball via .Rbuildignore) --- dev/bench-0.4.10/bench_one.R | 113 ++++++++++++++++++++++++++++++++++ dev/bench-0.4.10/gen_data.R | 62 +++++++++++++++++++ dev/bench-0.4.10/run_bench.sh | 27 ++++++++ 3 files changed, 202 insertions(+) create mode 100644 dev/bench-0.4.10/bench_one.R create mode 100644 dev/bench-0.4.10/gen_data.R create mode 100644 dev/bench-0.4.10/run_bench.sh diff --git a/dev/bench-0.4.10/bench_one.R b/dev/bench-0.4.10/bench_one.R new file mode 100644 index 0000000..16bfb36 --- /dev/null +++ b/dev/bench-0.4.10/bench_one.R @@ -0,0 +1,113 @@ +# Execute UN scenario du banc dans un processus R frais et ecrit une ligne +# JSON (temps, verdict, memoire R) dans le fichier de sortie. +# Usage : Rscript bench_one.R +# Scenarios : local_green | local_na | local_fail | lazy_green | lazy_fail + +args <- commandArgs(trailingOnly = TRUE) +libdir <- args[[1]] +scenario <- args[[2]] +out_file <- args[[3]] +data_dir <- args[[4]] + +.libPaths(new = c(libdir, .libPaths())) +suppressPackageStartupMessages({ + library(datadiff, lib.loc = libdir) + library(dplyr) +}) +ver <- as.character(utils::packageVersion("datadiff")) + +n_rows_local <- 200000L +n_cols_local <- 300L + +make_local_ref <- function() { + set.seed(4242) + m <- matrix( + data = rnorm(n_rows_local * n_cols_local), + nrow = n_rows_local, + ncol = n_cols_local + ) + df <- as.data.frame(m) + names(df) <- sprintf("c%03d", seq_len(n_cols_local)) + df +} + +result <- NULL +con <- NULL + +if (scenario %in% c("local_green", "local_na", "local_fail")) { + ref <- make_local_ref() + cand <- ref + if (scenario == "local_na") { + # NA (sans Inf) sur les 150 premieres colonnes, ~5 % des cellules, + # aux memes positions des deux cotes : cible le fast-path #23. + set.seed(777) + for (j in seq_len(150)) { + idx <- sample.int(n_rows_local, size = n_rows_local %/% 20) + ref[[j]][idx] <- NA_real_ + } + cand <- ref + } + if (scenario == "local_fail") { + set.seed(777) + for (j in seq_len(5)) { + idx <- sample.int(n_rows_local, size = 1000) + cand[[j]][idx] <- cand[[j]][idx] + 1 + } + } + + gc(reset = TRUE) + tm <- system.time({ + res <- compare_datasets_from_yaml( + data_reference = ref, + data_candidate = cand + ) + }) +} else { + suppressPackageStartupMessages(library(duckdb)) + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + ref_path <- gsub("'", "''", file.path(data_dir, "ref.parquet")) + cand_file <- if (scenario == "lazy_green") { + "ref.parquet" + } else { + "cand_fail.parquet" + } + cand_path <- gsub("'", "''", file.path(data_dir, cand_file)) + DBI::dbExecute(con, sprintf( + "CREATE VIEW ref_v AS SELECT * FROM read_parquet('%s')", ref_path + )) + DBI::dbExecute(con, sprintf( + "CREATE VIEW cand_v AS SELECT * FROM read_parquet('%s')", cand_path + )) + ref <- tbl(con, "ref_v") + cand <- tbl(con, "cand_v") + + gc(reset = TRUE) + tm <- system.time({ + res <- compare_datasets_from_yaml( + data_reference = ref, + data_candidate = cand, + key = "id" + ) + }) +} + +g <- gc() +mem_mb <- sum(g[, "max used"] * c(56, 8)) / 1e6 + +all_passed <- tryCatch(isTRUE(res$all_passed), error = function(e) { + NA +}) + +line <- sprintf( + paste0( + '{"version":"%s","lib":"%s","scenario":"%s",', + '"elapsed":%.3f,"user":%.3f,"sys":%.3f,', + '"all_passed":%s,"r_peak_mb":%.0f}' + ), + ver, basename(libdir), scenario, + tm[["elapsed"]], tm[["user.self"]], tm[["sys.self"]], + tolower(as.character(all_passed)), mem_mb +) +cat(line, "\n", sep = "", file = out_file, append = TRUE) +cat(line, "\n", sep = "") diff --git a/dev/bench-0.4.10/gen_data.R b/dev/bench-0.4.10/gen_data.R new file mode 100644 index 0000000..45b6b0c --- /dev/null +++ b/dev/bench-0.4.10/gen_data.R @@ -0,0 +1,62 @@ +# Genere les fichiers Parquet des scenarios lazy du banc CRAN 0.5.0 vs PR #58. +# Usage : Rscript gen_data.R +# Tables : 4M lignes x 125 colonnes (id + 74 DOUBLE + 30 INTEGER + 20 VARCHAR), +# le cas cite dans la PR (#22). La table candidate "fail" differe de la +# reference sur 5 colonnes, 40 lignes chacune (id %% 100000 == 7). + +args <- commandArgs(trailingOnly = TRUE) +data_dir <- args[[1]] +if (!dir.exists(data_dir)) { + dir.create(data_dir, recursive = TRUE) +} + +suppressPackageStartupMessages({ + library(duckdb) +}) + +con <- DBI::dbConnect(duckdb::duckdb()) +on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + +n_rows <- 4e6 + +dbl_cols <- sprintf("round(random() * 1000, 4)::DOUBLE AS d%03d", 1:74) +int_cols <- sprintf("(random() * 100000)::INTEGER AS i%03d", 1:30) +chr_cols <- sprintf( + "('code_' || ((range * %d) %% 5000)::VARCHAR) AS s%03d", + seq(7, by = 2, length.out = 20), + 1:20 +) + +sql_ref <- sprintf( + "CREATE TABLE ref AS SELECT range AS id, %s FROM range(%d)", + paste(c(dbl_cols, int_cols, chr_cols), collapse = ", "), + n_rows +) + +message("Generation table reference 4M x 125...") +DBI::dbExecute(con, sql_ref) + +ref_path <- file.path(data_dir, "ref.parquet") +DBI::dbExecute( + con, + sprintf("COPY ref TO '%s' (FORMAT PARQUET)", gsub("'", "''", ref_path)) +) + +# Candidate "fail" : 5 colonnes touchees sur les lignes id %% 100000 == 7 +# (40 lignes par colonne, 200 cellules en echec au total). +message("Generation table candidate (fail)...") +cand_path <- file.path(data_dir, "cand_fail.parquet") +DBI::dbExecute(con, sprintf( + paste( + "COPY (SELECT * REPLACE (", + " CASE WHEN id %% 100000 = 7 THEN d001 + 1 ELSE d001 END AS d001,", + " CASE WHEN id %% 100000 = 7 THEN d002 + 1 ELSE d002 END AS d002,", + " CASE WHEN id %% 100000 = 7 THEN i001 + 1 ELSE i001 END AS i001,", + " CASE WHEN id %% 100000 = 7 THEN s001 || '_X' ELSE s001 END AS s001,", + " CASE WHEN id %% 100000 = 7 THEN s002 || '_X' ELSE s002 END AS s002", + ") FROM ref) TO '%s' (FORMAT PARQUET)" + ), + gsub("'", "''", cand_path) +)) + +message("OK : ", ref_path, " et ", cand_path) diff --git a/dev/bench-0.4.10/run_bench.sh b/dev/bench-0.4.10/run_bench.sh new file mode 100644 index 0000000..4ae3493 --- /dev/null +++ b/dev/bench-0.4.10/run_bench.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Driver du banc CRAN 0.5.0 vs PR #58. Chaque mesure dans un processus R +# frais, versions alternees a chaque repetition pour lisser la derive machine. +# Usage : run_bench.sh +set -eu +SP="$1" +OUT="$2" +BENCH="$SP/datadiff-pr/dev/bench-0.4.10/bench_one.R" +DATA="$SP/benchdata" + +for rep in 1 2 3; do + for scen in local_green local_na local_fail; do + for lib in lib_cran lib_pr; do + echo "== rep $rep $scen $lib ==" + Rscript "$BENCH" "$SP/$lib" "$scen" "$OUT" "$DATA" 2>&1 | tail -1 + done + done +done + +for rep in 1 2; do + for scen in lazy_green lazy_fail; do + for lib in lib_cran lib_pr; do + echo "== rep $rep $scen $lib ==" + Rscript "$BENCH" "$SP/$lib" "$scen" "$OUT" "$DATA" 2>&1 | tail -1 + done + done +done