diff --git a/DESCRIPTION b/DESCRIPTION index f8fa1f1..006a831 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,7 +18,6 @@ Imports: dplyr (>= 1.0.0), pointblank (>= 0.12.3), rlang (>= 0.4.0), - stats, tidyselect (>= 1.0.0), utils, yaml (>= 2.2.0) diff --git a/NAMESPACE b/NAMESPACE index 9c201c6..1d82dd0 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -15,8 +15,6 @@ export(setup_pointblank_agent) export(validate_row_counts) export(write_rules_template) importFrom(dplyr,"%>%") -importFrom(dplyr,across) -importFrom(dplyr,arrange) importFrom(dplyr,collect) importFrom(dplyr,left_join) importFrom(pointblank,action_levels) @@ -26,7 +24,6 @@ importFrom(pointblank,create_agent) importFrom(pointblank,interrogate) importFrom(rlang,":=") importFrom(rlang,.data) -importFrom(stats,setNames) importFrom(tidyselect,all_of) importFrom(utils,modifyList) importFrom(yaml,read_yaml) diff --git a/NEWS.md b/NEWS.md index f2085af..7fa53e5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -86,6 +86,26 @@ projection (`paste0(character(0), "__ok")` yields `"__ok"`) that `any_of()` had been eating since 0.4.8 (issue #17). +## Chores + +* Packaging and code cleanup (issue #34): `dev/` is excluded from the tarball + (`.Rbuildignore`); the orphan `inst/templates/rules_template.yaml` + (referenced nowhere) is gone; unused `@importFrom` entries dropped + (`dplyr::arrange`/`across`, `stats::setNames` replaced by a base + construction); the `numeric_abs` default reads `1e-9` instead of a + 9-zero decimal literal; `is_non_local()` reuses `is_arrow()` instead of + duplicating the Arrow class list; `abs(ref_vals)` and the IEEE fp constant + are hoisted out of the kernels' expressions and loops; + `tol_col_counts()` counts failures without allocating intermediate + vectors; `format_key_examples()` truncates to 3 groups before formatting; + the lazy `preprocess_dataframe()` composes ONE `mutate()` for all + normalized columns instead of one or two layers per column (the dbplyr + query-construction anti-pattern eliminated elsewhere in 0.4.8); loop + variables no longer shadow `base::c`; the tolerance kernel documents the + integer-overflow caveat. Deliberately not changed: `print()` of a report + builds the gt object in non-interactive sessions too (printing IS the + render), and the `DBI` availability guard stays transitive via `duckdb`. + ## Documentation and input friction * Doc-vs-code drift resolved (issue #32): `@return` of diff --git a/R/compare_datasets_from_yaml.R b/R/compare_datasets_from_yaml.R index 14727d6..0954a26 100644 --- a/R/compare_datasets_from_yaml.R +++ b/R/compare_datasets_from_yaml.R @@ -33,7 +33,6 @@ #' (same caveat as `date_equal_mode`) #' @return The \code{path} to the written YAML file, returned invisibly. #' @importFrom yaml write_yaml -#' @importFrom stats setNames #' @importFrom dplyr collect #' @export #' @examples @@ -43,7 +42,7 @@ write_rules_template <- function(data_reference, key = NULL, label = NULL, path = "rules.yaml", version = 1L, na_equal_default = TRUE, ignore_columns_default = character(0), check_count_default = TRUE, expected_count_default = NULL, row_count_tolerance_default = 0, - numeric_abs = 0.000000001, numeric_rel = 0, + numeric_abs = 1e-9, numeric_rel = 0, integer_abs = 0L, character_equal_mode = "exact", character_case_insensitive = FALSE, character_trim = FALSE, date_equal_mode = "exact", @@ -108,7 +107,7 @@ write_rules_template <- function(data_reference, datetime = list(equal_mode = datetime_equal_mode), logical = list(equal_mode = logical_equal_mode) ), - by_name = setNames(replicate(length(types), list(), simplify = FALSE), names(types)) + by_name = as.list(structure(replicate(length(types), list(), simplify = FALSE), names = names(types))) ) write_yaml(x = y, file = path) invisible(path) @@ -363,7 +362,7 @@ validate_comparison_key <- function(key, ref_cols, cand_cols) { #' the fast path skips the per-column agent.} #' \item{summary}{Aggregate counts from \code{coverage} (n_checks, n_pass, #' n_fail, n_rows_failed_total, all_passed).} -#' @importFrom dplyr arrange across left_join %>% +#' @importFrom dplyr left_join %>% #' @importFrom pointblank interrogate #' @importFrom dplyr collect #' @export @@ -874,9 +873,9 @@ compare_datasets_from_yaml <- function(data_reference, # already carries __eq columns; col_vals_equal(na_pass = ...) alone cannot # express these semantics. if (!is_lazy && length(fail$eq) > 0) { - for (c in fail$eq) { - cmp_for_agent[[datadiff_eq_col(c)]] <- eq_col_bool( - cmp_for_agent, col = c, ref_suffix = ref_suffix, na_equal = na_equal + for (col_nm in fail$eq) { + cmp_for_agent[[datadiff_eq_col(col_nm)]] <- eq_col_bool( + cmp_for_agent, col = col_nm, ref_suffix = ref_suffix, na_equal = na_equal ) } } diff --git a/R/coverage.R b/R/coverage.R index 0807d09..5bfbfd9 100644 --- a/R/coverage.R +++ b/R/coverage.R @@ -9,7 +9,9 @@ # matching col_vals_equal(..., na_pass = FALSE). tol_col_counts <- function(tbl, col) { ok <- tol_col_bool(tbl, col = col) - list(n = length(ok), n_failed = sum(is.na(ok) | !ok)) + # length - sum(TRUE) counts FALSE and NA in one pass without allocating + # the two intermediate logical vectors of is.na(ok) | !ok + list(n = length(ok), n_failed = length(ok) - sum(ok, na.rm = TRUE)) } # n / n_failed for an equality column (resolved boolean, no NA). @@ -104,23 +106,23 @@ build_coverage <- function(tbl, tol_cols, eq_cols, # Existence checks: every common (non-type-mismatch) column gets a col_exists # check, distinct from its value check. These pass (the column is present in # both datasets by construction), mirroring a full per-column pointblank run. - for (c in c(tol_cols, eq_cols)) { - add(c, "col_exists", 1L, 0L) + for (col_nm in c(tol_cols, eq_cols)) { + add(col_nm, "col_exists", 1L, 0L) } - for (c in tol_cols) { - cnt <- counts[[c]] %||% tol_col_counts(tbl, col = c) - add(c, "tolerance", cnt$n, cnt$n_failed) + for (col_nm in tol_cols) { + cnt <- counts[[col_nm]] %||% tol_col_counts(tbl, col = col_nm) + add(col_nm, "tolerance", cnt$n, cnt$n_failed) } - for (c in eq_cols) { - cnt <- counts[[c]] %||% - eq_col_counts(tbl, col = c, ref_suffix = ref_suffix, na_equal = na_equal) - add(c, "equality", cnt$n, cnt$n_failed) + for (col_nm in eq_cols) { + cnt <- counts[[col_nm]] %||% + eq_col_counts(tbl, col = col_nm, ref_suffix = ref_suffix, na_equal = na_equal) + add(col_nm, "equality", cnt$n, cnt$n_failed) } - for (c in missing_in_candidate) { - add(c, "missing_column", 1L, 1L) + for (col_nm in missing_in_candidate) { + add(col_nm, "missing_column", 1L, 1L) } - for (c in type_mismatch_cols) { - add(c, "type_mismatch", 1L, 1L) + for (col_nm in type_mismatch_cols) { + add(col_nm, "type_mismatch", 1L, 1L) } if (isTRUE(row_validation_info$check_count)) { add("", "row_count", 1L, if (isTRUE(row_count_ok)) 0L else 1L) diff --git a/R/duplicate_keys.R b/R/duplicate_keys.R index 02f9708..7068315 100644 --- a/R/duplicate_keys.R +++ b/R/duplicate_keys.R @@ -10,14 +10,15 @@ # tables keep the SQL-native count()/group_by (cheap inside the database). format_key_examples <- function(uniq_keys, key) { - rows <- unname(apply(uniq_keys, 1, function(r) { + n_groups <- nrow(uniq_keys) + shown <- utils::head(uniq_keys, 3L) + rows <- unname(apply(shown, 1, function(r) { paste(key, "=", r, collapse = ", ") })) - if (length(rows) <= 3) { - rows - } else { - c(rows[1:3], "...") + if (n_groups > 3L) { + rows <- c(rows, "...") } + rows } #' @importFrom rlang .data diff --git a/R/pointblank_setup.R b/R/pointblank_setup.R index 2173113..7378450 100644 --- a/R/pointblank_setup.R +++ b/R/pointblank_setup.R @@ -71,8 +71,8 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ # Add dummy columns for missing columns BEFORE creating the agent # These columns are set to FALSE and we'll check they equal TRUE (will fail) - for (c in missing_in_candidate) { - dummy_col <- paste0(datadiff_prefix_missing_col, c) + for (col_nm in missing_in_candidate) { + dummy_col <- paste0(datadiff_prefix_missing_col, col_nm) if (is_non_local(cmp)) { cmp <- dplyr::mutate(cmp, !!dummy_col := FALSE) } else { @@ -82,8 +82,8 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ # Add dummy FALSE columns for type-mismatched columns. # These will generate a dedicated failing validation step per column. - for (c in type_mismatch_cols) { - dummy_col <- paste0(datadiff_prefix_type_mismatch, c) + for (col_nm in type_mismatch_cols) { + dummy_col <- paste0(datadiff_prefix_type_mismatch, col_nm) if (is_non_local(cmp)) { cmp <- dplyr::mutate(cmp, !!dummy_col := FALSE) } else { @@ -98,11 +98,11 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ # express the one-sided/two-sided NA distinction). eq_cols <- setdiff(x = common_cols, y = tol_cols) if (!is_non_local(cmp)) { - for (c in eq_cols) { - eq_col <- datadiff_eq_col(c) + for (col_nm in eq_cols) { + eq_col <- datadiff_eq_col(col_nm) if (!(eq_col %in% get_col_names(cmp))) { cmp[[eq_col]] <- eq_col_bool( - cmp, col = c, ref_suffix = ref_suffix, na_equal = na_equal + cmp, col = col_nm, ref_suffix = ref_suffix, na_equal = na_equal ) } } @@ -120,37 +120,37 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ # Skipped for the non-local path where cmp is a slim table containing only # boolean validation columns (the original data columns are not present). if (add_col_exists_steps) { - for (c in common_cols) { - agent <- agent %>% col_exists(columns = all_of(c)) + for (col_nm in common_cols) { + agent <- agent %>% col_exists(columns = all_of(col_nm)) } } # For missing columns, add a validation that will fail - for (c in missing_in_candidate) { - dummy_col <- paste0(datadiff_prefix_missing_col, c) + for (col_nm in missing_in_candidate) { + dummy_col <- paste0(datadiff_prefix_missing_col, col_nm) agent <- agent %>% col_vals_equal( columns = all_of(dummy_col), value = TRUE, na_pass = FALSE, - label = paste("col_exists:", c) + label = paste("col_exists:", col_nm) ) } # For type-mismatched columns, add a validation that will always fail. - for (c in type_mismatch_cols) { - dummy_col <- paste0(datadiff_prefix_type_mismatch, c) + for (col_nm in type_mismatch_cols) { + dummy_col <- paste0(datadiff_prefix_type_mismatch, col_nm) agent <- agent %>% col_vals_equal( columns = all_of(dummy_col), value = TRUE, na_pass = FALSE, - label = paste("type_mismatch:", c) + label = paste("type_mismatch:", col_nm) ) } - for (c in eq_cols) { - eq_col <- datadiff_eq_col(c) + for (col_nm in eq_cols) { + eq_col <- datadiff_eq_col(col_nm) if (!(eq_col %in% col_names)) { # Only reachable with a lazy cmp missing its precomputed boolean: # the documented contract requires it (deriving it here would need @@ -164,8 +164,8 @@ setup_pointblank_agent <- function(cmp, cols_reference = NULL, common_cols, tol_ col_vals_equal(columns = all_of(eq_col), value = TRUE, na_pass = FALSE) } - for (c in tol_cols) { - ok_col <- datadiff_ok_col(c) + for (col_nm in tol_cols) { + ok_col <- datadiff_ok_col(col_nm) agent <- agent %>% col_vals_equal(columns = all_of(ok_col), value = TRUE, na_pass = FALSE) } diff --git a/R/preprocessing.R b/R/preprocessing.R index 10e7c54..6a0dbfb 100644 --- a/R/preprocessing.R +++ b/R/preprocessing.R @@ -19,7 +19,7 @@ normalize_text <- function(x, case_insensitive = FALSE, trim = FALSE) { x <- trimws(x) } if (case_insensitive) { - x <- tolower(x = x) + x <- tolower(x) } x } @@ -63,6 +63,7 @@ preprocess_dataframe <- function(df, col_rules, schema = NULL) { } } } + lazy_exprs <- list() for (nm in names(col_rules)) { cr <- col_rules[[nm]] normalized <- identical(cr$equal_mode %||% "exact", "normalized") @@ -93,23 +94,26 @@ preprocess_dataframe <- function(df, col_rules, schema = NULL) { if (is_char && should_normalize) { if (is_non_local(out)) { - # Non-local path: build transformations via dplyr::mutate() - nm_sym <- dplyr::sym(nm) + # Non-local path: compose ONE expression per column and accumulate; + # a single mutate() at the end keeps the dbplyr query-construction + # cost O(1) instead of one or two mutate() layers per column + expr <- dplyr::sym(nm) if (trim) { if (is_arrow(out)) { # Arrow does not support trimws(); use stringr::str_trim() instead if (!requireNamespace("stringr", quietly = TRUE)) { stop("Package 'stringr' is required for trim on Arrow objects.") } - out <- dplyr::mutate(out, !!nm := stringr::str_trim(!!nm_sym)) + expr <- rlang::expr(stringr::str_trim(!!expr)) } else { # SQL path: trimws() -> SQL TRIM() (translated by dbplyr) - out <- dplyr::mutate(out, !!nm := trimws(!!nm_sym)) + expr <- rlang::expr(trimws(!!expr)) } } if (case_insensitive) { - out <- dplyr::mutate(out, !!nm := tolower(!!nm_sym)) + expr <- rlang::expr(tolower(!!expr)) } + lazy_exprs[[nm]] <- expr } else { out[[nm]] <- normalize_text( out[[nm]], @@ -119,5 +123,8 @@ preprocess_dataframe <- function(df, col_rules, schema = NULL) { } } } + if (length(lazy_exprs) > 0) { + out <- dplyr::mutate(out, !!!lazy_exprs) + } out } diff --git a/R/tolerance.R b/R/tolerance.R index d3ab891..edfba55 100644 --- a/R/tolerance.R +++ b/R/tolerance.R @@ -21,6 +21,10 @@ #' @param na_equal Logical; whether NA/NaN on both sides count as equal. #' @return A list with numeric vectors \code{absdiff}, \code{thresh} and the #' logical vector \code{ok}. +#' @note When both compared columns are integer, \code{cand - ref} is +#' integer subtraction: a difference beyond \code{.Machine$integer.max} +#' overflows to NA with a warning (and NA counts as a failure). Cast such +#' columns to double upstream if their differences can approach that range. #' @noRd compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal) { # is.na() covers NaN in R, so the NA masks subsume the NaN ones: NaN-specific @@ -35,9 +39,10 @@ compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equa absdiff <- abs(cand_vals - ref_vals) absdiff[same_inf] <- 0 - thresh <- abs_tol + rel_tol * abs(ref_vals) + abs_ref <- abs(ref_vals) + thresh <- abs_tol + rel_tol * abs_ref - fp_correction <- 8 * .Machine$double.eps * abs(ref_vals) + fp_correction <- 8 * .Machine$double.eps * abs_ref fp_correction[!is.finite(fp_correction)] <- 0 within_tol <- absdiff <= thresh + fp_correction @@ -72,8 +77,9 @@ compute_tolerance_col <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equa compute_tolerance_ok <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal) { if (!anyNA(cand_vals) && !anyNA(ref_vals) && all(is.finite(cand_vals)) && all(is.finite(ref_vals))) { - thresh <- abs_tol + rel_tol * abs(ref_vals) - fp <- 8 * .Machine$double.eps * abs(ref_vals) + abs_ref <- abs(ref_vals) + thresh <- abs_tol + rel_tol * abs_ref + fp <- 8 * .Machine$double.eps * abs_ref return(abs(cand_vals - ref_vals) <= thresh + fp) } if (!any(is.infinite(cand_vals)) && !any(is.infinite(ref_vals))) { @@ -81,8 +87,9 @@ compute_tolerance_ok <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal # NaN in R, so NaN follows the NA rules identically here and in the # kernel. Only Inf diverges (abs(Inf - Inf) is NaN, but same-sign # infinities must PASS), hence the infinity-only fallback test. - thresh <- abs_tol + rel_tol * abs(ref_vals) - fp <- 8 * .Machine$double.eps * abs(ref_vals) + abs_ref <- abs(ref_vals) + thresh <- abs_tol + rel_tol * abs_ref + fp <- 8 * .Machine$double.eps * abs_ref ok <- abs(cand_vals - ref_vals) <= thresh + fp # NA wherever an NA/NaN was involved: a one-sided one is a difference... ok[is.na(ok)] <- FALSE @@ -97,12 +104,12 @@ compute_tolerance_ok <- function(cand_vals, ref_vals, abs_tol, rel_tol, na_equal #' Add only the `__ok` tolerance columns (hot path) #' -#' Like [add_tolerance_columns()] but materialises only the boolean `__ok` +#' Like `add_tolerance_columns()` but materialises only the boolean `__ok` #' columns - which alone determine the verdict - in a single bind. The #' `__absdiff` / `__thresh` diagnostic columns are not produced here: #' no verdict logic reads them, and on the all-pass fast path there are no #' extracts to surface them in. On the failure path they are re-added for the -#' failing columns only by [add_diff_columns()], so the failing-row extracts +#' failing columns only by `add_diff_columns()`, so the failing-row extracts #' keep the explicit measured deviations at a cost proportional to the failing #' columns rather than the table width. #' @@ -115,11 +122,11 @@ add_ok_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { } ok_cols <- vector("list", length(tol_cols)) for (i in seq_along(tol_cols)) { - c <- tol_cols[i] + col_nm <- tol_cols[i] ok_cols[[i]] <- compute_tolerance_ok( - cand_vals = cmp[[c]], ref_vals = cmp[[paste0(c, ref_suffix)]], - abs_tol = col_rules[[c]][["abs"]] %||% 0, - rel_tol = col_rules[[c]][["rel"]] %||% 0, + cand_vals = cmp[[col_nm]], ref_vals = cmp[[paste0(col_nm, ref_suffix)]], + abs_tol = col_rules[[col_nm]][["abs"]] %||% 0, + rel_tol = col_rules[[col_nm]][["rel"]] %||% 0, na_equal = na_equal ) } @@ -134,7 +141,7 @@ add_ok_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { #' failing-row extracts - `pointblank::get_data_extracts()`, the HTML report and #' its CSV download - show the explicit gap next to the candidate and reference #' values, as they did up to 0.4.7. The `__ok` verdict columns are expected -#' to exist already (see [add_ok_columns()]) and are not touched. Cost is +#' to exist already (see `add_ok_columns()`) and are not touched. Cost is #' proportional to the number of columns passed, so the all-pass fast path and #' the passing majority of columns pay nothing. #' @@ -149,11 +156,11 @@ add_diff_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal) { absdiff_cols <- vector("list", m) thresh_cols <- vector("list", m) for (i in seq_len(m)) { - c <- tol_cols[i] + col_nm <- tol_cols[i] blocks <- compute_tolerance_col( - cand_vals = cmp[[c]], ref_vals = cmp[[paste0(c, ref_suffix)]], - abs_tol = col_rules[[c]][["abs"]] %||% 0, - rel_tol = col_rules[[c]][["rel"]] %||% 0, + cand_vals = cmp[[col_nm]], ref_vals = cmp[[paste0(col_nm, ref_suffix)]], + abs_tol = col_rules[[col_nm]][["abs"]] %||% 0, + rel_tol = col_rules[[col_nm]][["rel"]] %||% 0, na_equal = na_equal ) absdiff_cols[[i]] <- blocks$absdiff @@ -261,22 +268,22 @@ add_bool_cols_sql <- function(cmp, tol_cols, eq_cols, col_rules, ref_suffix, ) } - tol_exprs <- vapply(X = tol_cols, FUN = function(c) { - cc <- q(c) - rc <- q(paste0(c, ref_suffix)) - at <- col_rules[[c]][["abs"]] %||% 0 - rt <- col_rules[[c]][["rel"]] %||% 0 + tol_exprs <- vapply(X = tol_cols, FUN = function(col_nm) { + cc <- q(col_nm) + rc <- q(paste0(col_nm, ref_suffix)) + at <- col_rules[[col_nm]][["abs"]] %||% 0 + rt <- col_rules[[col_nm]][["rel"]] %||% 0 within <- sprintf("ABS(%s - %s) <= (%s + %s * ABS(%s)) + %s * ABS(%s)", cc, rc, num(at), num(rt), rc, num(fp_eps), rc) - sprintf("%s AS %s", case_tol(cc, rc, cond = within), q(datadiff_ok_col(c))) + sprintf("%s AS %s", case_tol(cc, rc, cond = within), q(datadiff_ok_col(col_nm))) }, FUN.VALUE = character(1), USE.NAMES = FALSE) - eq_exprs <- vapply(X = eq_cols, FUN = function(c) { - cc <- q(c) - rc <- q(paste0(c, ref_suffix)) + eq_exprs <- vapply(X = eq_cols, FUN = function(col_nm) { + cc <- q(col_nm) + rc <- q(paste0(col_nm, ref_suffix)) sprintf("%s AS %s", - case_eq(cc, rc, nan_aware = c %in% eq_num_cols), - q(datadiff_eq_col(c))) + case_eq(cc, rc, nan_aware = col_nm %in% eq_num_cols), + q(datadiff_eq_col(col_nm))) }, FUN.VALUE = character(1), USE.NAMES = FALSE) exprs <- c(tol_exprs, eq_exprs) @@ -309,32 +316,35 @@ add_tolerance_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal # (one for __absdiff + __thresh, one for __ok) regardless of how many columns # there are. Each individual mutate() adds one lazy_query node to the dbplyr # plan; with 100+ columns the nesting would exceed R's expression stack limit. - if (length(tol_cols) == 0) return(cmp) + if (length(tol_cols) == 0) { + return(cmp) + } + + # IEEE 754: floating-point subtraction introduces rounding errors + # proportional to the magnitude of the operands, not to the threshold. + # e.g. 100.01 - 100.00 = 0.0100000000000051 > 0.01 in double precision. + # Adding a few ULPs of the reference magnitude absorbs this error without + # meaningfully widening the user-specified tolerance. + fp_eps <- 8 * .Machine$double.eps exprs_diff <- list() exprs_ok <- list() - for (c in tol_cols) { - reference_c <- paste0(c, ref_suffix) - abs_tol <- col_rules[[c]][["abs"]] %||% 0 - rel_tol <- col_rules[[c]][["rel"]] %||% 0 - c_sym <- dplyr::sym(c) + for (col_nm in tol_cols) { + reference_c <- paste0(col_nm, ref_suffix) + abs_tol <- col_rules[[col_nm]][["abs"]] %||% 0 + rel_tol <- col_rules[[col_nm]][["rel"]] %||% 0 + c_sym <- dplyr::sym(col_nm) rc_sym <- dplyr::sym(reference_c) - absdiff_col <- paste0(c, "__absdiff") - thresh_col <- paste0(c, "__thresh") - ok_col <- datadiff_ok_col(c) + absdiff_col <- paste0(col_nm, "__absdiff") + thresh_col <- paste0(col_nm, "__thresh") + ok_col <- datadiff_ok_col(col_nm) absdiff_sym <- dplyr::sym(absdiff_col) thresh_sym <- dplyr::sym(thresh_col) exprs_diff[[absdiff_col]] <- rlang::expr(abs(!!c_sym - !!rc_sym)) exprs_diff[[thresh_col]] <- rlang::expr(!!abs_tol + !!rel_tol * abs(!!rc_sym)) - # IEEE 754: floating-point subtraction introduces rounding errors - # proportional to the magnitude of the operands, not to the threshold. - # e.g. 100.01 - 100.00 = 0.0100000000000051 > 0.01 in double precision. - # Adding a few ULPs of the reference magnitude absorbs this error without - # meaningfully widening the user-specified tolerance. - fp_eps <- 8 * .Machine$double.eps if (na_equal) { exprs_ok[[ok_col]] <- rlang::expr(dplyr::case_when( is.na(!!c_sym) & is.na(!!rc_sym) ~ TRUE, @@ -373,11 +383,11 @@ add_tolerance_columns <- function(cmp, tol_cols, col_rules, ref_suffix, na_equal ok_cols <- vector("list", m) for (i in seq_len(m)) { - c <- tol_cols[i] + col_nm <- tol_cols[i] blocks <- compute_tolerance_col( - cand_vals = cmp[[c]], ref_vals = cmp[[paste0(c, ref_suffix)]], - abs_tol = col_rules[[c]][["abs"]] %||% 0, - rel_tol = col_rules[[c]][["rel"]] %||% 0, + cand_vals = cmp[[col_nm]], ref_vals = cmp[[paste0(col_nm, ref_suffix)]], + abs_tol = col_rules[[col_nm]][["abs"]] %||% 0, + rel_tol = col_rules[[col_nm]][["rel"]] %||% 0, na_equal = na_equal ) absdiff_cols[[i]] <- blocks$absdiff diff --git a/R/utils.R b/R/utils.R index f38d9af..8bde131 100644 --- a/R/utils.R +++ b/R/utils.R @@ -13,8 +13,12 @@ get_col_names <- function(x) { if (length(nms) == 0) names(x) else nms } +is_arrow <- function(x) { + inherits(x, c("ArrowObject", "arrow_dplyr_query")) +} + is_non_local <- function(x) { - inherits(x, "tbl_lazy") || inherits(x, c("ArrowObject", "arrow_dplyr_query")) + inherits(x, "tbl_lazy") || is_arrow(x) } # Package-local mutable state (temp-table counter). @@ -29,10 +33,6 @@ datadiff_tmp_table_name <- function() { sprintf("datadiff_tmp_%d_%d", Sys.getpid(), counter) } -is_arrow <- function(x) { - inherits(x, c("ArrowObject", "arrow_dplyr_query")) -} - # Convert an Arrow dataset to a DuckDB tbl_lazy on `con`. # # For file-backed Parquet datasets, uses DuckDB's native read_parquet() so diff --git a/inst/templates/rules_template.yaml b/inst/templates/rules_template.yaml deleted file mode 100644 index 95a28f3..0000000 --- a/inst/templates/rules_template.yaml +++ /dev/null @@ -1,37 +0,0 @@ -version: 1 -defaults: - na_equal: yes - ignore_columns: [] # Columns to ignore during schema comparison - keys: [] - label: ~ -row_validation: - check_count: no - expected_count: ~ - tolerance: 0.0 -by_type: - numeric: - abs: 1.0e-09 - rel: 1.0e-09 - integer: - abs: 0 - character: - equal_mode: exact - case_insensitive: no - trim: no - date: - equal_mode: exact - datetime: - equal_mode: exact - logical: - equal_mode: exact -by_name: - # Column-specific rules override type rules - # Examples: - # amount: - # abs: 0.001 # Absolute tolerance - # rel: 0.001 # Relative tolerance - # category: - # case_insensitive: yes - # trim: yes - # name: - # equal_mode: normalized