From 7204b8259dbea04a0b3401e0d9de6e1f34177604 Mon Sep 17 00:00:00 2001 From: jrt1899 Date: Thu, 7 May 2026 13:26:43 -0700 Subject: [PATCH 1/6] factorization over 2 join tables done --- queries/factorization_demo.sql | 75 +++ src/include/ml_functions/list_add_vv.hpp | 17 + .../actions/MLFactorizationRewriteAction.hpp | 36 ++ src/ml_functions/CMakeLists.txt | 1 + src/ml_functions/list_add_vv.cpp | 100 ++++ src/ml_functions/ml_functions_scalar.cpp | 2 + src/optimization/CMakeLists.txt | 1 + .../actions/MLFactorizationRewriteAction.cpp | 515 ++++++++++++++++++ src/optimization/register_optimizers.cpp | 9 + 9 files changed, 756 insertions(+) create mode 100644 queries/factorization_demo.sql create mode 100644 src/include/ml_functions/list_add_vv.hpp create mode 100644 src/include/optimization/actions/MLFactorizationRewriteAction.hpp create mode 100644 src/ml_functions/list_add_vv.cpp create mode 100644 src/optimization/actions/MLFactorizationRewriteAction.cpp diff --git a/queries/factorization_demo.sql b/queries/factorization_demo.sql new file mode 100644 index 0000000..b78fbb1 --- /dev/null +++ b/queries/factorization_demo.sql @@ -0,0 +1,75 @@ +-- MLFactorizationRewriteAction demo +-- +-- Schema: left_feats has 3-dim embeddings, right_feats has 2-dim embeddings. +-- The join produces a 5-dim feature vector that is multiplied by W (5x2). +-- +-- Optimization (enable_extended_optimizer = '4') splits W into: +-- W_top (3x2) pushed to left_feats before the join +-- W_bottom (2x2) pushed to right_feats before the join +-- and replaces the single mat_mul with list_add_vv of the two partial results. +-- +-- Run: +-- duckdb < queries/factorization_demo.sql + +LOAD cactusdb; + +-- Disable DuckDB's own rewrites so the logical plan stays readable. +SET disabled_optimizers = 'expression_rewriter, empty_result_pullup, + cte_filter_pusher, regex_range, in_clause, join_order, deliminator, + unnest_rewriter, unused_columns, statistics_propagation, + common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, + top_n, build_side_probe_side, compressed_materialization, duplicate_groups, + reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, + sum_rewriter, late_materialization, cte_inlining'; + +-- ----------------------------------------------------------------------- +-- Tables: note DOUBLE[3] and DOUBLE[2] — fixed-size ARRAY types so the +-- optimizer can read k_left = 3 from the type at plan time. +-- ----------------------------------------------------------------------- +CREATE OR REPLACE TABLE left_feats (id INTEGER, features DOUBLE[3]); +CREATE OR REPLACE TABLE right_feats (id INTEGER, features DOUBLE[2]); + +INSERT INTO left_feats VALUES (1, [1.0, 2.0, 3.0]), + (2, [4.0, 5.0, 6.0]); + +INSERT INTO right_feats VALUES (1, [0.1, 0.2]), + (2, [0.3, 0.4]); + +-- ----------------------------------------------------------------------- +-- Weight matrix W (5 x 2), written to /tmp so mat_mul can read it. +-- Row-major: first 3 rows belong to left side, last 2 to right side. +-- ----------------------------------------------------------------------- +COPY ( + SELECT * FROM (VALUES + (1.0, 0.0), + (0.0, 1.0), + (1.0, 1.0), + (0.5, 0.0), + (0.0, 0.5) + ) t(c1, c2) +) TO '/tmp/demo_W_5x2.csv' (FORMAT CSV, HEADER FALSE); + +-- ----------------------------------------------------------------------- +-- 1) Baseline result (no optimizer) — used to verify correctness later. +-- ----------------------------------------------------------------------- +SET enable_extended_optimizer = ''; + +SELECT + l.id, + mat_mul(list_concat(l.features, r.features), '/tmp/demo_W_5x2.csv', 'dense') AS result +FROM left_feats l +JOIN right_feats r ON l.id = r.id +ORDER BY l.id; + +-- ----------------------------------------------------------------------- +-- 2) Same query with the factorization optimizer (option '4'). +-- The terminal will print the logical plan BEFORE and AFTER the rewrite. +-- ----------------------------------------------------------------------- +SET enable_extended_optimizer = '4'; + +SELECT + l.id, + mat_mul(list_concat(l.features, r.features), '/tmp/demo_W_5x2.csv', 'dense') AS result +FROM left_feats l +JOIN right_feats r ON l.id = r.id +ORDER BY l.id; diff --git a/src/include/ml_functions/list_add_vv.hpp b/src/include/ml_functions/list_add_vv.hpp new file mode 100644 index 0000000..418c463 --- /dev/null +++ b/src/include/ml_functions/list_add_vv.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/function/function.hpp" + +namespace duckdb { + +// Element-wise addition of two LIST vectors at runtime. +// result[j] = lhs[j] + rhs[j] for each element j in each row. +// Used by MLFactorizationRewriteAction to combine the two partial +// mat_mul results from each side of a join. +struct ML_ListAddVV { + static void Register(ExtensionLoader &loader); + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); +}; + +} // namespace duckdb diff --git a/src/include/optimization/actions/MLFactorizationRewriteAction.hpp b/src/include/optimization/actions/MLFactorizationRewriteAction.hpp new file mode 100644 index 0000000..b9661ce --- /dev/null +++ b/src/include/optimization/actions/MLFactorizationRewriteAction.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include "duckdb.hpp" + +namespace duckdb { + +// MLFactorizationRewriteAction +// +// Targets: +// PROJECTION( mat_mul(list_concat(L, R), W_file, mode) ) +// COMPARISON_JOIN +// left_child <- L references only columns from here +// right_child <- R references only columns from here +// +// Rewrites to: +// PROJECTION( list_add_vv(ref_left_result, ref_right_result) ) +// COMPARISON_JOIN +// PROJECTION( mat_mul(L, W_top_file, mode) ) <- pushed to left child +// left_child +// PROJECTION( mat_mul(R, W_bottom_file, mode) ) <- pushed to right child +// right_child +// +// W_top = rows [0, k_left) of W (shape k_left x n) +// W_bottom = rows [k_left, k) of W (shape k_right x n) +// +// k_left is read from the static type of L: +// - DOUBLE[N] array column -> ArrayType::GetSize() +// - list_value(c1, c2, ...) -> child count +// - LIST column -> optimization skipped (unknown at plan time) +class MLFactorizationRewriteAction { +public: + static bool check(OptimizerExtensionInput &input, unique_ptr &plan); + static bool apply(OptimizerExtensionInput &input, unique_ptr &plan); +}; + +} // namespace duckdb diff --git a/src/ml_functions/CMakeLists.txt b/src/ml_functions/CMakeLists.txt index 3d9987f..2924345 100644 --- a/src/ml_functions/CMakeLists.txt +++ b/src/ml_functions/CMakeLists.txt @@ -5,6 +5,7 @@ set(ML_FUNCTIONS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/one_hot_encoder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/decision_tree.cpp ${CMAKE_CURRENT_SOURCE_DIR}/matrix_addition.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/list_add_vv.cpp ${CMAKE_CURRENT_SOURCE_DIR}/argmax.cpp ${CMAKE_CURRENT_SOURCE_DIR}/softmax.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sigmoid.cpp diff --git a/src/ml_functions/list_add_vv.cpp b/src/ml_functions/list_add_vv.cpp new file mode 100644 index 0000000..e6185db --- /dev/null +++ b/src/ml_functions/list_add_vv.cpp @@ -0,0 +1,100 @@ +#include "ml_functions/list_add_vv.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/vector_operations/generic_executor.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" + +namespace duckdb { + +void ML_ListAddVV::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + result.SetVectorType(VectorType::FLAT_VECTOR); + + Vector &lhs = args.data[0]; + Vector &rhs = args.data[1]; + + // Unify top-level list entries + UnifiedVectorFormat lhs_uvf, rhs_uvf; + lhs.ToUnifiedFormat(count, lhs_uvf); + rhs.ToUnifiedFormat(count, rhs_uvf); + + const auto *lhs_entries = UnifiedVectorFormat::GetData(lhs_uvf); + const auto *rhs_entries = UnifiedVectorFormat::GetData(rhs_uvf); + + // Unify child (element) vectors + auto &lhs_child = ListVector::GetEntry(lhs); + auto &rhs_child = ListVector::GetEntry(rhs); + + const idx_t lhs_total = ListVector::GetListSize(lhs); + const idx_t rhs_total = ListVector::GetListSize(rhs); + + UnifiedVectorFormat lhs_child_uvf, rhs_child_uvf; + lhs_child.ToUnifiedFormat(lhs_total, lhs_child_uvf); + rhs_child.ToUnifiedFormat(rhs_total, rhs_child_uvf); + + const double *lhs_data = UnifiedVectorFormat::GetData(lhs_child_uvf); + const double *rhs_data = UnifiedVectorFormat::GetData(rhs_child_uvf); + + // Pre-scan: count total valid output elements + idx_t total_out = 0; + for (idx_t i = 0; i < count; ++i) { + idx_t li = lhs_uvf.sel->get_index(i); + idx_t ri = rhs_uvf.sel->get_index(i); + if (lhs_uvf.validity.RowIsValid(li) && rhs_uvf.validity.RowIsValid(ri)) { + total_out += lhs_entries[li].length; + } + } + + auto *res_entries = FlatVector::GetData(result); + auto &res_child = ListVector::GetEntry(result); + auto &res_valid = FlatVector::Validity(result); + + ListVector::Reserve(result, total_out); + ListVector::SetListSize(result, total_out); + double *out = FlatVector::GetData(res_child); + + idx_t cursor = 0; + for (idx_t i = 0; i < count; ++i) { + idx_t li = lhs_uvf.sel->get_index(i); + idx_t ri = rhs_uvf.sel->get_index(i); + + if (!lhs_uvf.validity.RowIsValid(li) || !rhs_uvf.validity.RowIsValid(ri)) { + res_entries[i] = {0, 0}; + res_valid.Set(i, false); + continue; + } + + const auto &le = lhs_entries[li]; + const auto &re = rhs_entries[ri]; + + if (le.length != re.length) { + throw InvalidInputException( + "list_add_vv: length mismatch at row %llu (%llu vs %llu)", + (unsigned long long)i, + (unsigned long long)le.length, + (unsigned long long)re.length); + } + + res_entries[i] = {cursor, le.length}; + res_valid.Set(i, true); + + const double *lp = lhs_data + le.offset; + const double *rp = rhs_data + re.offset; + for (idx_t j = 0; j < le.length; ++j) { + out[cursor + j] = lp[j] + rp[j]; + } + cursor += le.length; + } +} + +void ML_ListAddVV::Register(ExtensionLoader &loader) { + ScalarFunction fn( + "list_add_vv", + {LogicalType::LIST(LogicalType::DOUBLE), LogicalType::LIST(LogicalType::DOUBLE)}, + LogicalType::LIST(LogicalType::DOUBLE), + Execute); + loader.RegisterFunction(fn); +} + +} // namespace duckdb diff --git a/src/ml_functions/ml_functions_scalar.cpp b/src/ml_functions/ml_functions_scalar.cpp index 7c9d539..0126340 100644 --- a/src/ml_functions/ml_functions_scalar.cpp +++ b/src/ml_functions/ml_functions_scalar.cpp @@ -4,6 +4,7 @@ #include "ml_functions/one_hot_encoder.hpp" #include "ml_functions/decision_tree.hpp" #include "ml_functions/matrix_addition.hpp" +#include "ml_functions/list_add_vv.hpp" #include "ml_functions/argmax.hpp" #include "ml_functions/softmax.hpp" #include "ml_functions/sigmoid.hpp" @@ -20,6 +21,7 @@ void RegisterMLScalarFunctions(ExtensionLoader &loader) { ML_OneHotEncoder::Register(loader); ML_DecisionTree::Register(loader); ML_MatrixAddition::Register(loader); + ML_ListAddVV::Register(loader); ML_Argmax::Register(loader); ML_Softmax::Register(loader); ML_Sigmoid::Register(loader); diff --git a/src/optimization/CMakeLists.txt b/src/optimization/CMakeLists.txt index adaf6d7..ed7aa3e 100644 --- a/src/optimization/CMakeLists.txt +++ b/src/optimization/CMakeLists.txt @@ -11,6 +11,7 @@ list(APPEND EXTENSION_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/actions/MultiLayerUDF2TorchNNRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/DecisionForestUDF2RelationRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLDecompositionPushdownRewriteAction.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLFactorizationRewriteAction.cpp #Feature Extraction Helpers ${CMAKE_CURRENT_SOURCE_DIR}/feature_extraction/query2vec/query_annotation.cpp diff --git a/src/optimization/actions/MLFactorizationRewriteAction.cpp b/src/optimization/actions/MLFactorizationRewriteAction.cpp new file mode 100644 index 0000000..fe3dd48 --- /dev/null +++ b/src/optimization/actions/MLFactorizationRewriteAction.cpp @@ -0,0 +1,515 @@ +#include "optimization/actions/MLFactorizationRewriteAction.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/assert.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/value.hpp" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" + +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/operator/logical_comparison_join.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" + +#include "ml_functions/matrix_multiplication.hpp" + +#include +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +// --------------------------------------------------------------------------- +// Small plan/expression helpers +// --------------------------------------------------------------------------- + +static idx_t GetLogicalWidth(LogicalOperator &op) { + if (op.types.empty()) { + op.ResolveOperatorTypes(); + } + return op.types.size(); +} + +static bool IsMatMul(const BoundFunctionExpression &f) { + return StringUtil::CIEquals(f.function.name, "mat_mul"); +} + +static bool IsListConcat(const BoundFunctionExpression &f) { + auto name = StringUtil::Lower(f.function.name); + return name == "list_concat" || name == "list_cat" || name == "array_concat"; +} + +// Collect every table_index referenced by BoundColumnRefExpressions in expr. +static void CollectTableIndexes(const unique_ptr &expr, + std::unordered_set &out) { + if (!expr) { + return; + } + ExpressionIterator::EnumerateExpression( + const_cast &>(expr), [&](Expression &sub) { + if (sub.type == ExpressionType::BOUND_COLUMN_REF) { + out.insert(sub.Cast().binding.table_index); + } + }); +} + +// True when every table index in expr_tables appears in side_tables, +// and expr_tables is non-empty (i.e. the expression actually references something). +static bool TablesOnlyInSide(const std::unordered_set &expr_tables, + const vector &side_tables) { + if (expr_tables.empty()) { + return false; + } + for (idx_t t : expr_tables) { + if (std::find(side_tables.begin(), side_tables.end(), t) == side_tables.end()) { + return false; + } + } + return true; +} + +// Return the static element count of a list-producing expression, or -1 if unknown. +// DOUBLE[N] column -> N (ARRAY type carries size in the type) +// cast(DOUBLE[N] -> DOUBLE[]) -> N (DuckDB inserts this cast before list_concat) +// list_value(a, b, c) -> 3 (inline list literal; child count = element count) +// LIST -> -1 (variable length, unknown at plan time) +static int64_t GetStaticListLength(const unique_ptr &expr) { + if (!expr) { + return -1; + } + // Direct ARRAY type (e.g. DOUBLE[3] column ref used without casting). + if (expr->return_type.id() == LogicalTypeId::ARRAY) { + return static_cast(ArrayType::GetSize(expr->return_type)); + } + // DuckDB silently casts DOUBLE[N] → DOUBLE[] before passing to list_concat. + // Look through the cast to recover the original array size. + if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) { + return static_cast(ArrayType::GetSize(cast.child->return_type)); + } + } + // Inline list literal: list_value(col1, col2, ...) — each child is one element. + if (expr->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = expr->Cast(); + auto name = StringUtil::Lower(bfe.function.name); + if (name == "list_value" || name == "array_value") { + return static_cast(bfe.children.size()); + } + } + return -1; +} + +// --------------------------------------------------------------------------- +// Weight file helpers +// --------------------------------------------------------------------------- + +// Write a contiguous slice of a row-major weight matrix (rows [row_start, row_start+row_count)) +// to a CSV file under /tmp. The path is deterministic so the same split is never written twice. +static std::string WriteSplitWeightCSV(const std::vector &weights, idx_t n, + idx_t row_start, idx_t row_count, + const std::string &base_path, + const std::string &suffix) { + // Deterministic filename: hash of base_path + split parameters. + std::size_t h = std::hash{}( + base_path + "|" + std::to_string(row_start) + "|" + std::to_string(row_count)); + std::string out_path = "/tmp/matmul_fact_" + std::to_string(h) + suffix + ".csv"; + + // Idempotent: skip writing if the file already exists. + { + std::ifstream probe(out_path); + if (probe.good()) { + return out_path; + } + } + + std::ofstream f(out_path); + if (!f.is_open()) { + throw InternalException("MLFactorization: cannot write split weight file: %s", + out_path.c_str()); + } + + for (idx_t row = row_start; row < row_start + row_count; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) { + f << ','; + } + f << weights[row * n + col]; + } + f << '\n'; + } + + return out_path; +} + +// --------------------------------------------------------------------------- +// Expression builders +// --------------------------------------------------------------------------- + +// Build a new mat_mul(input, W_path, mode_str) BoundFunctionExpression with +// a pre-populated BindData containing the split weights. Looks up the scalar +// function from the catalog so we get the correct function pointer. +static unique_ptr BuildMatMulExpr(ClientContext &ctx, + unique_ptr input_expr, + const std::string &W_path, + const std::string &mode_str, + const std::vector &W_flat, + idx_t k, idx_t n, + ML_MatrixMultiply::MulMode mode) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = + catalog.GetEntry(ctx, DEFAULT_SCHEMA, "mat_mul"); + + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 3 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1].id() == LogicalTypeId::VARCHAR && + fn.arguments[2].id() == LogicalTypeId::VARCHAR) { + chosen = &fn; + break; + } + } + if (!chosen) { + throw InternalException( + "MLFactorization: mat_mul(LIST, VARCHAR, VARCHAR) overload not found"); + } + + auto bind_info = make_uniq(W_flat, k, n, W_path, true, mode); + + vector> children; + children.push_back(std::move(input_expr)); + children.push_back(make_uniq(Value(W_path))); + children.push_back(make_uniq(Value(mode_str))); + + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), + std::move(bind_info)); +} + +// Build list_add_vv(left_ref, right_ref). +static unique_ptr BuildListAddVVExpr(ClientContext &ctx, + unique_ptr left_ref, + unique_ptr right_ref) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = + catalog.GetEntry(ctx, DEFAULT_SCHEMA, "list_add_vv"); + + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 2 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1] == LogicalType::LIST(LogicalType::DOUBLE)) { + chosen = &fn; + break; + } + } + if (!chosen) { + throw InternalException( + "MLFactorization: list_add_vv(LIST, LIST) overload not found"); + } + + vector> children; + children.push_back(std::move(left_ref)); + children.push_back(std::move(right_ref)); + + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), nullptr); +} + +// --------------------------------------------------------------------------- +// Plan mutation helper +// --------------------------------------------------------------------------- + +// Wrap join.children[side_idx] in a new LogicalProjection that passes through +// all existing columns and appends extra_expr as an additional column. +// Sets new_col_idx to the index of extra_expr in the new child's output. +// Returns false if the child has multiple table indexes (can't safely rewrite). +static bool InsertProjectionOnSide(LogicalComparisonJoin &join, int side_idx, + Expression &extra_expr, idx_t &new_col_idx) { + D_ASSERT(side_idx == 0 || side_idx == 1); + auto &child_ptr = join.children[side_idx]; + if (!child_ptr) { + return false; + } + + auto side_tables = child_ptr->GetTableIndex(); + if (side_tables.size() != 1) { + return false; + } + const idx_t table_index = side_tables[0]; + + const idx_t child_width = GetLogicalWidth(*child_ptr); + if (child_width == 0) { + return false; + } + + vector> select_list; + select_list.reserve(child_width + 1); + + // Pass-through all existing columns by position. + for (idx_t i = 0; i < child_width; ++i) { + select_list.push_back(make_uniq(child_ptr->types[i], i)); + } + + // Append the new expression; record its position in the output. + select_list.push_back(extra_expr.Copy()); + new_col_idx = child_width; + + // Wrap the existing child in a projection that inherits its table_index, + // so bindings from above that reference this side remain valid. + auto new_proj = make_uniq(table_index, std::move(select_list)); + new_proj->children.push_back(std::move(child_ptr)); + child_ptr = std::move(new_proj); + + return true; +} + +// --------------------------------------------------------------------------- +// Pattern detection (used by both check and apply) +// --------------------------------------------------------------------------- + +// Return true iff expr is: +// mat_mul( list_concat(L, R), W_file, mode ) +// where L references only left_tables, R references only right_tables, +// and k_left = GetStaticListLength(L) is positive. +static bool MatchesFactorizationPattern(const unique_ptr &expr, + const vector &left_tables, + const vector &right_tables) { + if (!expr || expr->expression_class != ExpressionClass::BOUND_FUNCTION) { + std::cout << "[MLFact] skip: expr not BOUND_FUNCTION\n"; + return false; + } + auto &f = expr->Cast(); + std::cout << "[MLFact] top expr function: '" << f.function.name << "'\n"; + if (!IsMatMul(f) || f.children.empty()) { + return false; + } + + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) { + std::cout << "[MLFact] skip: no bound weights\n"; + return false; + } + + auto &first_child = f.children[0]; + if (!first_child || first_child->expression_class != ExpressionClass::BOUND_FUNCTION) { + std::cout << "[MLFact] skip: first child of mat_mul not BOUND_FUNCTION (class=" + << (first_child ? std::to_string((int)first_child->expression_class) : "null") << ")\n"; + return false; + } + auto &lc = first_child->Cast(); + std::cout << "[MLFact] first child function: '" << lc.function.name + << "' children=" << lc.children.size() << "\n"; + if (!IsListConcat(lc) || lc.children.size() != 2) { + std::cout << "[MLFact] skip: not list_concat or wrong child count\n"; + return false; + } + + std::unordered_set left_refs, right_refs; + CollectTableIndexes(lc.children[0], left_refs); + CollectTableIndexes(lc.children[1], right_refs); + + std::cout << "[MLFact] left_refs count=" << left_refs.size() + << " right_refs count=" << right_refs.size() << "\n"; + + if (!TablesOnlyInSide(left_refs, left_tables)) { + std::cout << "[MLFact] skip: left_refs not contained in join left side\n"; + return false; + } + if (!TablesOnlyInSide(right_refs, right_tables)) { + std::cout << "[MLFact] skip: right_refs not contained in join right side\n"; + return false; + } + + int64_t k_left = GetStaticListLength(lc.children[0]); + std::cout << "[MLFact] k_left=" << k_left + << " (type=" << lc.children[0]->return_type.ToString() << ")\n"; + return k_left > 0; +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// check +// --------------------------------------------------------------------------- + +bool MLFactorizationRewriteAction::check(OptimizerExtensionInput &input, + unique_ptr &plan) { + if (!plan) { + return false; + } + + bool found = false; + + std::cout << "[MLFact check] node type=" << EnumUtil::ToString(plan->type) << "\n"; + + if (plan->type == LogicalOperatorType::LOGICAL_PROJECTION && + plan->children.size() == 1 && + plan->children[0] && + plan->children[0]->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + + std::cout << "[MLFact check] found PROJECTION over COMPARISON_JOIN\n"; + auto &join = plan->children[0]->Cast(); + auto left_tables = join.children[0]->GetTableIndex(); + auto right_tables = join.children[1]->GetTableIndex(); + + auto &proj = plan->Cast(); + for (auto &expr : proj.expressions) { + if (MatchesFactorizationPattern(expr, left_tables, right_tables)) { + found = true; + break; + } + } + } + + for (auto &child : plan->children) { + found |= check(input, child); + } + + return found; +} + +// --------------------------------------------------------------------------- +// apply +// --------------------------------------------------------------------------- + +bool MLFactorizationRewriteAction::apply(OptimizerExtensionInput &input, + unique_ptr &plan) { + if (!plan) { + return false; + } + + bool changed = false; + + // Bottom-up: rewrite children first. + for (auto &child : plan->children) { + changed |= apply(input, child); + } + + // Only act on a projection whose direct child is a comparison join. + if (plan->type != LogicalOperatorType::LOGICAL_PROJECTION) { + return changed; + } + auto &proj = plan->Cast(); + if (proj.children.size() != 1 || !proj.children[0] || + proj.children[0]->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + return changed; + } + + auto &join = proj.children[0]->Cast(); + auto left_tables = join.children[0]->GetTableIndex(); + auto right_tables = join.children[1]->GetTableIndex(); + + // Both join sides must each have exactly one table index so we can safely + // insert a projection on each side. + if (left_tables.size() != 1 || right_tables.size() != 1) { + return changed; + } + + for (auto &expr_ptr : proj.expressions) { + if (!MatchesFactorizationPattern(expr_ptr, left_tables, right_tables)) { + continue; + } + + // ---- Unpack the matched expression tree ---- + auto &f = expr_ptr->Cast(); + auto &lc = f.children[0]->Cast(); // list_concat + auto *bd = dynamic_cast(f.bind_info.get()); + + const int64_t k_left_signed = GetStaticListLength(lc.children[0]); + D_ASSERT(k_left_signed > 0); + const idx_t k_left = static_cast(k_left_signed); + const idx_t k_right = bd->k - k_left; + const idx_t n = bd->n; + + if (k_right == 0 || k_left + k_right != bd->k) { + continue; // sanity: split must be non-trivial + } + + // ---- Split the weight matrix ---- + // weights is flat row-major [k * n]: row i starts at weights[i*n]. + // W_top = rows [0, k_left) + // W_bottom = rows [k_left, k) + std::vector W_top(bd->weights.begin(), + bd->weights.begin() + (ptrdiff_t)(k_left * n)); + std::vector W_bottom(bd->weights.begin() + (ptrdiff_t)(k_left * n), + bd->weights.end()); + + std::string W_top_path = WriteSplitWeightCSV(bd->weights, n, 0, k_left, bd->weights_file, "_left"); + std::string W_bottom_path = WriteSplitWeightCSV(bd->weights, n, k_left, k_right, bd->weights_file, "_right"); + + // Mode string for the reconstructed mat_mul children. + std::string mode_str; + switch (bd->mode) { + case ML_MatrixMultiply::MulMode::Dense: mode_str = "dense"; break; + case ML_MatrixMultiply::MulMode::InputSparse: mode_str = "sparse"; break; + default: mode_str = "dense"; break; + } + + // ---- Build partial mat_mul expressions ---- + // L and R are copies of list_concat's children; they carry the correct + // BoundColumnRefExpression bindings (T_left / T_right) and need no rewrite. + auto mat_mul_left = BuildMatMulExpr(input.context, + lc.children[0]->Copy(), + W_top_path, mode_str, + W_top, k_left, n, bd->mode); + + auto mat_mul_right = BuildMatMulExpr(input.context, + lc.children[1]->Copy(), + W_bottom_path, mode_str, + W_bottom, k_right, n, bd->mode); + + // ---- Record widths BEFORE any modification ---- + const idx_t L_width = GetLogicalWidth(*join.children[0]); + const idx_t R_width = GetLogicalWidth(*join.children[1]); + + // ---- Insert projections on each join side ---- + idx_t new_left_col = 0; + idx_t new_right_col = 0; + + if (!InsertProjectionOnSide(join, 0, *mat_mul_left, new_left_col)) { + continue; + } + if (!InsertProjectionOnSide(join, 1, *mat_mul_right, new_right_col)) { + // Left side already modified; keep going — the result is suboptimal + // but not incorrect (the new projection is benign). + continue; + } + + // ---- Compute global column positions in the JOIN output ---- + // After insertions: + // Left side occupies indices [0, L_width] (L_width+1 cols, new at L_width) + // Right side occupies indices [L_width+1, L_width+1+R_width] (R_width+1 cols, new at end) + const idx_t left_global_idx = L_width; + const idx_t right_global_idx = L_width + 1 + R_width; + + // ---- Replace original mat_mul with list_add_vv(ref_left, ref_right) ---- + auto left_ref = make_uniq( + LogicalType::LIST(LogicalType::DOUBLE), left_global_idx); + auto right_ref = make_uniq( + LogicalType::LIST(LogicalType::DOUBLE), right_global_idx); + + expr_ptr = BuildListAddVVExpr(input.context, + std::move(left_ref), + std::move(right_ref)); + + changed = true; + break; // one factorization per projection per pass + } + + return changed; +} + +} // namespace duckdb diff --git a/src/optimization/register_optimizers.cpp b/src/optimization/register_optimizers.cpp index 594db1d..87fc20e 100644 --- a/src/optimization/register_optimizers.cpp +++ b/src/optimization/register_optimizers.cpp @@ -23,6 +23,7 @@ #include "optimization/actions/MatMulDense2SparseRewriteAction.hpp" #include "optimization/actions/MultiLayerUDF2TorchNNRewriteAction.hpp" #include "optimization/actions/MLDecompositionPushdownRewriteAction.hpp" +#include "optimization/actions/MLFactorizationRewriteAction.hpp" namespace duckdb { @@ -147,6 +148,14 @@ struct CactusDynamicOpt : public OptimizerExtension { MultiLayerUDF2TorchNNRewriteAction::apply(in, plan); } + if (opt_setting.find("4") != std::string::npos) { + if (MLFactorizationRewriteAction::check(in, plan)) { + std::cout << "\n[MLFactorization] BEFORE:\n" << plan->ToString() << "\n"; + MLFactorizationRewriteAction::apply(in, plan); + std::cout << "\n[MLFactorization] AFTER:\n" << plan->ToString() << "\n"; + } + } + // pretty = plan->ToString(); // std::cout << "\n[CactusDynamicOpt] Final Plan:\n" << pretty << "\n"; } From dd1d9d76ef50f7faee79b1bbdf16ffc29975efe8 Mon Sep 17 00:00:00 2001 From: jrt1899 Date: Thu, 7 May 2026 18:47:09 -0700 Subject: [PATCH 2/6] initial implementation of greedy optimizer --- .../MLGreedyFactorizationOptimizer.hpp | 13 + src/optimization/CMakeLists.txt | 1 + .../MLGreedyFactorizationOptimizer.cpp | 902 ++++++++++++++++++ src/optimization/register_optimizers.cpp | 13 + test/sql/greedy_factorization.test | 105 ++ 5 files changed, 1034 insertions(+) create mode 100644 src/include/optimization/actions/MLGreedyFactorizationOptimizer.hpp create mode 100644 src/optimization/actions/MLGreedyFactorizationOptimizer.cpp create mode 100644 test/sql/greedy_factorization.test diff --git a/src/include/optimization/actions/MLGreedyFactorizationOptimizer.hpp b/src/include/optimization/actions/MLGreedyFactorizationOptimizer.hpp new file mode 100644 index 0000000..cef9248 --- /dev/null +++ b/src/include/optimization/actions/MLGreedyFactorizationOptimizer.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "duckdb.hpp" + +namespace duckdb { + +class MLGreedyFactorizationOptimizer { +public: + static bool check(OptimizerExtensionInput &input, unique_ptr &plan); + static bool apply(OptimizerExtensionInput &input, unique_ptr &plan); +}; + +} // namespace duckdb diff --git a/src/optimization/CMakeLists.txt b/src/optimization/CMakeLists.txt index ed7aa3e..08b38db 100644 --- a/src/optimization/CMakeLists.txt +++ b/src/optimization/CMakeLists.txt @@ -12,6 +12,7 @@ list(APPEND EXTENSION_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/actions/DecisionForestUDF2RelationRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLDecompositionPushdownRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLFactorizationRewriteAction.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLGreedyFactorizationOptimizer.cpp #Feature Extraction Helpers ${CMAKE_CURRENT_SOURCE_DIR}/feature_extraction/query2vec/query_annotation.cpp diff --git a/src/optimization/actions/MLGreedyFactorizationOptimizer.cpp b/src/optimization/actions/MLGreedyFactorizationOptimizer.cpp new file mode 100644 index 0000000..9b79ab1 --- /dev/null +++ b/src/optimization/actions/MLGreedyFactorizationOptimizer.cpp @@ -0,0 +1,902 @@ +#include "optimization/actions/MLGreedyFactorizationOptimizer.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/assert.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/value.hpp" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" + +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/operator/logical_comparison_join.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" + +#include "ml_functions/matrix_multiplication.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +// --------------------------------------------------------------------------- +// JoinTreeNode +// --------------------------------------------------------------------------- + +struct JoinTreeNode { + LogicalOperator *plan_node; // pointer into the live DuckDB plan tree + int side; // 0=left child of parent, 1=right child, -1=root + JoinTreeNode *parent; // nullptr for root + std::vector children; // up to 2 for a binary join + + // Populated during tree walk + idx_t num_rows; // estimated cardinality + int64_t num_features; // total input feature dims still un-reduced at this node + int64_t new_features; // model output width n (first-layer neuron count) + double cardinality_ratio; // num_rows / parent->num_rows (selectivity from this side) + int depth; // absolute depth (root = 1) + int max_depth; // deepest node in the whole tree (same for all nodes after BFS) + double cost; // current cost estimate + + // Assignment phase + int label; // 0=unassigned, 1=push mat_mul here, 2=combine children (list_add_vv) + bool processed; + bool subtree_processed; + + // Weight-split info set during apply phase + idx_t row_start; // start row in the original W for this node's feature block + idx_t row_count; // number of rows (= sum of k_i for tables on this side) +}; + +// --------------------------------------------------------------------------- +// Expression helpers (shared with MLFactorizationRewriteAction pattern) +// --------------------------------------------------------------------------- + +static bool IsMatMul(const BoundFunctionExpression &f) { + return StringUtil::CIEquals(f.function.name, "mat_mul"); +} + +static bool IsListConcat(const BoundFunctionExpression &f) { + auto name = StringUtil::Lower(f.function.name); + return name == "list_concat" || name == "list_cat" || name == "array_concat"; +} + +static bool AreConsecutivePartIndices(const std::vector &indices) { + if (indices.empty()) return false; + for (idx_t i = 1; i < indices.size(); ++i) { + if (indices[i] != indices[i - 1] + 1) return false; + } + return true; +} + +static int64_t GetStaticListLength(const unique_ptr &expr) { + if (!expr) return -1; + if (expr->return_type.id() == LogicalTypeId::ARRAY) + return static_cast(ArrayType::GetSize(expr->return_type)); + if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) + return static_cast(ArrayType::GetSize(cast.child->return_type)); + } + if (expr->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = expr->Cast(); + auto name = StringUtil::Lower(bfe.function.name); + if (name == "list_value" || name == "array_value") + return static_cast(bfe.children.size()); + } + return -1; +} + +// Collect every table_index referenced by BoundColumnRefExpressions in expr. +static void CollectTableIndexes(Expression &expr, + std::unordered_set &out) { + auto owned_copy = expr.Copy(); + ExpressionIterator::EnumerateExpression( + owned_copy, [&](Expression &sub) { + if (sub.type == ExpressionType::BOUND_COLUMN_REF) + out.insert(sub.Cast().binding.table_index); + }); +} + +// Recursively flatten list_concat(list_concat(A,B),C) -> [{A,k_A},{B,k_B},{C,k_C}] +struct FlatPart { + Expression *expr; + int64_t length; +}; + +static void FlattenListConcatImpl(Expression *expr, std::vector &out) { + if (!expr) return; + + // Unwrap implicit cast DOUBLE[N] -> DOUBLE[] inserted by DuckDB + Expression *inner = expr; + if (inner->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = inner->Cast(); + if (cast.child) inner = cast.child.get(); + } + + if (inner->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = inner->Cast(); + if (IsListConcat(bfe) && bfe.children.size() == 2) { + FlattenListConcatImpl(bfe.children[0].get(), out); + FlattenListConcatImpl(bfe.children[1].get(), out); + return; + } + } + + // Leaf: compute static length and record + // Use original expr (not inner) so GetStaticListLength can unwrap the cast itself + unique_ptr dummy; // we only need length from it + int64_t len = -1; + if (expr->return_type.id() == LogicalTypeId::ARRAY) + len = static_cast(ArrayType::GetSize(expr->return_type)); + else if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) + len = static_cast(ArrayType::GetSize(cast.child->return_type)); + } + out.push_back({expr, len}); +} + +static std::vector FlattenListConcat(Expression &expr) { + std::vector parts; + FlattenListConcatImpl(&expr, parts); + return parts; +} + +static unique_ptr BuildConcatFromParts(const BoundFunctionExpression &concat_proto, + const std::vector &parts, + const std::vector &indices) { + D_ASSERT(!indices.empty()); + auto expr = parts[indices[0]].expr->Copy(); + for (idx_t i = 1; i < indices.size(); ++i) { + vector> children; + children.push_back(std::move(expr)); + children.push_back(parts[indices[i]].expr->Copy()); + expr = make_uniq( + LogicalType::LIST(LogicalType::DOUBLE), + concat_proto.function, + std::move(children), + nullptr); + } + return expr; +} + +// --------------------------------------------------------------------------- +// Weight file helper +// --------------------------------------------------------------------------- + +static std::string WriteSplitWeightCSV(const std::vector &weights, idx_t n, + idx_t row_start, idx_t row_count, + const std::string &base_path, + const std::string &suffix) { + std::size_t h = std::hash{}( + base_path + "|" + std::to_string(row_start) + "|" + std::to_string(row_count)); + std::string out_path = "/tmp/matmul_greedy_" + std::to_string(h) + suffix + ".csv"; + { + std::ifstream probe(out_path); + if (probe.good()) return out_path; + } + std::ofstream f(out_path); + if (!f.is_open()) + throw InternalException("MLGreedyFactorization: cannot write weight file: %s", out_path.c_str()); + for (idx_t row = row_start; row < row_start + row_count; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) f << ','; + f << weights[row * n + col]; + } + f << '\n'; + } + return out_path; +} + +static std::string WriteCustomWeightCSV(const std::vector &weights, idx_t k, idx_t n, + const std::string &base_path, + const std::string &suffix, + const std::string &cache_key) { + std::size_t h = std::hash{}( + base_path + "|" + cache_key + "|" + std::to_string(k) + "|" + std::to_string(n)); + std::string out_path = "/tmp/matmul_greedy_" + std::to_string(h) + suffix + ".csv"; + { + std::ifstream probe(out_path); + if (probe.good()) return out_path; + } + + std::ofstream f(out_path); + if (!f.is_open()) + throw InternalException("MLGreedyFactorization: cannot write weight file: %s", out_path.c_str()); + for (idx_t row = 0; row < k; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) f << ','; + f << weights[row * n + col]; + } + f << '\n'; + } + return out_path; +} + +// --------------------------------------------------------------------------- +// Expression builders +// --------------------------------------------------------------------------- + +static unique_ptr BuildMatMulExpr(ClientContext &ctx, + unique_ptr input_expr, + const std::string &W_path, + const std::string &mode_str, + const std::vector &W_flat, + idx_t k, idx_t n, + ML_MatrixMultiply::MulMode mode) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = catalog.GetEntry(ctx, DEFAULT_SCHEMA, "mat_mul"); + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 3 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1].id() == LogicalTypeId::VARCHAR && + fn.arguments[2].id() == LogicalTypeId::VARCHAR) { + chosen = &fn; + break; + } + } + if (!chosen) + throw InternalException("MLGreedyFactorization: mat_mul overload not found"); + + auto bind_info = make_uniq(W_flat, k, n, W_path, true, mode); + vector> children; + children.push_back(std::move(input_expr)); + children.push_back(make_uniq(Value(W_path))); + children.push_back(make_uniq(Value(mode_str))); + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), std::move(bind_info)); +} + +static unique_ptr BuildListAddVVExpr(ClientContext &ctx, + unique_ptr left, + unique_ptr right) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = catalog.GetEntry(ctx, DEFAULT_SCHEMA, "list_add_vv"); + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 2 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1] == LogicalType::LIST(LogicalType::DOUBLE)) { + chosen = &fn; + break; + } + } + if (!chosen) + throw InternalException("MLGreedyFactorization: list_add_vv overload not found"); + vector> children; + children.push_back(std::move(left)); + children.push_back(std::move(right)); + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), nullptr); +} + +// --------------------------------------------------------------------------- +// InsertProjectionOnSide +// --------------------------------------------------------------------------- + +static idx_t GetLogicalWidth(LogicalOperator &op) { + if (op.types.empty()) op.ResolveOperatorTypes(); + return op.types.size(); +} + +static bool InsertProjectionOnSide(LogicalComparisonJoin &join, int side_idx, + Expression &extra_expr, idx_t &new_col_idx, + idx_t &new_table_idx) { + D_ASSERT(side_idx == 0 || side_idx == 1); + auto &child_ptr = join.children[side_idx]; + if (!child_ptr) return false; + + auto side_tables = child_ptr->GetTableIndex(); + if (side_tables.size() != 1) return false; + + const idx_t table_index = side_tables[0]; + const idx_t child_width = GetLogicalWidth(*child_ptr); + if (child_width == 0) return false; + + vector> select_list; + select_list.reserve(child_width + 1); + for (idx_t i = 0; i < child_width; ++i) + select_list.push_back(make_uniq(child_ptr->types[i], i)); + select_list.push_back(extra_expr.Copy()); + new_col_idx = child_width; + + auto new_proj = make_uniq(table_index, std::move(select_list)); + new_proj->children.push_back(std::move(child_ptr)); + child_ptr = std::move(new_proj); + + // If the join already projects a subset of child columns, explicitly expose + // the appended column so parent expressions can bind to it. + if (side_idx == 0) { + if (!join.left_projection_map.empty()) { + join.left_projection_map.push_back(new_col_idx); + } + } else { + if (!join.right_projection_map.empty()) { + join.right_projection_map.push_back(new_col_idx); + } + } + + new_table_idx = table_index; + return true; +} + +// --------------------------------------------------------------------------- +// Join tree construction +// --------------------------------------------------------------------------- + +static idx_t EstimateCardinality(LogicalOperator &op) { + if (op.estimated_cardinality > 0) + return op.estimated_cardinality; + // Fallback: product of children cardinalities (pessimistic cross-join) + if (op.children.size() == 2) { + idx_t l = EstimateCardinality(*op.children[0]); + idx_t r = EstimateCardinality(*op.children[1]); + return std::max(1, l * r); + } + return 1; +} + +// Walk the DuckDB plan tree and collect all LogicalComparisonJoin nodes, +// building a parallel JoinTreeNode tree. Returns the root JoinTreeNode. +// all_nodes is populated in post-order (children before parents). +static JoinTreeNode *BuildJoinTree(LogicalOperator *op, + JoinTreeNode *parent, + int side, + int64_t new_features, + std::vector> &storage) { + if (!op) return nullptr; + + // Recurse into children first (post-order) + if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + auto node = make_uniq(); + node->plan_node = op; + node->side = side; + node->parent = parent; + node->new_features = new_features; + node->label = 0; + node->processed = false; + node->subtree_processed = false; + node->row_start = 0; + node->row_count = 0; + node->depth = 0; // filled by CalculateDepths + node->max_depth = 0; + node->cost = 0.0; + + idx_t card = EstimateCardinality(*op); + node->num_rows = card; + + if (parent) { + node->cardinality_ratio = (parent->num_rows > 0) + ? static_cast(card) / static_cast(parent->num_rows) + : 1.0; + } else { + node->cardinality_ratio = 1.0; + } + + JoinTreeNode *raw = node.get(); + + // Build children recursively + if (op->children.size() >= 1) { + JoinTreeNode *left_child = BuildJoinTree(op->children[0].get(), raw, 0, new_features, storage); + if (left_child) raw->children.push_back(left_child); + } + if (op->children.size() >= 2) { + JoinTreeNode *right_child = BuildJoinTree(op->children[1].get(), raw, 1, new_features, storage); + if (right_child) raw->children.push_back(right_child); + } + + storage.push_back(std::move(node)); + return raw; + } + + // Not a join — recurse into children looking for joins below + for (idx_t i = 0; i < op->children.size(); ++i) { + int child_side = (op->children.size() == 2) ? (int)i : side; + JoinTreeNode *found = BuildJoinTree(op->children[i].get(), parent, child_side, new_features, storage); + if (found) return found; // return first found (handles single-path plans) + } + return nullptr; +} + +// --------------------------------------------------------------------------- +// Depth computation +// --------------------------------------------------------------------------- + +static int CalculateDepths(JoinTreeNode *root, + std::vector> &all_nodes) { + int max_depth = 0; + std::queue> q; + q.push({root, 1}); + while (!q.empty()) { + auto [node, d] = q.front(); q.pop(); + node->depth = d; + if (d > max_depth) max_depth = d; + for (auto *child : node->children) q.push({child, d + 1}); + } + for (auto &n : all_nodes) n->max_depth = max_depth; + return max_depth; +} + +// --------------------------------------------------------------------------- +// Feature-part attribution: which flat parts belong to which join-tree side +// --------------------------------------------------------------------------- + +// Returns true if the given LogicalOperator subtree contains table_index t. +static bool SubtreeContainsTable(LogicalOperator *op, idx_t table_index) { + if (!op) return false; + auto tables = op->GetTableIndex(); + for (idx_t t : tables) + if (t == table_index) return true; + for (auto &child : op->children) + if (SubtreeContainsTable(child.get(), table_index)) return true; + return false; +} + +// For a given JoinTreeNode, return which side (0 or 1) the table lives on. +// Returns -1 if not found in either child subtree. +static int TableSideOfNode(JoinTreeNode *node, idx_t table_index) { + if (!node || node->plan_node->children.size() < 2) return -1; + if (SubtreeContainsTable(node->plan_node->children[0].get(), table_index)) return 0; + if (SubtreeContainsTable(node->plan_node->children[1].get(), table_index)) return 1; + return -1; +} + +// --------------------------------------------------------------------------- +// Cost model +// --------------------------------------------------------------------------- + +static double ComputeCost(const JoinTreeNode &v) { + if (v.num_features == 0 || v.depth == 0 || v.max_depth == 0) + return 1.0; // safe fallback above threshold + + double var2 = static_cast(v.num_features); + double var4 = static_cast(v.new_features); + double var5 = static_cast(v.max_depth); + double var6 = var4 / var2; + double var7 = var4 / (var2 * v.depth); + double var8 = static_cast(v.num_rows) * var4 / var2; + double var9 = static_cast(v.num_rows) * var4 / (var2 * v.depth); + double var10 = var2 - var4; + double var11 = static_cast(v.depth) / static_cast(v.max_depth); + double var12 = v.cardinality_ratio; + double d4 = std::pow(static_cast(v.depth), 4.0); + + return -4.62412178e-04 * var2 + - 6.99311355e-04 * var4 + - 4.17964429e-03 * var5 + + 1.57416551e-03 * var6 + - 3.69161585e-04 * var7 + - 2.94571978e-05 * var8 + + 6.07897302e-05 * var9 + + 2.36899161e-04 * var10 + - 4.63010544e-02 * var12 + + 2.12526468e-03 * d4 + + 1.40041493e-04 * std::pow(var11, 4.0) + - 3.86043424e-02 * var11 * var12; +} + +// --------------------------------------------------------------------------- +// Greedy algorithm helpers +// --------------------------------------------------------------------------- + +static void ProcessSubtree(JoinTreeNode *node) { + if (node->subtree_processed) return; + std::vector stack = {node}; + while (!stack.empty()) { + JoinTreeNode *cur = stack.back(); stack.pop_back(); + cur->processed = true; + cur->subtree_processed = true; + for (auto *child : cur->children) + if (!child->subtree_processed) stack.push_back(child); + } + if (node->parent) node->parent->subtree_processed = true; +} + +static void UpdateParentFeatures(JoinTreeNode *parent) { + int64_t total = 0; + for (auto *child : parent->children) + total += (child->label != 0) ? child->new_features : child->num_features; + parent->num_features = total; +} + +// --------------------------------------------------------------------------- +// Greedy assignment +// --------------------------------------------------------------------------- + +static void PerformGreedy(std::vector> &all_nodes) { + constexpr double COST_THRESHOLD = 0.5; + + // Prefer deeper joins first. This avoids selecting a parent early and + // marking its entire subtree processed before child joins can be labeled. + using Entry = std::tuple; + std::priority_queue, std::greater> heap; + + for (auto &n : all_nodes) { + n->cost = ComputeCost(*n); + if (n->cost < COST_THRESHOLD) { + int depth_priority = n->max_depth - n->depth; // smaller => deeper + heap.push({depth_priority, n->cost, n.get()}); + } + } + + while (!heap.empty()) { + auto [depth_priority, current_cost, v] = heap.top(); heap.pop(); + (void)depth_priority; + + if (v->processed) continue; + if (current_cost != v->cost) continue; // stale entry + + if (v->children.size() >= 2 && + v->children[0]->label != 0 && v->children[1]->label != 0) { + v->label = 2; + } else if (current_cost < COST_THRESHOLD) { + v->label = 1; + } else { + break; + } + + ProcessSubtree(v); + + if (v->parent) { + double old_cost = v->parent->cost; + UpdateParentFeatures(v->parent); + double new_cost = ComputeCost(*v->parent); + if (new_cost != old_cost) { + v->parent->cost = new_cost; + if (new_cost < COST_THRESHOLD) { + int parent_depth_priority = v->parent->max_depth - v->parent->depth; + heap.push({parent_depth_priority, new_cost, v->parent}); + } + } + } + } + + // Final pass: promote nodes whose both children are labeled to label=2 + for (auto &n : all_nodes) { + if (n->children.size() >= 2 && + n->children[0]->label != 0 && n->children[1]->label != 0) + n->label = 2; + } +} + +// --------------------------------------------------------------------------- +// check +// --------------------------------------------------------------------------- + +// Returns true if the plan contains a projection over a multi-join with +// mat_mul(list_concat(...), W) where list_concat has 2+ parts. +static bool HasGreedyPattern(LogicalOperator *op) { + if (!op) return false; + + if (op->type == LogicalOperatorType::LOGICAL_PROJECTION) { + // Check if any child is a comparison join + bool has_join_child = false; + for (auto &child : op->children) + if (child && child->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) + has_join_child = true; + + if (has_join_child) { + auto &proj = op->Cast(); + for (auto &expr : proj.expressions) { + if (!expr || expr->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &f = expr->Cast(); + if (!IsMatMul(f) || f.children.empty()) continue; + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) continue; + // Check that the argument is a list_concat with 2+ parts + auto &arg = f.children[0]; + if (!arg || arg->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &lc = arg->Cast(); + if (IsListConcat(lc)) { + auto parts = FlattenListConcat(*arg); + if (parts.size() >= 2) return true; + } + } + } + } + + for (auto &child : op->children) + if (HasGreedyPattern(child.get())) return true; + + return false; +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// Public interface +// --------------------------------------------------------------------------- + +bool MLGreedyFactorizationOptimizer::check(OptimizerExtensionInput &input, + unique_ptr &plan) { + return plan && HasGreedyPattern(plan.get()); +} + +bool MLGreedyFactorizationOptimizer::apply(OptimizerExtensionInput &input, + unique_ptr &plan) { + if (!plan) return false; + + // Bottom-up: rewrite children first + bool changed = false; + for (auto &child : plan->children) + changed |= apply(input, child); + + if (plan->type != LogicalOperatorType::LOGICAL_PROJECTION) return changed; + auto &proj = plan->Cast(); + if (proj.children.empty() || !proj.children[0] || + proj.children[0]->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) + return changed; + + // Find the mat_mul(list_concat(...), W) expression in the projection + BoundFunctionExpression *mat_mul_expr = nullptr; + unique_ptr *mat_mul_slot = nullptr; + for (auto &expr_ptr : proj.expressions) { + if (!expr_ptr || expr_ptr->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &f = expr_ptr->Cast(); + if (!IsMatMul(f) || f.children.empty()) continue; + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) continue; + auto &arg = f.children[0]; + if (!arg || arg->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &lc = arg->Cast(); + if (!IsListConcat(lc)) continue; + auto parts = FlattenListConcat(*arg); + if (parts.size() < 2) continue; + mat_mul_expr = &f; + mat_mul_slot = &expr_ptr; + break; + } + if (!mat_mul_expr) return changed; + + auto *bd = dynamic_cast(mat_mul_expr->bind_info.get()); + const idx_t n = bd->n; + const std::string &W_file = bd->weights_file; + std::string mode_str; + switch (bd->mode) { + case ML_MatrixMultiply::MulMode::Dense: mode_str = "dense"; break; + case ML_MatrixMultiply::MulMode::InputSparse: mode_str = "sparse"; break; + default: mode_str = "dense"; break; + } + + // Flatten list_concat into ordered parts + auto parts = FlattenListConcat(*mat_mul_expr->children[0]); + if (parts.size() < 2) return changed; + const auto &concat_proto = mat_mul_expr->children[0]->Cast(); + + // Validate all part lengths are known + for (auto &p : parts) + if (p.length <= 0) return changed; + + std::vector part_row_prefix(parts.size() + 1, 0); + for (idx_t i = 0; i < parts.size(); ++i) { + part_row_prefix[i + 1] = part_row_prefix[i] + static_cast(parts[i].length); + } + + // --------------------------------------------------------------------------- + // Build join tree + // --------------------------------------------------------------------------- + std::vector> node_storage; + JoinTreeNode *root = BuildJoinTree(proj.children[0].get(), nullptr, -1, + static_cast(n), node_storage); + if (!root || node_storage.empty()) return changed; + + // Set num_features on each node (total k = sum of all part lengths) + int64_t total_k = 0; + for (auto &p : parts) total_k += p.length; + for (auto &node : node_storage) node->num_features = total_k; + + // Compute depths + CalculateDepths(root, node_storage); + + // --------------------------------------------------------------------------- + // Run greedy + // --------------------------------------------------------------------------- + PerformGreedy(node_storage); + + // Check if anything was labeled — if not, nothing to do + bool any_labeled = false; + for (auto &node : node_storage) + if (node->label != 0) { any_labeled = true; break; } + if (!any_labeled) return changed; + + // Conservative safety guard: if any join already carries a projection map + // (i.e., columns have been pruned/reordered upstream), skip this rewrite. + // The greedy rewrite currently appends columns on join children and assumes + // stable pass-through bindings. + for (auto &node_ptr : node_storage) { + auto &join = node_ptr->plan_node->Cast(); + if (join.HasProjectionMap()) { + return changed; + } + } + + // --------------------------------------------------------------------------- + // Apply push-downs + // Iterate nodes in storage order (post-order: children before parents). + // For each label=1 node, figure out which flat parts belong to each side, + // build a mat_mul for that side, and insert a projection on the join child. + // + // We track, for each node's plan_node (join), the column indices of inserted + // partial results so the parent can reference them. + // --------------------------------------------------------------------------- + + // Map from plan_node pointer -> {left_col_idx, right_col_idx} of inserted projections + // -1 means not yet inserted on that side. + struct InsertRecord { + idx_t left_col = idx_t(-1); + idx_t right_col = idx_t(-1); + }; + std::unordered_map insert_map; + + for (auto &node_ptr : node_storage) { + JoinTreeNode *node = node_ptr.get(); + if (node->label != 1) continue; + + auto &join = node->plan_node->Cast(); + + // Determine which flat parts land on left vs right side of this join + std::vector left_part_indices, right_part_indices; + for (int i = 0; i < (int)parts.size(); ++i) { + std::unordered_set refs; + CollectTableIndexes(*parts[i].expr, refs); + + for (idx_t t : refs) { + int s = TableSideOfNode(node, t); + if (s == 0) { left_part_indices.push_back(i); break; } + if (s == 1) { right_part_indices.push_back(i); break; } + } + } + + // Helper lambda: build mat_mul for a set of consecutive part indices + auto build_side_mat_mul = [&](const std::vector &indices, int side_id) + -> unique_ptr { + if (indices.empty()) return nullptr; + + idx_t row_count = 0; + for (auto idx : indices) row_count += static_cast(parts[idx].length); + + std::vector W_slice; + W_slice.reserve(row_count * n); + + const idx_t row_start = part_row_prefix[indices.front()]; + for (auto part_idx : indices) { + const idx_t start = part_row_prefix[part_idx]; + const idx_t end = part_row_prefix[part_idx + 1]; + for (idx_t row = start; row < end; ++row) { + auto row_begin = bd->weights.begin() + static_cast(row * n); + W_slice.insert(W_slice.end(), row_begin, row_begin + static_cast(n)); + } + } + + std::string suffix = (side_id == 0) ? "_left" : "_right"; + std::string W_path; + if (AreConsecutivePartIndices(indices)) { + W_path = WriteSplitWeightCSV(bd->weights, n, row_start, row_count, W_file, suffix); + } else { + std::ostringstream key; + key << "parts"; + for (auto idx : indices) key << "_" << idx; + W_path = WriteCustomWeightCSV(W_slice, row_count, n, W_file, suffix, key.str()); + } + + unique_ptr input_expr; + if (indices.size() == 1) { + input_expr = parts[indices[0]].expr->Copy(); + } else { + input_expr = BuildConcatFromParts(concat_proto, parts, indices); + } + + return BuildMatMulExpr(input.context, std::move(input_expr), + W_path, mode_str, W_slice, + row_count, n, bd->mode); + }; + + InsertRecord rec; + + if (!left_part_indices.empty()) { + auto mm_left = build_side_mat_mul(left_part_indices, 0); + if (mm_left) { + idx_t col_idx = 0; + idx_t table_idx = idx_t(-1); + if (InsertProjectionOnSide(join, 0, *mm_left, col_idx, table_idx)) { + rec.left_col = col_idx; + } + } + } + if (!right_part_indices.empty()) { + auto mm_right = build_side_mat_mul(right_part_indices, 1); + if (mm_right) { + idx_t col_idx = 0; + idx_t table_idx = idx_t(-1); + if (InsertProjectionOnSide(join, 1, *mm_right, col_idx, table_idx)) { + rec.right_col = col_idx; + } + } + } + + insert_map[node->plan_node] = rec; + join.ResolveOperatorTypes(); + } + + // --------------------------------------------------------------------------- + // Replace the top-level mat_mul with a tree of list_add_vv over + // BoundReferenceExpressions pointing at inserted partial results. + // Walk label=2 nodes (and the root join) to collect references. + // --------------------------------------------------------------------------- + + auto &top_join = proj.children[0]->Cast(); + top_join.ResolveOperatorTypes(); + auto top_bindings = top_join.GetColumnBindings(); + + auto find_top_index = [&](const ColumnBinding &binding) -> idx_t { + for (idx_t i = 0; i < top_bindings.size(); ++i) { + if (top_bindings[i] == binding) return i; + } + return idx_t(-1); + }; + + // Find all nodes with inserted projections and build list_add_vv chain. + // Resolve each inserted column through the top join's current binding map, + // then reference the matching position with BoundReferenceExpression. + std::vector> partial_refs; + + for (auto &node_ptr : node_storage) { + JoinTreeNode *node = node_ptr.get(); + if (node->label != 1) continue; + auto it = insert_map.find(node->plan_node); + if (it == insert_map.end()) continue; + + auto &jn = node->plan_node->Cast(); + jn.ResolveOperatorTypes(); + auto node_bindings = jn.GetColumnBindings(); + auto left_child_bindings = jn.children[0]->GetColumnBindings(); + + if (it->second.left_col != idx_t(-1) && it->second.left_col < node_bindings.size()) { + const auto node_binding = node_bindings[it->second.left_col]; + const idx_t top_idx = find_top_index(node_binding); + if (top_idx != idx_t(-1) && top_idx < top_join.types.size()) { + partial_refs.push_back( + make_uniq(top_join.types[top_idx], top_idx)); + } + } + if (it->second.right_col != idx_t(-1)) { + const idx_t node_idx = left_child_bindings.size() + it->second.right_col; + if (node_idx < node_bindings.size()) { + const auto node_binding = node_bindings[node_idx]; + const idx_t top_idx = find_top_index(node_binding); + if (top_idx != idx_t(-1) && top_idx < top_join.types.size()) { + partial_refs.push_back( + make_uniq(top_join.types[top_idx], top_idx)); + } + } + } + } + + if (partial_refs.size() < 2) return changed; // need at least 2 to combine + + // Fold into list_add_vv(list_add_vv(...), last) + unique_ptr combined = std::move(partial_refs[0]); + for (idx_t i = 1; i < partial_refs.size(); ++i) + combined = BuildListAddVVExpr(input.context, std::move(combined), std::move(partial_refs[i])); + + *mat_mul_slot = std::move(combined); + return true; +} + +} // namespace duckdb diff --git a/src/optimization/register_optimizers.cpp b/src/optimization/register_optimizers.cpp index 87fc20e..ebd4366 100644 --- a/src/optimization/register_optimizers.cpp +++ b/src/optimization/register_optimizers.cpp @@ -24,6 +24,7 @@ #include "optimization/actions/MultiLayerUDF2TorchNNRewriteAction.hpp" #include "optimization/actions/MLDecompositionPushdownRewriteAction.hpp" #include "optimization/actions/MLFactorizationRewriteAction.hpp" +#include "optimization/actions/MLGreedyFactorizationOptimizer.hpp" namespace duckdb { @@ -156,6 +157,18 @@ struct CactusDynamicOpt : public OptimizerExtension { } } + if (opt_setting.find("5") != std::string::npos) { + if (MLGreedyFactorizationOptimizer::check(in, plan)) { + std::cout << "\n[MLGreedyFactorization] BEFORE:\n" << plan->ToString() << "\n"; + auto rewritten = MLGreedyFactorizationOptimizer::apply(in, plan); + if (rewritten) { + std::cout << "\n[MLGreedyFactorization] AFTER:\n" << plan->ToString() << "\n"; + } else { + std::cout << "\n[MLGreedyFactorization] SKIPPED: no safe rewrite for this plan shape\n"; + } + } + } + // pretty = plan->ToString(); // std::cout << "\n[CactusDynamicOpt] Final Plan:\n" << pretty << "\n"; } diff --git a/test/sql/greedy_factorization.test b/test/sql/greedy_factorization.test new file mode 100644 index 0000000..cf7aa99 --- /dev/null +++ b/test/sql/greedy_factorization.test @@ -0,0 +1,105 @@ +# name: test/sql/greedy_factorization.test +# description: Test MLGreedyFactorizationOptimizer (optimizer option "5") +# 3-table join with mat_mul(list_concat(t1.f, t2.f, t3.f), W, 'dense') +# group: [sql] + +require cactusdb + +statement ok +LOAD cactusdb; + +# ------------------------------------------------------------------ +# Weight matrix W: shape (6 x 2) +# t1 contributes k1=2 rows, t2 k2=2 rows, t3 k3=2 rows +# +# W = [[1, 0], <- row 0 (t1 block) +# [0, 1], <- row 1 +# [1, 0], <- row 2 (t2 block) +# [0, 1], <- row 3 +# [1, 0], <- row 4 (t3 block) +# [0, 1]] <- row 5 +# +# For input [a,b | c,d | e,f]: +# result = [a+c+e, b+d+f] +# ------------------------------------------------------------------ + +statement ok +CREATE TEMP TABLE w_greedy(c1 DOUBLE, c2 DOUBLE); + +statement ok +INSERT INTO w_greedy VALUES + (1,0),(0,1), + (1,0),(0,1), + (1,0),(0,1); + +statement ok +COPY w_greedy TO '/tmp/w_greedy.csv' (FORMAT CSV, HEADER FALSE); + +# ------------------------------------------------------------------ +# Base tables: +# t1: id + 2-element feature vector +# t2: id + 2-element feature vector +# t3: id + 2-element feature vector +# ------------------------------------------------------------------ + +statement ok +CREATE TABLE t1_greedy(id INTEGER, f DOUBLE[2]); + +statement ok +INSERT INTO t1_greedy VALUES (1, [1.0, 2.0]), (2, [3.0, 4.0]); + +statement ok +CREATE TABLE t2_greedy(id INTEGER, f DOUBLE[2]); + +statement ok +INSERT INTO t2_greedy VALUES (1, [10.0, 20.0]), (2, [30.0, 40.0]); + +statement ok +CREATE TABLE t3_greedy(id INTEGER, f DOUBLE[2]); + +statement ok +INSERT INTO t3_greedy VALUES (1, [100.0, 200.0]), (2, [300.0, 400.0]); + +# ------------------------------------------------------------------ +# Baseline: compute expected results WITHOUT optimizer +# result for id=1: [1+10+100, 2+20+200] = [111, 222] +# result for id=2: [3+30+300, 4+40+400] = [333, 444] +# ------------------------------------------------------------------ + +query I +SELECT mat_mul(list_concat(list_concat(t1.f, t2.f), t3.f), '/tmp/w_greedy.csv', 'dense') AS result +FROM t1_greedy t1 +JOIN t2_greedy t2 ON t1.id = t2.id +JOIN t3_greedy t3 ON t1.id = t3.id +ORDER BY t1.id; +---- +[111.0, 222.0] +[333.0, 444.0] + +# ------------------------------------------------------------------ +# Same query with greedy optimizer enabled — result must be identical +# ------------------------------------------------------------------ + +# Intentionally disable built-in optimizers (but not extension) so profile 5 +# runs without projection-map-producing passes. +statement ok +SET disabled_optimizers='expression_rewriter,filter_pullup,filter_pushdown,empty_result_pullup,cte_filter_pusher,regex_range,in_clause,join_order,deliminator,unnest_rewriter,unused_columns,statistics_propagation,common_subexpressions,common_aggregate,column_lifetime,limit_pushdown,top_n,build_side_probe_side,compressed_materialization,duplicate_groups,reorder_filter,sampling_pushdown,join_filter_pushdown,materialized_cte,sum_rewriter,late_materialization,cte_inlining'; + +statement ok +SET enable_extended_optimizer = '5'; + +query I +SELECT mat_mul(list_concat(list_concat(t1.f, t2.f), t3.f), '/tmp/w_greedy.csv', 'dense') AS result +FROM t1_greedy t1 +JOIN t2_greedy t2 ON t1.id = t2.id +JOIN t3_greedy t3 ON t1.id = t3.id +ORDER BY t1.id; +---- +[111.0, 222.0] +[333.0, 444.0] + +statement ok +SET enable_extended_optimizer = ''; + +statement ok +SET disabled_optimizers = ''; From 2b1ebf8ade24d1b2ffa3bc2b9d616facb372f52a Mon Sep 17 00:00:00 2001 From: jrt1899 Date: Mon, 25 May 2026 22:40:59 -0700 Subject: [PATCH 3/6] Frontend added --- Frontend/__pycache__/app.cpython-312.pyc | Bin 0 -> 31751 bytes Frontend/app.py | 778 ++++++++++++++++++++ Frontend/archive/app.py | 454 ++++++++++++ Frontend/archive/index.html | 611 +++++++++++++++ Frontend/templates/index.html | 661 +++++++++++++++++ queries/synthetic_query/synthetic_query.sql | 2 +- 6 files changed, 2505 insertions(+), 1 deletion(-) create mode 100644 Frontend/__pycache__/app.cpython-312.pyc create mode 100644 Frontend/app.py create mode 100644 Frontend/archive/app.py create mode 100644 Frontend/archive/index.html create mode 100644 Frontend/templates/index.html diff --git a/Frontend/__pycache__/app.cpython-312.pyc b/Frontend/__pycache__/app.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b2ecb8ea028e1ccb9c6bf705bece59d1208d113 GIT binary patch literal 31751 zcmeHwZEzb$cHj&?2oL}X@SBtfdDl# z6h$n#rgwd3YNIPjS#3mTwP$*>sg=vVing+MQIblcP1d^xl9yCa|LhHa_K{btc5EKy46Zq>yQT^9Ag{Q82y*?uzBy;-ZsfG zG{t$i!m&C%T@A7JeGr5)HmfoE@0eE)tD&D6a;sXB{>zoCEzJHMs#v8yzn)tSaTRR7 z+TNk6C(7s0@~vT=uTn#6p;y?c_E#TIjrv(#?S0U@ZZe~4f5nD_` zC2T2MMndIm1zSl%RqPseEeTb#>)09+TF)|UEeUO4H?nmkw29r!)?;WSf2d}JVjI3o z53Ogt>=t$_+sHPtzUx-DnQdYHY%9BsJZ)!pushjZ>@)Z@Vjg1H-RvHAFT3x$IaE7J zvHO3W4sC#v_CZM-*#qpe>_JQ$s)IJPv4_}pwu9|tyRKU!^w1{uFnfgUW{*PpX2|p0 zcWJoykb5t=2gtpT+>gQC%pSjP8LDUdxrSjAJMgMy$P2V}LLCSFmmOqJjL`3#RC|{_ z^(v+I>d@9z@;tB70Q-Ugx^WfgAr1OP19a0W&|hL-%E0HB40(L3f)mGj z`r0~zU41nbD# zvjJfU4>)w zs{p-6;X3P>aR9yzGmcy(sV$my&N%s!8K+WSncikKq(a7M#Gj^Ow1t|nvkt;(<{8^; z-o**l3GJ#-bLhWn-g6XBU;O%vbqtZtx@NSt)6~UpLD|-sykXM-wUP2u0zCyY$O{w= z|1Q%PYtNfspr+_B#Dz_><|%*?*QmwSQM_-~C047?V>N0>{~fERtF8g%cZ!~(*T})2ZA*kpMsyf5In_LDZlf4m;>zMG06;3QeHGZG6Lzg zsSqEF#74XY5IY=8kxH zavV?*n){9o9^zu*(dI6mV8=E?1AG(HQYoMaf14x434z(eR2ni6qbJU7uy|wag zLD6z?U8-Qy3PtCyzfqJfuedgRW%!$;^N0V@3^5<)Q6**9>aWyaYrN8U*HxKZ)BV6~ zDsjx)Rv=}~wdO0$Y1jUAq5HCJetLnP4?M7%NvfUN)cF10H+yexOx5lDOHaqb*;}=L zSpPfqzwfIn-7Oc5!F-F}eTB_W7r_#Sv=}%Q|yKI$8D$~`qZ#?(7vG)mfv>#!0}tIrJ{Xl zPw9uA%4_Z`?nTS{o=q$30HKu{%2T;eo^-EI+SmW%F6fqf*jxX6zmzKKq<{Kx9px&! zPnn=M?aoV%3ywSXHB0t2$+ZV=)64clX?M}(?epjVo%^3wtN`<=5P&iGn!B@vx&pb) z-)$>{;E#5dbrxFwsGu0{>)Wgl^~amq*22%bw6)V|dpEBD!zKA$4)eR!8@sILcb}yp z{3li`+@*Xz6gvZ&#aVx1>+xVedDw&ew6(W~!ABrurXdhu;^#P?8RkMDVK~7XT7bz| zW?2W4q2P;*12GGz4Fn^sMwx8`Oh!T@fgbloLBW8MgM56-01nWZ2`cym4{Xhg(?M~1 zf-@GLk)S|FgCTxoJaoQ+)WtAiU_2o{EE=L1W;`SYLoBPpV3`;-J{dJe<%C-m784Nv zAh3KmBxV+biP%q*0`xYZjk;Ly92b_6mQ&Tc;1jqIA0EYmM3985!>Qe+3=9G)tyaro z81KVXtSYf%F%$ryaUtm6(GuL@Zws zoD)Mh$1<2U4#k`p=nfoa8r1QQuwF-BKj`Glp%=25Z?D%ZnIf!YK~-Neg8-NE&%%T{ z0V{=2V#D!ZryShFuv|ET9LtD2$1#!EFbC_5Fc$()=RiTpG+Nk$v0;d!VtHljqF2GPo5QPD<*KnKIoNcarsWAc+)v5NXM5Jn?2 z4lF;~$V_qEnfS0A1qu+L54CfowsSCM&<%R;8BrnKY;3>W20LY0rFMc*8n*oUuMgT)eM4V)ib)xauh!SS?XjYGgzQ7A5CWs;Y z0l9-e+d)Vu{DmPga*k8^RR~8!gw8mZh{v!-RuEMG_xK<5k4J4VMH>r(D-^0br$43GRP0bAa}OAG7%}Un}NoB{g8aR?OFm`C74- zR?N|gIa(oyj25A~cCZu*GR4Vl3$}-NR-fOMUOQGTh>y#661R!4vH*{yofdy|8cXtY8y*2kF+sE;RGl?R!Z5wvfrifK2m9OF2OD1KZ0l!;BGukDFxUV}5{LZL(KguW z^^!(KV&~$h6_Cjj3WB0e^fsKG48=emOfy?#rGbFr@FVZk^G z_7t#F%2^wk7H?*DIX$bCcrZK(ddfKH`NA2(+v=k_7srEf9?iKxxNrgX>LXx5nnt7} z7<)kCxJVKf43DnqgRNvaPTVJMprybKA`BbaDKL_r7D zBcWJQ;Rcd>10uLeO_-bY=`8}V!~K0Hj-g_j(G!jEsNn)?wmjyjF~xI@xz>kjr?s6t z+@K7I*5OPu6FAY+)7{meMd2XHW2+1fRscwmel^mW5m7^B4h^+oVq=emr&lLTLq3^3 zAEv$bz7v7LhK}xm!R|o&Aa)iRc!DB%5Q4C@CB*s!ST@46t5bOnoRO-sxac8du}8B| zAZTd~&qPRzR{G-+9jMv~f_!Ky2PVAlAzX%iTJdsE@;-I2A09x3Xn4SYP^mnuS^>;3 zBoGxbqaq;mVPnPEYbv5SkbDS5kY6CYf?z3&PB+w}q=RKdD9Z8mAXEGM+g^B4!=RBd zBL9-ytBUPJ?NqgD>?4QQp^3Y!de4lb00k)plTcVx>Qy4gL1~CW?P^h^o^TYz4WThv zu&arxeBL&R<~$;wx9wp%RMQSD{Gc=z7C<0YKu)t98XM7Es1+aD2?`#O*J%39(PbpB zJbZd~`8hT@j*AXl*2obA7Q|0bu9$Bi;+(`{$=!j$&cmJkGAn1nDvLe63at!1%8L}B zTu_{zpuj3+kV*}>fk5%^*y(FQ6#6^cdemaz(Et^U#d%zm%jwLDi#9zu=K0iyq6ZqmcyGm;#s1&2lNCVVoqm?WZc$UuhMf;s6=0fCjWPJb%x zJ(KVf!h6O*ctiqeG{!V2raEuvH^9V5s0|=8yVNj~G%^`#z+iW8=fGfF?=jK^A}Wz& zkd%20M+i!5eME7%UxLe4g;*OzffR*=3(;zMPzaVa=Fcn%k{Kp$mmfp~>RMsoF;rW(i{BodVXl{@)a z$3cG9@FUGK>UU(cKrUw(DNun_ng|>X%$zlxV`x$^s4avhiLr%{gC+_a8MR_;3QE?; zXp7zoBWJ4x;Q~@aR>ccQP^&YSaQiFU;!q z`&$y5kh%RWGII~}p-DI@!-=rJUuCZy+IJbfHcVy=`EKVA}ciL6Cxyr z!Dbi>N5Hyhz(F9)lnc-EJ_8sM3!z2vyl=JIAW5!Jh8zxM6cB}NPE_Z!Pzi;J-GCVm zL7J@XoJVGZ+{2-amBxMsD$siB5d@kGZ6j&QNWxTT1SIG+G7q=yvuJV5NQuT0*TE!ISW>(&1kYAo8fVsaXtqz8;SF|+R_tf zKmiFO+=OKlSzBu$*@%!E_;3LRWU&yn042YcO3F~1Lg)+E&h7nu0|PPxD^$YzNMmIE z0)jHxAU){|e$o>}6q8*bvZxuC^TF&2ndC6!(Pl5~0wc@8)+`ZCIQ{@fNDgSgm)T~p z&rIl1gq8>}z#mY6GaIO7(4zHB8FU530A*n>E0YHt>cF9pwZn|@6dZtxTA@-X5qZKCbQ|9h($JVW%al zj`h7E5en3VU|H^_0v_(?ovk_lmr#o@qAq9PDyw1YED19l~v zl)QQ3{T;R{=}v}_3RE90^v9TY&Ys%z)UE8kl2L1!N5B7 zGzXHBV1h?;mG~IXg{f7gBzbs+y>4p|F&iD`(%8JIixkUB*CWE2izI1(YbWY{f+ zebAQ}?6zPSEV!V^!11MNFfG6+;cgDswBw;tABQF#iQjpWD!`zQc&qp*rUy{CVsL;f zuAoEA$J<(-4j)6WducqFNkGZ5EkNOEXFzaiWjVcxV+ zk%I+hYNV;X`X4^px#iiBe}3!Ud%dT>{H$b#lM-I5RDu1jY#;^=oBL83Nl-WL)s20r zJojKpWjQ-dDo~@RBCI$%Ih{{YA6>op(bc)1_y5a|kFI`=OpxS84jBZ&maATnszUrM zRYP%xl9=o>403HU#MQ^H=bgufT!1z_e((M8vCGZuJ>_g+H>m5%WoYwbmziUTvwM#m zzEF-|340!{3u1JVgr!PO2$e+9l&*pp@P#~}i9K`-ONSWO4qh2fTtO}kq5)4>;h`)1 z1(~is>ktg1r6D$W{s~Tq7|Vcg!$U`?AktWddc?+fdMX1#ZK=vraB9li$kc0P;DR{k zDJTtskq;9w$^jzG4Vg~+bvy`w60e>;LOgYd?9B4vAyi4_9{X2H_w9YX$J+Y42l@g* zvRutDWl7r@VN z#!{+!67reL0*3KW2}B>7LDc+I$t^=ta$>(V72u>i|x)#|#0x+2q7VakdPsZc9Il~k+g5?W>i z%vi!{Md&?j2tlf%(fE8mTsg@vXFv#_(NeEr@$=FpoHlB!Jw@G<_iF3E(DyX<9_&Sg}-g^w;ld=z~4^z+Xa8mz~64keB9rHpyN^g z8O%DW-Bx|X1gH)@Z}BJbb_ydLBXFnYZhw66e_?={_|FGP4At|$m6)EtP-5m&O5o~} z8uPbGVCH=#(DT=7(r+D9%BFt!juQ9$?3|NGz9=PB%GkD;_b6Lb)5aa?sVIs%+@C1u`n1_i7;BN=d-@)M+l7CefJ@{rj zyc8dg;@f@9)Mx|_)aYknun-13+i-$UstQHntWR(f+=q?C;6Rv$FL6MY2gM9WMiP1G ze2l5*(XCdZcA#^RS+#1|U#o{x7adq=V@T;6%zm#p&MC`uM>UA8pd^;F-dXzSW}OWN+bbI(11dcavyvt&NpiSF5KV0amgvqBi- z)eAk65waf!)LwG};j1ZBF2c`A(zUH07G;4X)uIK2` z`~s?EK1+$j+a~77Ro{$KenhKbmVOGXt!$o(;jHN?^4nOumf!po`ST6=Ei)F~H{3az zH?j6vLrl(Iu&|o%%2_M=uAI#i3zcrBzz>{F@dM{UUu`osc}zrjlCN{%jqIX(#s(hV ziowI1eb#28X7XlChF-}hF=hjv4iN;Ee6cQl48>A4cE&yf0%|6I#sPlMEbkP&t&HRV zcsL_o&TLbZw=e*&N23IRJq_Z;i<}b7kj6-o$UzUb!|hx2bMT(@HE_`P5^yxiT9&w# zx?n=5ZO{zY4E?M0H5weQ6gm) z-1=$$z7a0Q!4myIq7pn>aOZvd(9=j%IN;M#kWGYjfYeX_j{4~w^QdRFK4^Xy$y`Yl5VymR3B2AVBquemQJ1ZX{E7s$|`v^F@fCM~rCKbUE zzetRn7Qj<}7>{26!p4~hT&++z7!O{B+iTQo^fl^&fq9XLF3{hgMC?g&%~Qa7G_W4# zB$v*5tjFo?>mZ=Jf8mufn2wQZ^7*-!Wj0Ps0C zh`)rba;OJ7Bka%AtaDk-D(_l2E|cc1J*k$?t7n(WyX0m8OO(Pp@!cmogW%DvySIC& zvp?9~F#xWAU_ylxjB4Zn^!#POk^dD$)C}E8F#PhDmHEx4baJ9Rcjuo)l=eg3Pv)q! zvuL@hY024?E?Ym}b+@oMRr2ih#B#~Aw@XunUGwI&vv7Xz-Qx08dDr*0ESGnsio539 z)9&JV2R{QPBph-TqvCkfo6mz6W=gWj2VDhz0-p*q*_+IFaJ*HJDzpS>fb0~IFrrkM zBY|+DR6tJ328CDzeD;Y_MJ7WkWx#-g-^5QMoK+robXVSgEK z7xDHTyj{Z^A*CYhkaL`VG+5xjjfvOs_8Qy-obN$BQLlrto?<0^%YPpdQOF4OprwLz z`$^^cH(al~?o>7`RW{sM^TSiOj{jijwqh4AUVW#^yHw>(ZaHu}FI9Co zX(@jQemGUx3ALqdmCMYICEJd4^@gOq>Tap`PHEFpX;aeo?Cs)IX~$geeM_FTVEyJ~{q9>k zQ|k{UE!979u1VWV?|LfWf4ZbDU9lA1^Sa#-DB%; z7Vsep7~%DeJcGWL2jV{u#J{GI872+Q$2d;Ob8ZoxOOlJVv-uHPogP>R`acD&%}IQ$ zzM^X>{8#cRQS)aD)4Gf3jq0Kbrj=YCVc7z`gl|*d(UwPJkMQ;io^VZ%^5_c7F-=Eh z3ty#X%!YQlbNP8?J(*ByYt>!^{Y~$Wq8W?c9|M%QLG7uWnqBS%9{3qo%Vx{ekg=8- z3kx2|uWS8*K4&xh$!?joK7~Ko;u(!H(046so3(z`dhxXUYw#mmnRvNh_9KfcO67f# ziP32R{jTXRjD+OH<5o=+!y+-l`y42dx?pt&uS0~wXYhRrECxIqf&U|1h#u={3&8vM z;LCuC48wcN@P?keM1#XK@KR$WD){(Upj&l?QgUf^s4CYlVm{~J0xWV&w*rh&yGEDyvozWwSW45bP1)MD*CZ9FLmR}_1Tyus zGZKAH=>l0(kX5wL?t<&s1#1B(S-bY^yCPmbyHLA$_C{Z-e(%+yCC|Q31XQlSR!~7- zwpGx4F+AjM8h+M5YZ`F3)-<&_PFO5ZxLEMgS@;L}n~?dbTJRlg0j`IX^+D5{mPLBe zkL!Y_PXwIhzveEY!Iv=fTB5E8=@^5AggNjDh;JN^z*97ee1P7G`CiCGEQ^U^ct49k zht2`TF7N=y^{6vo9xsB_p2D6z<3KBL=y-`JkplT4LIu| zgENexa1>@T1OK}K@XOe9`+#~u^8o9Jr{dDgg_%3<^-J#ci;i~=-)wv9=uL5{abL>4 zf9}X#a0R^Y%Dy`#8<$EpE(UHMPL=GN>-mYJFy(H$zGvCpc6)ouab&JNZF9_h<*vJY z!IrGuo~+oBa__ulPP+Fd?R!78xi9a1&sOsrdsa-2{G;^Ul64E=#ipg2mQ;y>hfGVdg18Bp1UO#A9(gHh|3=I{vLED%p~BZ9(Ht(WTuNc3&R2cIwKhg`pd3Qf1qd zJCCNE&&_q*brvsl++deVwxyigla}p2{Wu@|2ZIF&k#xDS-aD?EC09*yedq17cU;{| zuI_&VYY-%VD&P$IN2a#j9WB(4Tk<8vroTVsWAqL}C=@CIP;G#Elh$3y&? z4D|%3d7_^D9Sp|jqnv;uU^%l3hB{9~Mkbfw7M*IC1;dO!S+X`%9!5YW=>`%oZh%X3 z;^ZF886c}nig80Te<;QclWg1o=jtY~v4WaBYtB$Oz_gK}a9DKn#u&12)}^|CM<@8X zlnK2(sxpvEaZpPEl>-GGxgM|H8EXQ@hs{sGn!{lH$pdWAB9V3J^?=2AeO4ko&Y0z1 zgI_21!b~3Px^A#fUwqv{>7#;X9oA!1E);`~iMXw2)bhb?{ot(4px~gy$hyYrvm)S; zEzt0>y^RzI7mc>=$H1H|6uoi|Xoco}A3m#MQOAKOE`_tUuiC)vzLg~*gX&V0IMP1} z-fThPP#0W^iF*=1pTTh8!+Kir;;7HG#5OMbYhr+-(3`|VI*2?G<4`18h^*Jx4y8O; z$nix_MgTwY;H?t09DJo#b*Ih01KG$V9{D@yk-rDYD8EVzi*nR4j|h+RhGE5u25rfL z&6Zs7de^Y*#ufAuoJsiqgsE2W%ok(%-^Peqyk*TKhyi#S7J08-o=5%wfp96y>obZ; zY?1NB`GidV@>HT4*^6*4$q;G$j{)-gIGvzo0JBNnXUrzrR4}GY6$8nN!Ib+%^3+h$ zJ(RQ$eOTz3JA8NTroVJ|t2){8_9O3kkNgV#p1b=#Rbq9|n^#Q5`R=S)hLwB%}9b~WFvs`;Mf4&z^9{3&MJ&C^TFGq*aH)*rlG zo2u%VKdQ|%X;;amz=c4X*|^y8W`4R7W|Y^R>6(UnHcNHk|Dh}e?iDxS@$nYQRkCQm zd1|R-f693vX*ux8{jHSeD7cNUB-4lukW^=p>Lpioa^0R=?YAwz(|!Bwva5TQS~lFR zs$Mw$?ecs1*6PB41n2l}^pKAp=ApPx1=O{EDYAp%@uWJ1>)JMW;3khv@flwq6JsB`g)Yj+XtC+D|^oMYo3}nW7TQcPgt#PHjmAR zul_jRw#b;N<$&?k8IVf#<&JKyB8KBcQ=e-1!Z-L8(=v)5_%4t*y%)cYF^U2SMh15A zU^grjgD+ALdk}d8I?crHIf(3KFvu?5#pxn=lzhw;RO74U&qOsWhvV z*P$59K+uizi_i=nziYzdkn{LG9Ug~^|8=~fSq#3A!LqP-WkDY(0+0ixSRM;9h=~6S zNcmG__t_M4Gq z%fY+OlBz!E&CswjZvT#1|47_w45sE+e}STMHz(A-*7gYptcU5ehGnkg5AVpXUHqIjTk~R;9DliLXb|ug8&>tsnez>n=tu1 zkoo~K5zf6}{dE>!+J9mH9cT5D6DEnKl+#D}=JNJy`>*U@JoH}4X2L>SZ#=(j-dQv zC4c_81^eYhvUYdUvFBFzvZYI5vQ%kZ%DE|N*(5Vnp~_UU{r?M&g7KpMHF1~u__5+{k%spp;>41X7cpSe`T$0 z9@FL)wG@!|D51zTLnO6Y8m#Orx!`9^Ggib~hr75BR-u|9og6MOgQ44^ukAoaTLWZ2 zkdH%F{w&_c;Rc^UEIQN~Xg|{1*8g0vzjNS3&)|TR&w+bV4jkNYQi+}lyK_O}vP9mD ztA*^6M^El)I|RXHhHvnlOVnN)xT2K#SdnuG66NFMD`Id%riEaD{N zJlRzhaP*Y5jJyw5j7yo!;pTq=Ko5{u(SQe1xp4hr{!-xDXZrSF= zuPl{qn+tqcT0YmCuBgIox3zW4o1RIn-F=@jTlZgZ&)Y7S;~s38_dTaK?W(%tVwPOY za;^X7=B3)*%dS0XSM?p&#w8b;({?S@?S0?1Z^Z^#KX$-^B3-g3U9kc8z{)nk8lW`q z0cCaBRw%30Mr5*N55id|a3d51ZOzJZrzc0e+vHD5p5c+M$Ofmpw$$}JqV^%(NQLl) zv@;2Aa>O?&f-Rp%!jUJnKg7@^!ax-`3kHM47AZO8&rqTlDIO_Avg2VZ_^PEqWIrJ#g?QqcS&PnYL0RF1dpc$>x>S@OP#q3_}CTX_3*ypiqN z|Bj)*!P|H7b_;KJ@%Be}BTHjk{DMQ}?_&hoH_57$XsTqUB~pltu4IPQD>0yxu9Od> z*!O|^HgMN;fWHTMQS%k1pdZ1^N7H{p4c?&!m#D$NrndYQmH$_i^RK9a|B>gMKl1hX zoH<>zVb1(&;fJ28xq>w1oHM=Ve8oB6yV&tQ=Lzh1+1*am!>XE zU3&S#%M1J)Q?F0G@$&00-{609>aD5YeEF@HZ=FvyA4#q6PPvcHxza@?_#L%t)mN(1 z7EiJ`0N-URHq8~nciGA-(TsWROIN-`Ql5pBa#NBjzVE1|EAJhlsp^*0TK^JNIp>-m zTPdX6Yto+mX*bw3Ydq^#sG6E}_iP)ic`E>kN!wRUC7uoUY!wjIN@8}dn2J2*_iSYl z3r%y5m7Uf?%Sxf8z_Q}96h4U-o5<% zGAO@jr6!;DEVMtM;0HcKXR^Ll^-9(JwuQ0<+gw$OX6~8l=)4DJ%2W$|MCU;b3QW>j zoub#>GgZ<%Fu7(0Ln}MA)b%9uIvR+BWQGuSw_HKexpr}Q@p-Jlch6Kn56}qJyh4I2 zj0RP6qx^;)q5Stu`SfN4YF)w5N~H#4EJ=ZZf-+&46kU7IR7~5jgbiT0p=~Q$D2r=8 e@t(OVZLz&}_?5$#>*fx>Z>d-@Q 1 + AND prop_location_score2 > 0.1 + AND prop_log_historical_price > 4 + AND count_bookings > 5 + AND srch_booking_window > 10 + AND srch_length_of_stay > 1;""", + }, + { + "id": "q_flights", + "name": "Q_Flights: decision forest inference", + "sql": """SELECT + Flights_S_routes_extension2.airlineid, + Flights_S_routes_extension2.sairportid, + Flights_S_routes_extension2.dairportid, + udf(slatitude, slongitude, dlatitude, dlongitude, + name1, name2, name4, acountry, active, + scity, scountry, stimezone, sdst, + dcity, dcountry, dtimezone, ddst) AS codeshare +FROM Flights_S_routes_extension2 +JOIN Flights_R1_airlines2 + ON Flights_S_routes_extension2.airlineid = Flights_R1_airlines2.airlineid +JOIN Flights_R2_sairports + ON Flights_S_routes_extension2.sairportid = Flights_R2_sairports.sairportid +JOIN Flights_R3_dairports + ON Flights_S_routes_extension2.dairportid = Flights_R3_dairports.dairportid +WHERE name2 = 't' + AND name4 = 't' + AND name1 > 2.8;""", + }, + { + "id": "q_credit", + "name": "Q_Credit: XGBoost inference", + "sql": """SELECT + Time, + Amount, + udf(V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, + V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, Amount) AS Class +FROM Credit_Card_extension +WHERE V1 > 1 + AND V2 < 0.27 + AND V3 > 0.3;""", + }, + { + "id": "q_uc01", + "name": "Q_UC01: KMeans++ + min–max scaling", + "sql": """WITH table_groups AS ( + SELECT + MIN(EXTRACT(YEAR FROM CAST(date AS DATE))) AS invoice_year, + SUM(quantity * price) AS row_price, + SUM(COALESCE(or_return_quantity, 0) * price) AS return_row_price, + o_customer_sk AS customer_id, + o_order_id AS order_id + FROM lineitem li + LEFT JOIN order_returns oret + ON li.li_order_id = oret.or_order_id + AND li.li_product_id = oret.or_product_id + JOIN "order" ord + ON li.li_order_id = ord.o_order_id + GROUP BY o_customer_sk, o_order_id +), +table_ratios AS ( + SELECT + customer_id, + AVG(return_row_price / NULLIF(row_price, 0)) AS return_ratio + FROM table_groups + GROUP BY customer_id +), +table_frequency_groups AS ( + SELECT + customer_id, + invoice_year, + COUNT(DISTINCT order_id) AS orders_per_year + FROM table_groups + GROUP BY customer_id, invoice_year +), +table_frequency AS ( + SELECT + customer_id, + AVG(orders_per_year) AS frequency + FROM table_frequency_groups + GROUP BY customer_id +), +features_raw AS ( + SELECT + r.customer_id, + f.frequency, + r.return_ratio + FROM table_ratios r + JOIN table_frequency f + ON r.customer_id = f.customer_id +), +features AS ( + SELECT + customer_id, + minmax_apply('uc01_scaler', ARRAY[frequency, return_ratio]) AS features + FROM features_raw +) +SELECT + customer_id, + kmeans_predict('uc01_kmeans_model', features) AS cluster_id +FROM features;""", + }, + { + "id": "q_uc03", + "name": "Q_UC03: DNN over encoded store–department features", + "sql": """WITH base AS ( + SELECT + store, + department, + num_of_week, + store_id_encoder(CAST(store AS INTEGER)) AS store_id_encoded, + department_encoder(department) AS department_encoded, + CAST(num_of_week / 156.0 AS REAL) AS num_of_week_norm + FROM store_dept +), +feat AS ( + SELECT + store, + department, + num_of_week, + CAST(concat(store_id_encoded, department_encoded, num_of_week_norm) AS REAL[]) AS features + FROM base +) +SELECT + store, + department, + num_of_week, + dnn_predict('uc03_model', features) AS prediction +FROM feat;""", + }, + { + "id": "q_uc08", + "name": "Q_UC08: DNN over order features (softmax)", + "sql": """WITH ord AS ( + SELECT o_order_id, CAST(date AS TIMESTAMP) AS ts + FROM "order" +), +ord2 AS ( + SELECT o_order_id, ts, day_of_week(ts) AS weekday + FROM ord +), +j1 AS ( + SELECT + o.o_order_id, o.ts, o.weekday, + li.li_product_id, li.quantity + FROM ord2 o + JOIN lineitem li ON o.o_order_id = li.li_order_id +), +j2 AS ( + SELECT + j1.o_order_id, j1.ts, j1.weekday, + j1.quantity, p.department + FROM j1 + JOIN product p ON j1.li_product_id = p.p_product_id +), +agg AS ( + SELECT + o_order_id, + ts AS date, + department, + quantity, + SUM(quantity) AS scan_count, + MIN(weekday) AS weekday + FROM j2 + GROUP BY o_order_id, ts, department, quantity +), +feat AS ( + SELECT + o_order_id, + date, + CAST(concat(quantity, scan_count, weekday, department_encoder(department)) AS REAL[]) AS features + FROM agg +) +SELECT + o_order_id, + date, + softmax(dnn_predict('uc08_dnn_model', features)) AS prediction +FROM feat;""", + }, + { + "id": "q_uc10", + "name": "Q_UC10: DNN fraud detection (sigmoid)", + "sql": """WITH tx AS ( + SELECT + transaction_id, + sender_id, + CAST(hour(CAST(time AS TIMESTAMP)) AS DOUBLE) AS business_hour_norm, + amount + FROM financial_transactions +), +j AS ( + SELECT + tx.transaction_id, + tx.business_hour_norm, + tx.amount, + fa.transaction_limit + FROM financial_account fa + JOIN tx ON fa.fa_customer_sk = tx.sender_id +), +feat AS ( + SELECT + transaction_id, + CAST(concat(amount / transaction_limit, business_hour_norm) AS REAL[]) AS features + FROM j +) +SELECT + transaction_id, + sigmoid(dnn_predict('uc10_dnn_model', features)) AS prediction +FROM feat;""", + }, + { + "id": "q_uc04", + "name": "Q_UC04: Naive Bayes spam detection", + "sql": """WITH docs AS ( + SELECT + r.id, + ARRAY( + SELECT stem_token(t) + FROM UNNEST(tokenize(clean(lower(r.text)))) AS t + WHERE t <> '' AND t !~ '^[0-9]+$' + ) AS tokens + FROM review r +), +base_priors AS ( + SELECT + LN(SUM(CASE WHEN spam = 1 THEN 1 ELSE 0 END) * 1.0 / COUNT(*)) AS log_spam_prior, + LN(SUM(CASE WHEN spam = 0 THEN 1 ELSE 0 END) * 1.0 / COUNT(*)) AS log_non_spam_prior + FROM uc04_train_preprocessed +), +token_scores AS ( + SELECT + d.id, + SUM(LN(COALESCE(m.spam_prob, 1e-9))) AS log_spam_score, + SUM(LN(COALESCE(m.non_spam_prob, 1e-9))) AS log_non_spam_score + FROM docs d + CROSS JOIN UNNEST(d.tokens) AS tok(token) + LEFT JOIN uc04_model m + ON m.token = tok.token + GROUP BY d.id +) +SELECT + s.id, + CASE + WHEN s.log_spam_score + p.log_spam_prior + > s.log_non_spam_score + p.log_non_spam_prior + THEN 1 ELSE 0 + END AS predicted_spam +FROM token_scores s +CROSS JOIN base_priors p +ORDER BY s.id;""", + }, +] + +# All available actions +ACTIONS = [ + "TorchNN2TwoLayerUDFRewriteAction", + "Mul2JoinAggHorizontalRewriteAction", + "DecisionForestUDF2RelationRewriteAction", + "Mul2JoinAggRewriteAction", + "MultiLayerUDF2TorchNNRewriteAction", + "MLDecompositionPushdownRewriteAction", + "MatMulDense2SparseRewriteAction", + "TreeModelTeePruningRewriteAction", +] + +# Optimizers catalog. +# opt_setting: the value passed to SET enable_extended_optimizer. +# "0" means no extended pass fires (baseline). Each digit maps to the +# corresponding if-block in register_optimizers.cpp CactusDynamicOpt::Run. +OPTIMIZERS = [ + { + "id": "noopt", + "name": "No optimizer (baseline)", + "description": "DuckDB default plan for the SQL+ML query", + "plan_key": "baseline", + "opt_setting": "0", + }, + { + "id": "opt1", + "name": "Optimizer1: ML decomposition pushdown", + "description": "Push NN below the join (dense matmul)", + "plan_key": "optimizer1", + "opt_setting": "1", + }, + { + "id": "opt2", + "name": "Optimizer2: ML decomposition + sparse matmul", + "description": "Push NN below the join and switch to sparse matmul", + "plan_key": "optimizer2", + "opt_setting": "2", + }, +] + +# --- Optimizer definitions shown in the Optimizer Definition pane ---------- + +OPTIMIZER_DEFINITIONS = { + "noopt": """// Baseline: uses DuckDB's built-in optimizer only. +IF + [builtin] duckdb_default_optimizer_enabled +THEN + [action] (no extra SQL+ML rules) +""", + + "opt1": r"""IF + [metric] join_cardinality_ratio > 1e-5 +THEN + [action] MLDecompositionPushdownRewriteAction +""", + + "opt2": r"""IF + [metric] join_cardinality_ratio > 1e-5 +AND + [metric] feature_nonzero_ratio < 0.30 +THEN + [action] MLDecompositionPushdownRewriteAction + [action] MatMulDense2SparseRewriteAction +""", +} + + +ACTIVE_OPTIMIZER_IDS = {"noopt", "opt1", "opt2"} + +BENCHMARK_OPT_ORDER = ["noopt", "opt1", "opt2"] +BENCHMARK_LABELS = { + "noopt": "NoOptimizer", + "opt1": "Optimizer1", + "opt2": "Optimizer2", +} + +METRICS = { + "ranker_q1": { + "card_search_impressions": 500_000, + "card_listing_metadata": 200_000, + "join_cardinality_ratio": 0.0002, + "feature_nonzero_ratio": 0.002, + "feature_width": "150", + } +} + + +# Box-shaped DuckDB plans (fallback when live DuckDB execution is unavailable) +PLANS = { + "baseline": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ ml_argmax(cdb_softmax │ +│ (mat_add(mat_mul(mat_add │ +│ (mat_mul(list_concat │ +│ (user_profile_vec, │ +│ search_context_vec, │ +│ listing_feature_vec), ' │ +│ nn_params_150_512_2/nn_W1 │ +│ .csv', 'dense'), 'nn_b1 │ +│ .csv'), 'nn_W2.csv', │ +│ 'dense'), 'nn_b2.csv'))) │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ Table: ││ Table: │ +│ search_impressions ││ listing_metadata │ +│ ││ │ +│ Type: Sequential Scan ││ Type: Sequential Scan │ +└───────────────────────────┘└───────────────────────────┘""", + + "optimizer1": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ #5 │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ PROJECTION ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ ml_argmax(cdb_softmax ││ Table: │ +│ (mat_add(mat_mul(mat_add ││ listing_metadata │ +│ (mat_mul(list_concat ││ │ +│ ...), 'dense'), ...))) ││ Type: Sequential Scan │ +└─────────────┬─────────────┘└───────────────────────────┘ +┌─────────────┴─────────────┐ +│ SEQ_SCAN │ +│ ──────────────────── │ +│ Table: │ +│ search_impressions │ +│ │ +│ Type: Sequential Scan │ +└───────────────────────────┘""", + + "optimizer2": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ #5 │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ PROJECTION ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ ml_argmax(cdb_softmax ││ Table: │ +│ (mat_add(mat_mul(mat_add ││ listing_metadata │ +│ (mat_mul(list_concat ││ │ +│ ...), 'sparse'), ...))) ││ Type: Sequential Scan │ +└─────────────┬─────────────┘└───────────────────────────┘ +┌─────────────┴─────────────┐ +│ SEQ_SCAN │ +│ ──────────────────── │ +│ Table: │ +│ search_impressions │ +│ │ +│ Type: Sequential Scan │ +└───────────────────────────┘""", +} + +# Benchmark results (latency in ms) — hardcoded for the full suite display +BENCHMARK_RESULTS = { + "queries": ["Q1", "Q2", "Q3", "Q4", "Q5", "Q6", "Q7", "Q8", "Q9", "Q10"], + "query_types": ["ml", "sql", "ml", "ml", "sql", "ml", "ml", "sql", "ml", "sql"], + "optimizers": ["NoOptimizer", "Optimizer1", "Optimizer2"], + "latencies": [ + [85000.0, 12000.0, 60000.0, 15000.0, 20000.0, 18000.0, 40000.0, 22000.0, 30000.0, 5000.0], + [3674.0, 7000.0, 9000.0, 8000.0, 12000.0, 9000.0, 12000.0, 15000.0, 11000.0, 4500.0], + [1976.0, 6800.0, 6000.0, 7800.0, 11500.0, 8500.0, 8000.0, 14000.0, 9000.0, 4400.0], + ], +} + + +# --- DuckDB helpers --------------------------------------------------------- + + +def _make_conn(opt_setting: str): + """Open a DuckDB connection with the cactusdb extension loaded and the + synthetic query preamble executed. opt_setting is passed to + enable_extended_optimizer, controlling which optimizer pass fires.""" + conn = duckdb.connect(config={"allow_unsigned_extensions": True}) + conn.execute(f"LOAD '{EXTENSION_PATH}'") + for stmt in SQL_PREAMBLE: + conn.execute(stmt) + conn.execute(f"SET enable_extended_optimizer = '{opt_setting}'") + return conn + + +# --- Routes ----------------------------------------------------------------- + + +@app.route("/", methods=["GET"]) +def index(): + selected_query_id = request.args.get("query_id", QUERIES[0]["id"]) + selected_query = next(q for q in QUERIES if q["id"] == selected_query_id) + selected_metrics = METRICS.get(selected_query["id"], {}) + + optimizer_left_id = request.args.get("optimizer_left_id", "noopt") + optimizer_right_id = request.args.get("optimizer_right_id", "opt1") + + definition_opt_id = request.args.get("definition_opt_id", "opt1") + if definition_opt_id not in OPTIMIZER_DEFINITIONS: + definition_opt_id = "opt1" + rules_snippet = OPTIMIZER_DEFINITIONS.get(definition_opt_id, "") + + def get_opt(opt_id, default_id): + if opt_id not in ACTIVE_OPTIMIZER_IDS: + opt_id = default_id + opt = next((o for o in OPTIMIZERS if o["id"] == opt_id), None) + if opt is None: + opt = next(o for o in OPTIMIZERS if o["id"] == default_id) + return opt + + active_optimizers = [o for o in OPTIMIZERS if o["id"] in ACTIVE_OPTIMIZER_IDS] + selected_optimizer_left = get_opt(optimizer_left_id, "noopt") + selected_optimizer_right = get_opt(optimizer_right_id, "opt1") + + return render_template( + "index.html", + queries=QUERIES, + actions=ACTIONS, + optimizers=active_optimizers, + selected_query=selected_query, + metrics=selected_metrics, + selected_optimizer_left=selected_optimizer_left, + selected_optimizer_right=selected_optimizer_right, + rules_snippet=rules_snippet, + definition_optimizer_id=definition_opt_id, + ) + + +@app.route("/api/query_plan") +def api_query_plan(): + """Return the DuckDB physical plan for a query+optimizer pair. + Falls back to the hardcoded PLANS dict if live execution fails.""" + query_id = request.args.get("query_id", QUERIES[0]["id"]) + optimizer_id = request.args.get("optimizer_id", "noopt") + + opt = next((o for o in OPTIMIZERS if o["id"] == optimizer_id), None) + if opt is None: + return jsonify({"error": "Unknown optimizer"}), 400 + + if query_id == "ranker_q1" and SQL_SELECT: + try: + conn = _make_conn(opt["opt_setting"]) + rows = conn.execute(f"EXPLAIN {SQL_SELECT}").fetchall() + conn.close() + plan = next( + (v for k, v in rows if k == "physical_plan"), + "\n".join(v for _, v in rows), + ) + return jsonify({"plan": plan}) + except Exception as e: + app.logger.warning("Live plan failed, using fallback: %s", e) + + plan = PLANS.get(opt.get("plan_key", "baseline"), "No plan available.") + return jsonify({"plan": plan, "fallback": True}) + + +@app.route("/api/run_benchmark") +def api_run_benchmark(): + """Run the selected query under every active optimizer and return runtimes.""" + query_id = request.args.get("query_id", QUERIES[0]["id"]) + + if query_id != "ranker_q1" or not SQL_SELECT: + return jsonify({"error": "Live benchmarking is only supported for Q_Ranker"}), 400 + + results = [] + for opt in [o for o in OPTIMIZERS if o["id"] in ACTIVE_OPTIMIZER_IDS]: + try: + conn = _make_conn(opt["opt_setting"]) + t0 = time.perf_counter() + conn.execute(SQL_SELECT).fetchall() + runtime_ms = round((time.perf_counter() - t0) * 1000, 1) + conn.close() + results.append({ + "optimizer_id": opt["id"], + "name": opt["name"], + "runtime_ms": runtime_ms, + }) + except Exception as e: + app.logger.error("Benchmark failed for %s: %s", opt["id"], e) + results.append({ + "optimizer_id": opt["id"], + "name": opt["name"], + "runtime_ms": None, + "error": str(e), + }) + + return jsonify({"query_id": query_id, "results": results}) + + +@app.route("/add_optimizer", methods=["POST"]) +def add_optimizer(): + global ACTIVE_OPTIMIZER_IDS, OPTIMIZER_DEFINITIONS + + text = request.form.get("optimizer_definition", "").strip() + if not text: + flash("Optimizer definition is empty; nothing added.", "danger") + return redirect(url_for("index")) + + OPTIMIZER_DEFINITIONS["opt2"] = text + ACTIVE_OPTIMIZER_IDS.add("opt2") + + flash("Optimizer2 has been added and activated.", "success") + return redirect(url_for("index", definition_opt_id="opt2", + optimizer_left_id="noopt", optimizer_right_id="opt2")) + + +@app.route("/upload_optimizer", methods=["POST"]) +def upload_optimizer(): + file = request.files.get("optimizer_file") + if not file or file.filename == "": + flash("No optimizer file selected.", "danger") + return redirect(url_for("index")) + + filename = secure_filename(file.filename) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + flash(f"Optimizer '{filename}' uploaded successfully.", "success") + return redirect(url_for("index")) + + +@app.route("/upload_action", methods=["POST"]) +def upload_action(): + file = request.files.get("action_file") + if not file or file.filename == "": + flash("No action file selected.", "danger") + return redirect(url_for("index")) + + filename = secure_filename(file.filename) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + flash(f"Action definition '{filename}' uploaded successfully.", "success") + return redirect(url_for("index")) + + +@app.route("/api/benchmark_data") +def api_benchmark_data(): + queries = BENCHMARK_RESULTS["queries"] + query_types = BENCHMARK_RESULTS["query_types"] + base_latencies = BENCHMARK_RESULTS["latencies"] + + active_labels = [] + active_latencies = [] + + for row_idx, opt_id in enumerate(BENCHMARK_OPT_ORDER): + if opt_id in ACTIVE_OPTIMIZER_IDS: + active_labels.append(BENCHMARK_LABELS[opt_id]) + active_latencies.append(base_latencies[row_idx]) + + return jsonify({ + "queries": queries, + "query_types": query_types, + "optimizers": active_labels, + "latencies": active_latencies, + }) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/Frontend/archive/app.py b/Frontend/archive/app.py new file mode 100644 index 0000000..d81f5f8 --- /dev/null +++ b/Frontend/archive/app.py @@ -0,0 +1,454 @@ +from flask import Flask, render_template, request, redirect, url_for, flash, jsonify +from werkzeug.utils import secure_filename +import os + +app = Flask(__name__) +app.secret_key = "demo-secret-key" + +# Where uploaded files will go (for demo purposes) +UPLOAD_FOLDER = "uploads" +os.makedirs(UPLOAD_FOLDER, exist_ok=True) +app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER + +# --- Demo in-memory data ---------------------------------------------------- +# Single SQL+ML query: 150-dim NN over feature vectors with join + +QUERIES = [ + { + "id": "ranker_q1", + "name": "Q1: SQL+ML Ranking (NN over features)", + "sql": """SELECT + s.impression_id AS impression_id, + m.listing_row_id AS listing_row_id, + m.property_type AS property_type, + ml_argmax( + cdb_softmax( + -- layer 2: (h * W2) + b2 -> 1 x 2 logits + mat_add( + mat_mul( + -- layer 1: (x * W1) + b1 -> 1 x 16 hidden + mat_add( + mat_mul( + list_concat( + s.user_profile_vec, + s.search_context_vec, + s.listing_feature_vec + ), -- 150-dim feature vector + '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/queries/synthetic_query/nn_params_150_512_2/nn_W1.csv', + 'dense' + ), + '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/queries/synthetic_query/nn_params_150_512_2/nn_b1.csv' + ), + '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/queries/synthetic_query/nn_params_150_512_2/nn_W2.csv', + 'dense' + ), + '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/queries/synthetic_query/nn_params_150_512_2/nn_b2.csv' + ) + ) + ) +FROM search_impressions AS s +JOIN listing_metadata AS m + USING (listing_id) +ORDER BY s.impression_id;""" + } +] + +# All available actions +ACTIONS = [ + "TorchNN2TwoLayerUDFRewriteAction", + "Mul2JoinAggHorizontalRewriteAction", + "DecisionForestUDF2RelationRewriteAction", + "Mul2JoinAggRewriteAction", + "MultiLayerUDF2TorchNNRewriteAction", + "MLDecompositionPushdownRewriteAction", + "MatMulDense2SparseRewriteAction", + "TreeModelTeePruningRewriteAction", +] + +# Optimizers catalog (baseline + two configs) +OPTIMIZERS = [ + { + "id": "noopt", + "name": "No optimizer (baseline)", + "description": "DuckDB default plan for the SQL+ML query", + "plan_key": "baseline", + }, + { + "id": "opt1", + "name": "Optimizer1: ML decomposition pushdown", + "description": "Push NN below the join (dense matmul)", + "plan_key": "optimizer1", + }, + { + "id": "opt2", + "name": "Optimizer2: ML decomposition + sparse matmul", + "description": "Push NN below the join and switch to sparse matmul", + "plan_key": "optimizer2", + }, +] + +# Metrics for this query +METRICS = { + "ranker_q1": { + "card_search_impressions": 500_000, + "card_listing_metadata": 200_000, + "join_cardinality": 20_006_670, + "feature_nonzero_ratio": 0.002, + "feature_width": "150", + } +} + +# Box-shaped DuckDB plans +PLANS = { + "baseline": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ ml_argmax(cdb_softmax │ +│ (mat_add(mat_mul(mat_add │ +│ (mat_mul(list_concat │ +│ (user_profile_vec, │ +│ search_context_vec, │ +│ listing_feature_vec), ' │ +│ /home/local/ASUAD/jrtandel│ +│ /cactusdb-duckdb-extension│ +│ /queries/synthetic_query │ +│ /nn_params_150_512_2/nn_W1│ +│ .csv', 'dense'), '/home │ +│ /local/ASUAD/jrtandel │ +│ /cactusdb-duckdb-extension│ +│ /queries/synthetic_query │ +│ /nn_params_150_512_2/nn_b1│ +│ .csv'), '/home/local/ASUAD│ +│ /jrtandel/cactusdb-duckdb │ +│ -extension/queries │ +│ /synthetic_query │ +│ /nn_params_150_512_2/nn_W2│ +│ .csv', 'dense'), '/home │ +│ /local/ASUAD/jrtandel │ +│ /cactusdb-duckdb-extension│ +│ /queries/synthetic_query │ +│ /nn_params_150_512_2/nn_b2│ +│ .csv'))) │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ Table: ││ Table: │ +│ search_impressions ││ listing_metadata │ +│ ││ │ +│ Type: Sequential Scan ││ Type: Sequential Scan │ +└───────────────────────────┘└───────────────────────────┘""", + + "optimizer1": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ #5 │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ PROJECTION ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ Expressions: ││ Table: │ +│ #0 ││ listing_metadata │ +│ #1 ││ │ +│ #2 ││ Type: Sequential Scan │ +│ #3 ││ │ +│ #4 ││ │ +│ ml_argmax(cdb_softmax ││ │ +│ (mat_add(mat_mul(mat_add ││ │ +│ (mat_mul(list_concat ││ │ +│ (user_profile_vec, ││ │ +│ search_context_vec, ││ │ +│ listing_feature_vec), ' ││ │ +│ /home/local/ASUAD/jrtandel││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_W1││ │ +│ .csv', 'dense'), '/home ││ │ +│ /local/ASUAD/jrtandel ││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_b1││ │ +│ .csv'), '/home/local/ASUAD││ │ +│ /jrtandel/cactusdb-duckdb ││ │ +│ -extension/queries ││ │ +│ /synthetic_query ││ │ +│ /nn_params_150_512_2/nn_W2││ │ +│ .csv', 'dense'), '/home ││ │ +│ /local/ASUAD/jrtandel ││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_b2││ │ +│ .csv'))) ││ │ +└─────────────┬─────────────┘└───────────────────────────┘ +┌─────────────┴─────────────┐ +│ SEQ_SCAN │ +│ ──────────────────── │ +│ Table: │ +│ search_impressions │ +│ │ +│ Type: Sequential Scan │ +└───────────────────────────┘""", + + "optimizer2": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ #5 │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ PROJECTION ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ Expressions: ││ Table: │ +│ #0 ││ listing_metadata │ +│ #1 ││ │ +│ #2 ││ Type: Sequential Scan │ +│ #3 ││ │ +│ #4 ││ │ +│ ml_argmax(cdb_softmax ││ │ +│ (mat_add(mat_mul(mat_add ││ │ +│ (mat_mul(list_concat ││ │ +│ (user_profile_vec, ││ │ +│ search_context_vec, ││ │ +│ listing_feature_vec), ' ││ │ +│ /home/local/ASUAD/jrtandel││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_W1││ │ +│ .csv', 'sparse'), '/home ││ │ +│ /local/ASUAD/jrtandel ││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_b1││ │ +│ .csv'), '/home/local/ASUAD││ │ +│ /jrtandel/cactusdb-duckdb ││ │ +│ -extension/queries ││ │ +│ /synthetic_query ││ │ +│ /nn_params_150_512_2/nn_W2││ │ +│ .csv', 'sparse'), '/home ││ │ +│ /local/ASUAD/jrtandel ││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_b2││ │ +│ .csv'))) ││ │ +└─────────────┬─────────────┘└───────────────────────────┘ +┌─────────────┴─────────────┐ +│ SEQ_SCAN │ +│ ──────────────────── │ +│ Table: │ +│ search_impressions │ +│ │ +│ Type: Sequential Scan │ +└───────────────────────────┘""", +} + +# Rule snippet shown as if built via drag-and-drop +RULE_SNIPPET = r""" +IF + [metric] join_ratio > 1e-5 +AND + [metric] feature_nonzero_ratio < 0.30 +THEN + [action] MLDecompositionPushdownRewriteAction + [action] MatMulDense2SparseRewriteAction +""" + +# Benchmark results (latency in ms) for this query +BENCHMARK_RESULTS = { + "queries": [ + "Q1", + "Q2", + "Q3", + "Q4", + "Q5", + "Q6", + "Q7", + "Q8", + "Q9", + "Q10", + ], + # Crude labels for filtering in the UI + "query_types": [ + "ml", # Q1 + "sql", # Q2 + "ml", # Q3 + "ml", # Q4 + "sql", # Q5 + "ml", # Q6 + "ml", # Q7 + "sql", # Q8 + "ml", # Q9 + "sql", # Q10 + ], + "optimizers": ["NoOptimizer", "Optimizer1", "Optimizer2"], + # Latencies in milliseconds: [optimizer][query_idx] + "latencies": [ + # NoOptimizer (baseline / no extended optimizations) + [ + 85000.0, # Q1: SQL+ML Ranker (real: 1 min 25 s) + 12000.0, # Q2 + 60000.0, # Q3 + 15000.0, # Q4 + 20000.0, # Q5 + 18000.0, # Q6 + 40000.0, # Q7 + 22000.0, # Q8 + 30000.0, # Q9 + 5000.0, # Q10 + ], + # Optimizer1: ML decomposition pushdown (dense matmul) + [ + 3674.0, # Q1 (real: 3.674 s) + 7000.0, # Q2 + 9000.0, # Q3 + 8000.0, # Q4 + 12000.0, # Q5 + 9000.0, # Q6 + 12000.0, # Q7 + 15000.0, # Q8 + 11000.0, # Q9 + 4500.0, # Q10 + ], + # Optimizer2: ML decomposition + sparse matmul + [ + 1976.0, # Q1 (real: 1.976 s) + 6800.0, # Q2 + 6000.0, # Q3 + 7800.0, # Q4 + 11500.0, # Q5 + 8500.0, # Q6 + 8000.0, # Q7 + 14000.0, # Q8 + 9000.0, # Q9 + 4400.0, # Q10 + ], + ], +} + + + +# --- Routes ----------------------------------------------------------------- + + +@app.route("/", methods=["GET"]) +def index(): + # Single query for now, but keep structure + selected_query_id = request.args.get("query_id", QUERIES[0]["id"]) + selected_query = next(q for q in QUERIES if q["id"] == selected_query_id) + selected_metrics = METRICS.get(selected_query["id"], {}) + + # Which optimizers to compare? + optimizer_left_id = request.args.get("optimizer_left_id", "noopt") + optimizer_right_id = request.args.get("optimizer_right_id", "opt2") + + # Helper to look up optimizer by id with fallback + def get_opt(opt_id, default_id): + opt = next((o for o in OPTIMIZERS if o["id"] == opt_id), None) + if opt is None: + opt = next(o for o in OPTIMIZERS if o["id"] == default_id) + return opt + + selected_optimizer_left = get_opt(optimizer_left_id, "noopt") + selected_optimizer_right = get_opt(optimizer_right_id, "opt2") + + plan_left = PLANS.get(selected_optimizer_left["plan_key"], "No plan available.") + plan_right = PLANS.get(selected_optimizer_right["plan_key"], "No plan available.") + + return render_template( + "index.html", + queries=QUERIES, + actions=ACTIONS, + optimizers=OPTIMIZERS, + selected_query=selected_query, + metrics=selected_metrics, + selected_optimizer_left=selected_optimizer_left, + selected_optimizer_right=selected_optimizer_right, + query_plan_left=plan_left, + query_plan_right=plan_right, + rules_snippet=RULE_SNIPPET, + ) + + +@app.route("/upload_optimizer", methods=["POST"]) +def upload_optimizer(): + file = request.files.get("optimizer_file") + if not file or file.filename == "": + flash("No optimizer file selected.", "danger") + return redirect(url_for("index")) + + filename = secure_filename(file.filename) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + flash(f"Optimizer '{filename}' uploaded successfully.", "success") + return redirect(url_for("index")) + + +@app.route("/upload_action", methods=["POST"]) +def upload_action(): + file = request.files.get("action_file") + if not file or file.filename == "": + flash("No action file selected.", "danger") + return redirect(url_for("index")) + + filename = secure_filename(file.filename) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + flash(f"Action definition '{filename}' uploaded successfully.", "success") + return redirect(url_for("index")) + + +@app.route("/api/benchmark_data") +def api_benchmark_data(): + # Simple JSON endpoint used by Chart.js on the frontend + return jsonify(BENCHMARK_RESULTS) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/Frontend/archive/index.html b/Frontend/archive/index.html new file mode 100644 index 0000000..c052573 --- /dev/null +++ b/Frontend/archive/index.html @@ -0,0 +1,611 @@ + + + + + OPTBench + + + + + + + + + + + + + +
+ {% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} + {% for category, message in messages %} + + {% endfor %} + {% endif %} + {% endwith %} +
+ + + + +
+ +
+ +
+
+
+ Query Selector +
+
+
+ + +
+ +
+ SQL + ML + +
+ + 
{{ selected_query.sql }}
+
+
+
+ + +
+
+
+ Query Plan Visualization +
+
+

+ Compare the logical plans produced by any two optimizers for the same SQL+ML query. +

+ +
+ +
+
+ Left plan +
+ + + + + +
+
+ 
{{ query_plan_left }}
+
+ + +
+
+ Right plan +
+ + + + + +
+
+ 
{{ query_plan_right }}
+
+
+
+
+
+
+ + +
+ + +
+
+
+ Statistics Window +
+
+

+ Data / plan statistics for this SQL+ML query (used as input to optimizer decisions). +

+ {% if metrics %} +
+ + + {% for k, v in metrics.items() %} + + + + + {% endfor %} + +
{{ k }}{{ v }}
+
+ {% else %} + No statistics available for this query yet. + {% endif %} +
+
+
+ + +
+
+
+ Rewrite Actions +
+
+

+ Available rewrite actions that can be used by any optimizer. +

+
    + {% for action in actions %} +
  • + {{ action }} + rewrite +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+ Optimizers +
+
+

+ Existing optimizer profiles available in this demo. +

+
    + {% for opt in optimizers %} +
  • +
    + {{ opt.name }} + {% if opt.id == 'noopt' %} + baseline + {% else %} + optimizer + {% endif %} +
    + {% if opt.description %} + {{ opt.description }} + {% endif %} +
    • + {% endfor %} +
+
+
+
+ + + +
+
+
+ Optimizer Definition +
+
+

+ Definition authored using a drag-and-drop statistics & action builder. +

+ + 
{{ rules_snippet }}
+ +
+ +
+ +
+
+
+ + +
+ + +
+ +
+
+
+ Upload Optimizer / Action +
+
+

+ Upload optimizer configs or action definitions to extend the workbench during the demo. +

+ +
+ +
+ + +
+
+ +
+ +
+ + +
+
+ +

+ Uploaded files are stored under uploads/. +

+
+
+
+ + +
+
+
+ Benchmarking Results +
+
+ + +
+
+ + +
+ + +
+ + +
+
+
+
+
+ + + + + + + diff --git a/Frontend/templates/index.html b/Frontend/templates/index.html new file mode 100644 index 0000000..d41e2d5 --- /dev/null +++ b/Frontend/templates/index.html @@ -0,0 +1,661 @@ + + + + + OPTBench + + + + + + + + + + + + + +
+ {% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} + {% for category, message in messages %} + + {% endfor %} + {% endif %} + {% endwith %} +
+ + + + +
+ +
+ +
+
+
+ Query Selector +
+
+
+ + + +
+ +
+ SQL + ML +
+ + 
{{ selected_query.sql }}
+
+
+
+ + +
+
+
+ Query Plan Visualization +
+
+

+ Compare the logical plans produced by any two optimizers for the same SQL+ML query. +

+ +
+ +
+
+ Left plan +
+ + + + + + +
+
+ 
Loading plan…
+
+ + +
+
+ Right plan +
+ + + + + + +
+
+ 
Loading plan…
+
+
+
+
+
+
+ + +
+ + +
+
+
+ Statistics Window +
+
+

+ Data / plan statistics for this SQL+ML query (used as input to optimizer decisions). +

+ {% if metrics %} +
+ + + {% for k, v in metrics.items() %} + + + + + {% endfor %} + +
{{ k }}{{ v }}
+
+ {% else %} + No statistics available for this query yet. + {% endif %} +
+
+
+ + +
+
+
+ Rewrite Actions +
+
+

+ Available rewrite actions that can be used by any optimizer. +

+
    + {% for action in actions %} +
  • + {{ action }} + +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+ Optimizers +
+
+

+ Existing optimizer profiles available in this demo. +

+
    + {% for opt in optimizers %} +
  • +
    + + {{ opt.name }} + + {% if opt.id == 'noopt' %} + baseline + {% else %} + optimizer + {% endif %} +
    + {% if opt.description %} + {{ opt.description }} + {% endif %} +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+ Optimizer Definition +
+
+

+ Definition authored using a drag-and-drop statistics & action builder + (rendered here as pseudo-code). +

+ +
+
+ +
+
+ +
+
+ +
+
+
+ +
+ + +
+ +
+
+
+ Upload Optimizer / Action +
+
+

+ Upload optimizer configs or action definitions to extend the workbench during the demo. +

+ +
+ +
+ + +
+
+ +
+ +
+ + +
+
+ +

+ Uploaded files are stored under uploads/. +

+
+
+
+ + +
+
+
+ Benchmarking Results +
+
+ + +
+
+ + +
+ + + +
+ +
+
+ + +
+
+
+
+
+ + + + + + + diff --git a/queries/synthetic_query/synthetic_query.sql b/queries/synthetic_query/synthetic_query.sql index b9205d5..22255e9 100644 --- a/queries/synthetic_query/synthetic_query.sql +++ b/queries/synthetic_query/synthetic_query.sql @@ -5,7 +5,7 @@ SET schema 'synthetic_db.synthetic'; SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; -SET enable_extended_optimizer = '2'; +SET enable_extended_optimizer = '4'; SELECT s.impression_id AS impression_id, From 9fd33d691e9ae6f7b4dbbae2439846e381e8ba51 Mon Sep 17 00:00:00 2001 From: jrt1899 Date: Mon, 25 May 2026 23:50:28 -0700 Subject: [PATCH 4/6] Frontend duckdb version changed to the built one --- Frontend/__pycache__/app.cpython-312.pyc | Bin 31751 -> 32100 bytes Frontend/app.py | 53 ++++++++++++++--------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/Frontend/__pycache__/app.cpython-312.pyc b/Frontend/__pycache__/app.cpython-312.pyc index 1b2ecb8ea028e1ccb9c6bf705bece59d1208d113..fe14724268938d39b7010d0b6f200351768a77f9 100644 GIT binary patch delta 4616 zcmb7H4Nz3q6~1@h?y}1+Kg;s}`xZf2{KE*MA{qoV{=`2}aV4&>_W`>uyLjJQ5rlxo ziPG4BxlLjcotR`AHR`CLlQfOfnlv_P(@vLQMrI|G*vU+jOw%FJ&ZKtQ>A4Rc5c@ZM zGyC0h&pG$rbI(2J+`C7)~w#5y2nhiqRnn zOR?uMS3$@XNAyyhJAS-}5HYxHpb_oM(>PPDWFy84QRj8z9E>EJ6vr;(#Asq#20R#4 zz>4uKVWOQ4?bKk6^3Bz#sa8s1^*F63mCrE6)hORWjhbquRASZ)ai#xX8BlAxhND)~;9v}-zvDzyki%2P1OqTFov*5}lOGz17M#@hL4KrF0S^f%c$b*^6 zVWxbtqL4gDD!9-!6LztZRFW!EO;(ZBCk&UDM=6lXCZ@eD^f@0!{8{AdTN!#WPLofk`cLpO!RDYvH6; z(|uaa)3k~;t^Afri@ek1B~mQ$?SWvYcF=4E4LA#=#YazZ-DIV7cY_s@jOYF-SThMk*;G8t_cyr|N41pa8!+zX5m_ z5v-_e4lf|0J0&#ZZk;kYq6$5dj8M1GrDMIdNhtymIze;FtYfKq1hQ*2NlE&yi5aG( zYb zwkbTRBGTu=R}L}1=|#FtG834Fiugtf#x})(jo;oh}C$)0>a1Ke#?@FTT9< z^265>!nUd#`s$H!?m*%R;duHDeddTIu7642l7X$mmW;4I;|?r`=wbj5W)IdU;;0 zML8%!TsJ7*eUjMLptOK;KwX(ia?rm|n%8Nb8-f664oIZdsgD@EfwqtmiS;zJL8wda zU`XM8y#j#}5j^b#bfV=@BUt2tP>KjN6o^FkRBf$XSGCf;a?J*s0o!z9)kQMD0w)g9 z;s#Ckcy7@{z;EEE=LDh`-8CX>a{uza<)dc%z|OGAF%oMVNy!+Av5i_&?^=vj;}|j; zjI;x^@8S6oVRdz_6Gx(2C8Z@uWDgn)Zz&V7JZRmv2N(8%7X1N$Q=m^k6T!iV5hB^` zB{I)cQS4RYiayC++CC2-WZ~`kxSySBNMmtzOA_uMPM>D3x-2}d{1`h^$CZCqS7dyK zuXBP4I~L(b+0q@Wa0ffM<0trew#b!de1Qv}W4m36H7{}-;$^5V9z`NSfH8~oa}3$qJ*%~A2NwW=i7)bV{dP3{bA@le#*s_&~`Le2EOx zw3HP#>K1yjHh&FLcu7z{U#3P)wd|dyL?elV=|p~; z*`i2%I?_zWNXcVEdox*JSBeMT0vewrYE-vj%BU^_eIu zm_lOf(reRWv;&c-ZvCVBV~yi&J8nn&g&n9JJ5$(9_aS_QopT4B{~BGeF1-YBe{5Fl zo{3*%ExUQRJ-NHo5z)8#8v>n)>(x{SQj`@UQMy#jm%sb~CSY}q+c()MuQc?>9A4#c zhQsq5)Q~g-6^Vw@;d+L+3JJeLhE)dk0hw(W1FN-ca38U+^(kxLBu#W+V!(xR76woyq?;C*I)tJRQap<2g?8 zR|VX4-c~7MZIft|+Y<~06bSu8Tws7pIzZ@~oVLOpZsCg6lm;n%i&c12%lYaNK~|{K zJf0obfT7m&X?dbkis;*%uNutrMdntBY{2V?ejlhSJY8h=wzqj834r$d)^7%R9PqDtFi}pXivDKcmUHr;E>&y;OEqxh#Zd6b~ntgyT!t zO_#S1*~^D5%ftHRfBkkYieHPryPM8_?Q3w@_3w!KD!|z*Q>#*S z@1>fnOAPNhD$LbIhU-O!>XN~AdoE&pk^R+Q*`wZMlNXyaT6KIx-xQ>+(u=ILttj!gK%R>ox5d!p^}{RkRd%4QEFog_1OrXp=5so*5%~0DcD*gv z5CQ$q*`L~8gt?CG{RS70&&4O?7h9*j0`6vbLlddV8}O>+Ecy`_^Ofi@hY@x~&M|xf znb+A)5PeI07T&#>BkPQxET_z!ky-zj)Fi0vu3lr3(rj_(0(#H2{AZK_4@ zu*cfx;hXIB_W3;r`NSa({Pv~A9QJWgL-`D+UgMw^`bn?S+no0Y4re+1jKilKu5(aB zKE86|Zvxpg{;rz}c!+-f(4bclSVnn#ss}*1lPKRG<@=z$9QdP4eWa=Ht9zzptw9p< zOUvnBK){n&ego1O9^V*$ifV77+96c?2`apW%(sy37K*!tEVqr8qYph6>=j0AxxK>S z*w5nAdSgbArB`=^UE9ApF($5md*Al{`o8)TbHWA7hhtXsMh_Z0zQ7AiIP-Mn7=q@k ztTvbBJG?iRY{tfcnlS{;3D?c6#Y0((@9+-$uCrur5-zzTAYJyD4ydsrq%#~zJD3)> c1RTfmp$P302!w!r4|7Bq z+jVN(&;hbZAPt`2q#fI1(s72g$#f>oF-|8>IW zf}HG4Sy&-NM2{6T5}Na4oZJ7hl^7Fe1q$Wn53J#70sz%1#_4O{3CD2FcVYCs{$VG%B0qkX(&gNhFe|QTe2R z6lzreDzcia;bf!PliP^M+E=h=C2^4=vW^s!5>h%YlJ%sFl#{#225ogW*+?qLCUOs7 zHR?SQ*-Y*wTS(=&-j~;e$ktb}FCRQrf~Nwqjoe4>=Q2+ryhIh*PIeGCsU{DMi$UyJ z^$>ZGJVYKQJ3(PJ7}dOj0c$mUM8jPg?$&S*pq@NBE_l|E$K!xF8snAR$PJR-70UM?pQxn(Uv)DD_5n^7F8lYy8jMm_o-y{nO> zi!V&QFa986=hQoizi{XJ7i&dJF%eJ?zxj^;I*~96j zszI{?cnNCjOk4pY)fAJs(IGoReL67dcruOM|ESr7(O{Z@G#_dGhRBbMBe)K^jJ+T2 zZkNN7PjYwqTin|ve_J>#`_*7uSn3L@O_JIqOMahU?NmsEBzLQFSmClzn~%su@`VZ4 zNRZ%Bdcr`eLBG_|DbpUQoyxw}hL8+aa=*V*l}V}8)~3iJ1%gyoN~2C+DAd;F?Svp3 z!{8zoa7I|Ecj2f3`~-rHy@uM|Rc>hwO<~s`U7hU`qXq(RPa2}e#n+D-A!r^mGd-5u z@6yv<@S&pum(mg-6@Gm1&U6FkCwNR<=>Y4gFJq}FW+}DDOeggZ_lK>F))9@F? zZBYaN?m@B`^)W4KI>2f?rC4RJ?q7vZv(J2wr7kz6L#)J;gBKK!uwD;WobZ$xhWQ@P zvcGyNa0g2|@Bogm`U7v^=a~6mVgK`7$T$dmdt=p(`iFP#s&iTBi+pWW;$TM&CJCz39CJyMPojw49q)@IrP;}Q=T_^T{FuvF zFYOc5q__`TL{sx{#ziQaTKKG^f-vxR#2tyl;phIwl;KR2M^F~GxZ?g&>v_y0-;86_y z%qE~7{F)vD&_$Eln|hR>-xu<>hkW5(ccy$x6d$>xM(F_T)0^jO4|=ukq#2b~hFG}4 zx?!BxYZfSciEbT3f3a4MsKXs&c@rJych9cb@|01HT!SFzU)-W=Lvn8!vu%39G>IwJZtx-9pWvCb^%090a^7oL63}z{!p7DQ=S7f zpF;tMLJq3{qT-&~DyUf_J<+z>K(}gZfL2Ftg#0s++*8j?M2g; zBHm+dLz_=;9X)}kNomwsKJu8UWtw5W!lD(m}< zDnA_Ufm887nY2G8v%9@L^B$*OBX0udQweFH9?LX6fbJh{(! z)HsX>MWRtlYTU+t-BgpNoj)8cq7@uAa=6HngB976oaA#c_$7q!FMh$Hn*d<$TwsA< zdH*h#ptS*wOW)>lelc|Ew4{m}VPbcNRK=C{e-rpOT%UJPdV8SoOpXdL_~odjU8VsT zxt(EErdoE>-*Tx)fko(>2(g0Z^i6!js9sU2%e0WnzvmwLU|EvXI<;&eJf6~8p|c~+ zc?p+cb&hAXpPg-Pth9?M1Nu2#sySs@Zr`w_UA5)U*z#v>g#+7pF4vx}9bYkPDZFYa znX!~yv8-oXS|n2}$yubOKuQ63-xMK>eZq8U-;90hY~r?QVcWM~uS3?I_|`X>?41^0 zUW)LZB)9?BZFjgGy2}o8b-8#sr^;MiEPhZdR+mpzg9Q#k}jvQ@x#%HN3LHnvv zA<#x!=>*#y&P;!mzc!!DQN524xDN>`T##5KoF#q?+z;7!cnC+Pirc=!h7Y;!Iab_J z=J+F!7m^p;iGg59R={6-$CebTa)}Q104AMfZ+GO2SAq2@`&i9j-**gvvy=3zxU;9! zTPj&Rts)N^GnSe-Rt|iIU}9J-5h#2Xj!E-$Y1iMmRj-8 zLjT5jH#xk*;Vlj~IK0n6%RZi8ka7HiL3!JwJWDA5e3ZXB<*!0{$kA#&?F?&UiSod; zOV@X%%GNg08IrfruYkvEM(Kj;nbTv8Z=(9EsD1|3e};;#BlC5XcpW9NKlC(AZSDOh zPB3HZ*p7Jwe{64LV@W37IIl;#+&LXka}`J@o^l>{PA9IM#nM;09DE;_=FM?x?tb=W J str: + """Assemble a complete DuckDB session script: extension load, preamble, + optimizer setting, then the body (EXPLAIN or SELECT).""" + lines = [f"LOAD '{EXTENSION_PATH}';"] for stmt in SQL_PREAMBLE: - conn.execute(stmt) - conn.execute(f"SET enable_extended_optimizer = '{opt_setting}'") - return conn + lines.append(stmt + ";") + lines.append(f"SET enable_extended_optimizer = '{opt_setting}';") + lines.append(body + ";") + return "\n".join(lines) + + +def _run_duckdb(sql: str) -> subprocess.CompletedProcess: + """Run a SQL script through the bundled DuckDB CLI (project build). + The CLI is invoked from PROJECT_ROOT so relative paths in SQL resolve.""" + return subprocess.run( + [DUCKDB_BIN, "-unsigned"], + input=sql, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + ) # --- Routes ----------------------------------------------------------------- @@ -661,14 +674,11 @@ def api_query_plan(): if query_id == "ranker_q1" and SQL_SELECT: try: - conn = _make_conn(opt["opt_setting"]) - rows = conn.execute(f"EXPLAIN {SQL_SELECT}").fetchall() - conn.close() - plan = next( - (v for k, v in rows if k == "physical_plan"), - "\n".join(v for _, v in rows), - ) - return jsonify({"plan": plan}) + sql = _build_session_sql(opt["opt_setting"], f"EXPLAIN {SQL_SELECT}") + proc = _run_duckdb(sql) + if proc.returncode == 0 and proc.stdout.strip(): + return jsonify({"plan": proc.stdout.strip()}) + app.logger.warning("Live plan stderr: %s", proc.stderr[:200]) except Exception as e: app.logger.warning("Live plan failed, using fallback: %s", e) @@ -687,11 +697,12 @@ def api_run_benchmark(): results = [] for opt in [o for o in OPTIMIZERS if o["id"] in ACTIVE_OPTIMIZER_IDS]: try: - conn = _make_conn(opt["opt_setting"]) + sql = _build_session_sql(opt["opt_setting"], SQL_SELECT) t0 = time.perf_counter() - conn.execute(SQL_SELECT).fetchall() + proc = _run_duckdb(sql) runtime_ms = round((time.perf_counter() - t0) * 1000, 1) - conn.close() + if proc.returncode != 0: + raise RuntimeError(proc.stderr[:300]) results.append({ "optimizer_id": opt["id"], "name": opt["name"], From 443cf2d572a40f79df58680ca0796956090d6e6f Mon Sep 17 00:00:00 2001 From: jrt1899 Date: Tue, 26 May 2026 16:01:06 -0700 Subject: [PATCH 5/6] Optimizer Creation support added --- Frontend/__pycache__/app.cpython-311.pyc | Bin 0 -> 44091 bytes Frontend/__pycache__/app.cpython-312.pyc | Bin 32100 -> 39670 bytes Frontend/app.py | 191 ++++++++++++++++++-- Frontend/templates/index.html | 74 +++++--- src/include/optimization/dynamic_rule.hpp | 44 +++++ src/include/optimization/metric_catalog.hpp | 38 ++++ src/optimization/CMakeLists.txt | 1 + src/optimization/dynamic_rule.cpp | 166 +++++++++++++++++ src/optimization/register_optimizers.cpp | 16 +- 9 files changed, 483 insertions(+), 47 deletions(-) create mode 100644 Frontend/__pycache__/app.cpython-311.pyc create mode 100644 src/include/optimization/dynamic_rule.hpp create mode 100644 src/optimization/dynamic_rule.cpp diff --git a/Frontend/__pycache__/app.cpython-311.pyc b/Frontend/__pycache__/app.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e983343c2d759d09e0c97f0e743e6b438b8ea9c GIT binary patch literal 44091 zcmeIb3wT_|l^%Heg>G~=(0D&ckh~y30!@OA2VWuxK0rK3Pyk2*pd><)TTR>s2ogYp zw;O!40Sob`L?D+L!CHEWai}%yk!*OReCEu`pE({+hBLGFcr(eS8*{bnbb>_diFWPH zW*3U&U9FP+zWq<#N8j6MP^4^4Hu-#UQC+tlr(UN{ojP@@>bp5PP7cpE%StYO`|okw z-_VEpxE0Gs$Fex?4kvJeX^a~;4VuQygXVF|pk>@TXdSl=+Qzd6v&QX%b`ym$k2%J( z2eX;oGUgm~!p$<~8g#K=_n@2o<_zYr-`v4mGZ!=kJ>M~L+}rq!wC=<$=ZG*5;s9?5D!e(I$v++Wu zu$9@Wglb_Mvuzh@gj%wVWDiybE8Dpj+XuG>w=Py`$?GhFSJ+|9HQIl@usa3IV~4F9 z;nKt2>R`FqPPq|wm20^WaxO9K6U$Ij)j{6a>Imy43>18@?PR#P1-3e}-Kr7lQN!Ct zIHBR&rort(qtGPm7WN2xg?)Ez!hYc?;egOA9AsZD!Xe?Xa6~vtUn7>m8lhD{5H@Y^MH2`7c8S$H2>u3P94dWBQMY2nOW z+o)-l_P&FC{%DP!Njk)!Fe(OzB1!v{IOZP?iAmcq;#^4DFNQ-Cqr=y|rlcnv z9GVh?{^8NF;6z|N*c8IE%r^u1mq|h&?$L3JBo=1a}0ljfNxs(4}O~+0)&p zTHF1dr@Gra`l3bWr$)zwdNDW_41|OA!qm_u;e52&=o-8h2~LDZLlgBwfuYD$SU9g) zd~yuG79_e8r4UDh;rj6PiO7XuWOT?+F4xn)ec{VvLu%F0FUDEUg8z^9z`Vm{a}jMs zyJ|9V=QtDo zB9tTLxoA~wi#FA!|Ba~p0vrXiF@+iPv}w_<53Sk-i(s8L{{v32zL7?ukyqO*9O60D zu=-yi%UB2PX_R8d0t#lDvCUYgEf`1MoyijHGxq7Mb)Y%Yp}iQKaZEeb37u^WEjUvn z!eX{Qm#RIj6|$$Z#XZy6nW3kJbxu3Q4bx6T8jNRCVXME!coQZnFIv`gYk1l&xRjo2 zna-NA-I^2Js0o)EQ~zs(a)lF3x4t=TUDUO0D)e+(8<@CTKbW>o+lI~kToo75@io<) z=$mmxDpGMTdQ|&jo@&$o3OUnhJc)^;79BPZna@$ny1wE9U17ghO;3epn2R^J%@5dZn*7y6dw$u@Hy^ z&Vy$7KriZRo+y0Hr>2@AUFYVga}jg<(U)M{K~bl<7kA8`0(hBqvlIBo2 zX`Kv2E+nlNL!%Q(_vyYo0^;qVn7=UUByop zsth%k%%#w z;6;5nY(^!|a;w%Ywz8F+{Pzl~<2l=;oNcR|DSOLOexjuG&hYEQU%PPo!rbvy3*49o zGd`psACr-C%VhiHlTR`^|0;_sEWTZHr|$K-6?a)|L)V8EbD{HN4#u2)6`mXJ)W2Sz za34yzH_Y4Tdg<>&n}veFKoAF)-(dpwEFS-E&$oN-RmH3KN!9z{5YKCu^4jNYiH(~U zBJHSZqIy2V$XpX-@0g947}<1Ztk~p z<9vg}H^lgc`{H~TvM}GVYDXO8JFk4<#ImP4=BZAUZ<^1V&r0Oi-LwAC`G?LwaKG!0 z<+tEXRhE$2hW zhGoZw*v7*TOmW9C$#E>^IF|6_&+nPLHs?wGpc!-NlCE$Z09{Wrz?EpNMF z`Jwfo{f{lM{3Fvr`=7RNf$b+-oSmhXpLnY}3oJi%n_&NGfemIdTMSHG0{?fpF}m}N zzmI)1Q{UU!)6LT-(4MdD?d3yPf+9a041lf&!`{FGm?OD+I|*;YzR_`Df0(#4|EQp$ za%(?t@Tg;<$9+U_0NF2wuIhp#w9!KiJ}H6(h>`35$o0vf9=O3n2|eZyh$G{HYqcyb zo*zOt4Tpv!IwzhV4@CR{K~TdGOmZ5Z8q+&v47YAr@}T_tK_G?#5hEb-q`-H7!Ug! z_cr+VHa7X2>f!cmqi-mDr6vO)0UUX-CN0f1DL!d>=jHSaVAFs;DadD=G8Ca^P4*3` zJZT!|veYao$3J##F{qMEy6tTKI}Yss;gH+|M6fXK%5Ove&Yahw!f?Q zIA5!__o(1?p6bKQi9dEOz4IOPT9W2bA!(%vS<-@$KAC+PgYP8fbAjmAGyV?Q+RO{V zp;79wd_)We`O%5tAZCL@!GH;jkk+}C(EfhEl6W{Z%WCpb2Sg7h%y@LbFyXTta7q{k zV~BXff6!>_4-bXJU}GJh!L>um%s(fUuwc?)~^FG>iBVV`3rUY)nM>a zXjpavvxMv;^&m^_3fc_Skg7d>R#-Lb%P&_sQ3K|MY5GXyf2JyY3Kqb=|B<1QK=XPLJWn*4PnDHQV4O|=ck3<@|PpCoq8vPeSk>FUk2??L-{S16l5R*Tog-gX(u{I5UlV9$e!SLfGz>1Bb zPM1ImfKTytK6|30uOky)H}Z%1Mx@i)+n&k03EmC9v;b>+07C6ifEbYq9(xs!3{6aF zH|V{MXhGfwiD(RuQR9W9)o1*jvQtPWJrvBc1Xbxk|<0V;;hlglt5i)EHp8q*oBm{ka9*{&_fgxWQcx1cj#vi zh=kI=$QBvB5>)$ZcxW_2JxKEg(HRd$Llcyu5RRzf1Ua0bg%eWY1cdVvjUNi3(_RRO zL3xmPM5V|`s|Em4E48V13)Sb56%VXc!|5UTw3bOr1(>Z=u+lfw*qKsMgR;({2JF*; z@TEeerL)_wHsoWZqXspmqXXefg-|*oYgrV&CPF+5^*z?eYh0*zHu;`9m~>qBLy{?s zMxq`ViBlP=A2vHGv;zv=cVM$7g_8&j72Q*^Pgo)iZnXzYqWW z@&6S558%JCfeZ@SNI@GZY$FA3+^eF~GmR9fkzzGcv_^{8MDdzf3KXx2;x$pcCQ7A= zVl+{VCd81l#jvgo^4zeDadO%GZ2=Lh2&$k}skLYH#Oj$Qz6GSIX}?l|&+JBBHTZTP zOlDp7PYpFRPE}(@a>jqQt)a1*f4T=V-SEzx{7!y+bmHf;|24R9beRD9|485>WSGx( z4V>U1!T~ot0!lH-xFKSlHA=$sJzc%E9nTH)wYCk^p6h7s;~68>*4jT%3nnQ@{inTk zpu_8Bg&Li>5+bgE4W0l5P-lB;FHZ#~Ko73-J7lJT1*PeS%t>hB+D^50cl5V))MA8% z2pdu?$c^Qwb$o-@7+v;HgA(!&O@U7t2cI9l#QL`SsgA`Ve@G;uI}jHSa<6_O7p!QE zIzl85eh^OG9i0O_V-MwUa<;;V6^yWaV=x>W#hgk9_7L;+p%6OQ4B_RF7#<-qk5Cp; zK~s(Z%M#qmVsE7ku9FhQ7JR7^A=vT0Q)f>TV``|0I(%w*0h=wiIdM#(Ol_{!p;~FJ z&m6B+8bqsbzMk(r+uhyOS*tlwBgt*6G!7+zOtE^^l3D0Q!|<&lWVdF`UT?UhhT8*dTk9+I!q%1V?nV7G_|j<_1yDX z4(fP)=3kU+Rgs;jm8vF9b>#FqHFB4h?xk@eprBG1goYwYx{BjCn1(T=u7YCe42^-@ zFdUN!yXvXZ=iO7ZBFyOX?qlO$u-zz6n$!DIBA1nwEC zjK!6kFBM(>o|qb^Ne4}9WCs=&n|&Rhv{SMNZ_@s2*ZE2o83CVQzQG#Du- zFJXZ409?vL39GCkYuKO_DrK`>KJ z;h&0oPer{9@t)El9-}}Sj`6jMsLmUB3nE5_Z3tnkQr$>W#~avyfv%p8{(;t>)2s?2 z#6+@2QpPcAAwX+-1UOoyqG_vwthJFa14W1nXB8fR5Uh)hMo94H`H*fj#GvORh#{W> zKt}%bF}RM#Z_%Y-T z-)deU%J8L-Le1GlnwPR%DI5boNqIs%@E|EB0{Oy9l+J{22 z$=|6w$lqH2Sh4i{9cwMnWrL7{7)Ygv=+WrRY0Y_&L<%~#h2bO~NE^wCld=BTW~ z6uu zG-T{u4{E~`L7gDszXn;we(=_Ns1J@Zh64l25c+PAjkv68W$}HcvI<(eQIU~HZ8|^h zQ_=~YuVZ{+V{q32Y7FYyS^?QYgj~S20CbrpASwVQzUIqds6}D;MRVu2zEl1EatBth zMDVe~$ovJYGTIP6r6&BWCKxGZt3Yg0GcL!&=!!_P9dWeLi>g51<*+o%NE5X`!ZDNs z4)}~^hJ0o+<-~A_1xEP8N?>CFwFsJXDv|-O00x*vwX6&tPynH(khw!cdxo95TGAu+ zq+zj3fLKnPnqW#LozS%yx)2->h>e)lhzDULvo$rBHQGJs_S{FEFkXdzk6y8FjvIzj z-YX8hf?v8in~qLLf)QE8QK#6Wdf$M2GE6n}$uGEHFy)?uvAl!xmlrQD6}@@&UhQ&CbF8L$J|9Nv z&7&|PY1%_Y$%=HK@^a>EM_*S*zt<%8z&%XbQ!mNdD$Y~}#4~W7qaKi)94E4`2nhhojqNDnJx5n9?LU93oBblWLag9{>F zD2iYqIaLMxq*4dbZB!dPBn;<+7z0^ONI{FQWy|iQa4|kL=1n@a0yQR0jVy_#Xq}cs zQ-&mV(jW%yK?O#%X`Pg?-pGfqjz)$qfLI#i=#p(pnwp}w>g&NP$ko}*Lro*h%T-zv z2G=t>7TJZIYdPgZ6JytX&aO_7x96Gn=){Y>qJ^xqK}ydJJUmIb1t(EPiXrp2FY>h$ zAs+oo45&>cPK`mm#u*LBkf8^F748>#sy6=+Wb40GuB)E_{!4!?eKqL8rt8A%j-x5x*uyFV=C+9I!%9DyfmE^(djn}Y5qd? z-!-R3#Ko+&sY1sdj;Vc5GY&nx>2!F`OH3zT^sT4oKJ5@(sTA?$HhksizN|v-U8}Jb zu#3Wj8vGdbWmoEJ=+~;RJR^LuW!jusKR>M=)8eJoqlPv5`ZU&~F-ZX( zuxTzcJ}1ZqxKlZ5GyC8HQL*SqN(-&T9_Cv@lQ6CX#-@UYl|Z09)2I5LB!)s}DPngO(n%^e8KcUu@^${qy;&Mn(OP6=C00UwpMPhSsX_tC(Wn&l2%F*io}dT zi8@q5+51cCv`iiPm((!7r2g);CbMKh3f1Mwq>UApbdg6!hZsb}S8|0sIJNY`NxSSE z_BxVIrp-)ae^^#&cCx(EV0S^>PKnmQNLrxTA#S5jV#U1KNh{4l#VEOanF3G|vF#^^ z6reG{L#qh6NLsKcA?_uA^VFocAC@rfsbKXNl_)BEqi|vHcemW#5-a!Om5O#eENWOTYKRv#NkvU_9V?!K z`Mo!yx1tMYZoWMCaw2cT!@SDnyvoGps>Rm37ZMw`ESm1tCN^zdY`ptiV$+V*Y+F?U z^p&d$KIClqg&&iFPZ(~la{GMFCyAVb`3uWAn`1ef7x#R(`Pj1sh1B}f-CcYiyZ67 zyD`WAfP2FVLYEW$YCkrKfuo8B`GC3~g^-xIMIwC)Q>}2n5d4VfBT&|v2xI$&z@NWP zp%^V`gN{S{b4uQ6RCJ|ha}?gsfib_ zlgkw{u99&LM%2b&d}^xt)KtfME=WUm5KA;cu!2ExE5Z<6VLcMMpOjY8G@3LGg{d`C zeKr^R$8w4NthfFSeE$RW)^EXr-kQa^b7$LEpy`y-8`e3~ zTw}uJp1XW=@7&&mJ1Owj z67tKKKgJ6tUA-Y~#*lPs)<&JZX_Wt#of|!xW8p@2wN?DvpYPjoWaK}7@5AMu^DiAq zTA=X^KC+a!Y(){!uT#=Z7Bdf30T=X}Dd;9kGPCB%;*2bEGDmg3Ix0jiOkD?6{rT-% zKfgWun||k=KfnD|YIgZy%EEr=WT+2()eyf<(g3G}#HZ^b%c;pBZa;Q9?>u(M1(f0O zYwz91hHI=nWj|3D#O}(YEAwN+%+SQ?wa1jGm2DK^=Cx_TC54iZ24xJPqA01vs6ll3 zMI1;}*KDUqg&5a1Q0R6}AeI)QmUc?e<_Y@63|{?Ch2VM3k8(r%7@+B`hXH%-nynCU z)Wc9!5=7403=$LYwSN$yl*?zbGo&!tN&E5pDjBX%S^?e{oJp<_6RP?^7sMl?P-SM zM=y3Y(4*lS`^see{Px`^$%3I%w`sE%wVzRf##GcC=(b(3wRU;$5|%R8i}Hmud8WL5 zA++yrgQP;e^Nhd0tyQC_RF#GX$Q0l=H)1JCJqh~ErUgcOs0fm3ngNpYX&A*q7)r%w z2%wfqrFSV3A11(vuYpi0p2~i(gxBUGl^@xr2a)cV214$I{d5Qf^koBMd_U9^5Jm&> zhyW?sMS_0z36@f6IMkAmYE}zDTB}_F-e^N_9<*wMrubY7hOq+*hi5 zR_#sMRZ{c#-Ru8Xt@vABLc@~Z@)8+Hjyx8;tE2gk9vuvv9G`J|hJKssKL2^~5|5aG z8A`Q2FQLxJv{{%kVUvp+%S`CGyAjO1L!&Bq=hwBn^KYcUjC%l}hL+fv=*xBOnJsUFeH>8iGWLZIR2l1!?(N%cL0?LR5M2Q|UxX^}WUh5*CsOq$O$;=c+1 zyYasV|9kPj5C8k|{}lcYBrRte8z|`bn0Sbyep@rv{Ui@0_Zo$qC*y16;2eR;%|7_} z=>JLl?mLRxbAPOW zy!~g2<>;Rt6@L$@if@s@Y;VFQ=V1CLO1#^Dpuu|ozg67kf23F@f2dfl{UgP4<)@0} z8xM|(q*W$^d+x7}vSj41hCev^J!{gg9LAY+$;#^ytnY+%TDRe3HuSNv{THY7ihh7f zTF-}s>&Yy6@AMF@@~aOi_R;r8{~iWZ>dmwEi0X&m8M9#4YwklY#PWt^#6W4rbf8%;F1lu$x!%vU1UJTNM$&`k$j!NgW#RwUiD)#8VZp9WD zI!cufK>q}XHAAx`rJlkL-os8p(%x<+jbizrWNd5+u=AG z81!AJ0eGCXak{$|r;$N>T;B2I_3;Dn1+&afbLB_Fn%;|i(j|0ZSR+GM8p`jO9!91n zNzI$-q9d@ZO{4A0&;qLvy*TVOiw=b5Yog|wgJKz)BwE?uF~F~Tb|_ZjYQzl)2elVB zI}sE+o_nz4FPV$8LOOc;v0Tqi31ymZvVx3_;hFdjVzK=qP`YCZbhKw9OgP4fm#o}Y z;Af{#Lls0m)ez^XiGPS7zXvUyF!gaL%Q!ekUd&#)Qn&?2Rp!)weIk}qtGsUSY)2vw z2Rxc=mCS&SizDYX_f7Zwz(U*C2ICH1a_}+sCal@7c7L@ymfwEwc^L8=w{}R@j+nJW z@trSPuzYPp+*%=7D`M7)gv;~Vftv^B&&FLFCD+E7btBV_VH9f!`aIfWhmCceZdi<) zUM_0kB33OtRKG^dr0#_-m6dXR$MPLi(A!i=>;Y0*qyJUk)CBMmwXfhF&;p0Jj6mC- zkZB61ssl3=C91aWE;zmk@=W39RsW>=;)*9Kx9j zw0{Fz`smPA3d<741THjoCFqN0@0!Bt;3E@3VE`y$8=aV(iX?M~0+V!Pr5}e-;ahwe znNM1Vt_ld^3{Ra`HkU!+acUyzY(Lxfbo(*?v94aR3IVapg$O&_<)WxOx}Y^R;g|LN z#UH`*4+tF^VVUJ{E~>la*5Q@x!ujW8j><%Cexh_Ev=nhj>Z&Wto%JCH<6|;bxhz{2 zYp+jD^-0U|4mup&4nf?7kPsdMFztb=oZNc<3hBJ+dew~<)I2#Z)qkbVW$lQ+eiG*cfHS#wZ;wq;HHelB*dmqB|QqbH0FIy7WiWHdaCPP)#1c zpIJs5&LKzbyp4bPnK3_6d^cuLg-5oMx z6I`!wI5{16fkblE5^`L68_d|v+;kSyFVpM7!Q3rt@|Z2;JR0In_naGTpNqEFoesVx zU1x+gsVR@((c?*^NOkA9Y5O$Jk)L)X-l(h zNd`JbH0;;fTPHpZ2>Ok6B2mO-zB+=*o9EK^*onOe6fMKah{00b4__SnJh<%aH9L-#@@jMU2lvHqXT*M?*NSa6sQWRmq63sUYl(h?^q z!s8{DUyQ1w6HM`*G_T2>dh(El{rDhiPOb70acHk{ubS>~Hw<)xF!F}!CKsVvRUT~% zq;4!oZ*3vHr<%cv@dhX+;EWkK^9U$u^a3!@`j+4DR?0Qaoiv4#S#pgTc2UYL>`iN7 z+D(bT8Jav!O->YFZSuk+vBDz@l}l%qBJVtVKWll{;n=Rj^Q|yaFAKw3{EJ5FpCV^T zr@93#ZO7=7mi4dG%UaSIE9qP~Bcu2I=H-TtSVPDBWf-ZKg<)mINwh^cw5{uz4nNMR z@9OCq?CA4%wfBpsSSB^Me$1Wz5>Dd(K<*aQJ8PkrUQ$L~yV4rblFY62KTi2DgP znq9k_Dm4z3iGpmf^% z)(%+XC7lw?f=;QRbFK~B6Xu+2n^Eg;4akFXaX?}Eo?rc#a)tYT{a&Ivnv6_?V7_ZXgYFBe1>=I`PBhrAIfw)zzi#9%O2RSjA z;d;bZ$y!NAtfXV{%ze}S>L1%5H2%na-#veMq2%`U`>@d;yz#QiWBo7CJ`kHPQ)u|Co&CpZ;qIx?>sH5n3Yhe#6P33e@@05{#vJ7gm>4ST&bBabWZH-S{YbrMFrtWN^r2DHuF7Di&*_uns#Z*G+~w_<9a z*s^`$YOLnK{eAH*$D}PdEujJ%bqab+poMQW!-(g1OZnZiCs)k|EM}bp)6#;1BsHV1mBn_u3Rm3)4UbN$7HN> zZe`vY_28bF(Rx|3M@I!1X04jc;S=zeuiC%(IVO}k;vYX|wy z0{Pd?W7m-Xw0kd!n9mkQ2*H&{a0uC>CL^yaIO$%dh(`^Bc~P3^5w~=i=S?Dyy};-R zYahX7gov=tb;G)iVZ~o%bCLWMOb8}*d!B_{%pP@H=CDZ)d08hHX?KNUHiwpe8~^e% zW6{qF(*>Fk$67CHbt@=mp@^+;)#ihlZ@Z%T%-d=r*wbpL@BAl?pWZrX84@Te&qZyv zy{Kz1)fe}x6(BurW-Qo+Lj8VrBYv5GH_*;prDG-fhcH(spcm%80x2*p4OPBbw03^zqIF_(tF*@-lH+^(RmY$)XTy! z{-rl*3yNY$jOL%6xHLh6d9D1voW~F=b|Pc)8W*j;Q&p9#k?A)%ykwH7iLE$U{H zS6HzN7H1fXbUD{Ae@4-&Q(jczE6mkfp8d?N0r{1e(2enSN##B;8il!FGzxRUXcXqowf)M;6>V7v#|x{Z!YXVF_7vWF zd9iZYQx)@6Em_}jJ*+*jTzep1dr+!92q*Asi&<;jx3t7d4@sqmVtI#F+@4z}AG$X$ zyEn(S?7bIx_sb9VJZO8cN7{cP?(UM@T`~54vf`n|7i3*;8X|W*xU7#M$5Ud{4O^HU zoHW5ga{YX^*wTOJ+1H<47+l&AFWw^+?}_a@8FxJ`xt_*`X;;BQ`;rhV+>JNx+9SF4 z#H@S5TYv?BVLz7JY2|)WvJ2)7 zGerX>4Xhvi_b~rfSSg+Oe+_#AHb>mC?$D8TWD1VBS#OZB8CjEXq5fzW0T+d)56*&v zSSLE1N63cY+Kn1fo^;lX9mR8S#sX$XH*n)@0o@g$V10Dw60DocG-m4^cN&{x)w5F< zwR$tMDH!Ip%g@2^rPUDzQ#xWzc^(Jev@Hsp5OSiJ{Rz2gLo=>dy{Odxy?`grEtV{37*L)yhYA&~Jq8F&muJTD z6$g&Z*(=+0Y?sEs5^E2Jx;O)4R1!Tw3#YV3s!p?I9*D<^DX!TI;;y~x%55{LA)BYe z%;W)%2eqJ6Z`W`*x)J?K=7#sO)$o~Vj(O&?FNOQ&B(K?{HETyurL0vus?ce>@!B$Zl38g3S0y zrLMMNZy4?+wWcD8|A7*r3I`k0gCDwdG)&iLd5bf0F=r?|Q45)sA+-V=%!39^y0I~T zSiVRL3<@nUCv8|!pAf`5Ju_7@SOJ{N%4`8_|M_^H5`K0Dr@ly+4DqF)|>oJkp1F$+Tem zX7)F-7k9+Vc1dNsWVd@~e|YW>&)uJj@9lt?M|M;An&)Dr=i;70$uk&x!5{PZV-ElO zxp}k46ATfEE$WJUx+G86Dpzdtd`QNeWz}4i?E&wy*uS(%+P43G@%^bES0sG9zd!o> zqxYS0Uz_A>!#H2qVZxoLFmWj=%(=D?@{6VXt?~S7DZhH|*h=B%6a%wTzWFVy#5czI z-4eh1-uXEH)cyAOmZJ|USvIa8p5CRE`H+RE`Iqit!29sXW|?YR`tFYA9Sy z#j?91wyF7k+XL&5yL7zGIw@^kDX&;K^SdQ?OK>C+B?$vZ5>b*c=tk9y1`7hN2se-- zePWLaICtTqM}4H!DE_EG0NhT2G(q9~eS3?ICgsRO3GSF<9Ay-@|o!>6WlYUjlc`>xM1;++MI7*)-!C3bG9hD)>!d*rFn79mk8?wMIs^DS+(gd$X^}FZkey;zpdc8-xsA9>dfH}S5v^Dy zf8J1YE@KF&0lI~p%#tbt!%4q^d=`pd z;f!ge;}$!)v&~RDCF|k9be6HMSxe*4Gb79XYvi^ReT7f8TzXz_=1iOQ3^qFj)O^!D z(KVL?33nMJ+*{X=J%h7E(`vf>HL!P3TudIE##}*yyg?6rk@Nx9JZoc3nj zE{mx~NI7yh(pC;kaEXvqRCz!dpV5D3&dpjVk$oxzP?fEzq%+4CHjbF|t!9W_m5Obw zDVi1=T`)1^BE;Y-k5UVI3RlgPy0_hME7W+qv8^7}M{l@=nj7wSYQF=ld7EJlC1cD1 zq~YG^zrY=f;CD!^nr@Wjp~%h7EY?l&nBFTSqw0~Xv@9&^UXu-TH>NPF9pNXCR6HOb zFGxz)bVRp$NMYmxPTZm?<|9rfWYB_i{wVSl<^vXd zTqb!?mCuF^wY_Nz=GqT}7U(R;P2VndYi(;4;HBra~L1vQ2)M&b(E%82Hxq zH@9m}@)VT_3_I>Iny1R?y+w(}s~xkmXHrmUY) zsMeQ!nr3a-+stP!50SlJTc=6o zJNthSitX&cJJ&iV+(LQDj+d1)bCD_l=zmUNM?nRcym&WH!WG;=%jdP@RY>eN#mc}uyAJ(U#!4Ewk zAAcIxqT@jQAW^drWM7Q5_?RAWuQ-L=O(9+G=d<5o>qi6LtdtIv_#%AdwI!1DB(r3a zYN#=ayc_?Y`&N$3%X&#Wl>`fk#@b~1!jHlR^-c~F%^DW>idthutqYB)P3t=&_e+<3 zM`OOD^JieBUKU0s8*fv|mtcJAO3~1osH=t#2d+R?g~~)0r-tbA9HvtY4XnSQIDbaQ zpTqEOQWS_F0vwkY*+>K{VyexFS!9qR2RqPN{2Q{lAo)q!a61Jqh{$5cR)7`YVbkk?}iZe1(itFp^nQlXMlCEWr9VTW%Ti;TRvni!81s_5LvWR^VMh&Ks* z4zi6~CsnPczt569AHb2hxV;nCW>P)~r?7N75fQH^9l>i5Kw?}Ki%4F`dQUF=by`fk zs>YO;P6I?Q;_Rk&Ua-m3vabNbZuP5rk#qqrSoxDqvUFQ=-R*R+8=ivkX#3_ znv`F5yW(Mf<#K*yZ0l3^A&5F8K@tmimoTp|x-*-WRW47&>kFuW?M@}raB9dn#s@sz${ zU$n=|YNfK;xW_AbyfKIO100CGBc8KM%GovBiM6jT(`i$zXmI}NueW`z<95gGr{&@6 z2Q@$R{h=@3bWCbG2A6ncn}m_Bs2%TnXP=9o9hA-v&f?VV!uf6~2V0;VHXk$QObHiO z=FHi?g)BZMdoIY}8T7iShti#2aLV+CjM-fO(~+`9*5JDL5@$K5YT?iXV27ao#Xne^9_Lmc$EcrGnbor{Q|X`?`0bbMb&wv1{r4(&eS|Qbl9Dph+rd`jha3 zeLwR`Z3FQmXQd-&W6zyiK5{O0onefs^};zV9iqI7GbXfv4wC9661 z9B8V+nC*aUGOq|!ReRsUDb=m;tASaeO~5DekN@|Fo_(`*^eM zFAtbtLP9$=NjpeW651hz)E*ikK-;*F9)QX{sa)1LKbpY|O>|9j7YQDFG_5eB>=JjC zJod~iZHd{KVw&tym$ES&_$qFZ2E;W_L|d-J9yKh;%1>r_Wu#2?M_5fqP=EaWbecN3 zF&Am_I6FJ~-th+Hn7^so@H%_-bGTvX@vu2ubfe+MDRS2qEq!7(jfuZN*+eRoSd7n4 z%W8NTZNP&l?8orb7;Z}vOW^&zT$$&v5u`9(?25a5 zlFJ7wsE(U^ZXa4awp_S9R=AxFo}W#!Kk#f`h}@ltdv-~lU7##h$E#gm?fU!YZ;iY* zc602F-FKT8L(;~5_Z{)vRw=hNZapSh>GHD`>eud5^7^MsZe%}0b`pO@%g{MxN1hzl?U~WSR(3Ou zUaC?fvK4UZl;SQjn$o*Zj?#VP6vX}Tze#=P$4J`HcNSiZyLL*hosZ}`wM(^e$8O27 zJLcG(nIrkNy3Vmhbe&`J`v6_1SF-lTti6xwI)zN{O4E(-V=@vj^dC)&y#)m~j&J(y zIY!-c{T^%MK4!hb%#0a|OcwlG^?N7vdnS$aA8S-y?1~i|tv~4Z*g~d%5C>0Sda?G= z>#tAiu39m|hIG`LIsUYHTBjtH9_!Xg{YoL*-D{!aG}0AjAp5lS%A04thdhdG-uhkG zympZ%yt_fJTgXNeKzTk`NuMO3MW7{erKri=^K{d#c4|m6hcFXcPoP8^91AD&v`^(& z5$IvCOAxNH-GyOlP-Whxobf=)4Mq$Q4^d+imVY0if#tbd7PFpen~^Fa8Lh_cC>eq_~@ z)I6UJzEtPQ+^a$HQZzU<;)6EzSXiWSn2o?mvltYANI@L919_4WqW_nD`kC0*@0Ukr zXE=ztaD?uUWSl|LF&@CV4x?gNY^0X0q&Rkb)2NSYt=Un}0(#CXRT8 zY)mG>(2gM>L%Y8rk6)3&(9=X2W-2;)ZOXwb!fE+V;*^38{<4w zv*gD(3XjP@$S`czWj}sm&A9je*fj)fBF8kc>M{f z{zQCBm$U`iIwvL1$yqlp=DPLlYcJerCVq-Y&gU@p?tVninhR z{m^1AFn`R!n9ar1wWRF!vv*#2{RQTE1fC`2Sz?ZH1*^_V6XXk;al*t^G{iR6$2T_O za;!L4HtU|dxSGp(HYD=T~O?l@!cAiq(^h889!prWUhVm6&-{*sVBA_QS*v5#YSpcFp*?P`l&qC8Yo+S8W8u8! z2Iu#k`A|eMZD$6c*|X=WU%57aaM`{&X5XAB-?UJ)a50W6(#pMa=DFq@xgc1uJaxmJ zDBmW#KoDXwwb8sYWjeE6=2cELkngRWrvysoiPBa!r{93XQ>biY+T*LZCy$u-Y9SNGX+t*g1#9P6sv4r5bEInsp@_I%O2dfK$D@Ojf}uly5$KtCIB$egvUf>+4H zo?hr*$Tnro_am|?Yhh5@gvx*|-PjCSzxFC>q1YFFc1jW@AqgQNv>+A!O|NZaXci(->UEY1%{}TPfU&@kxX)tIhc-~A;9enGXb4ls4yNbEe&aYt# z%ZHMBl4KN%SmV2h&`?y53Dc~}VUB%K6jtT1ER`hQtOYCIQ@tyTl^}|>uqNJw ztaE>>LF@UKjwbW1$ST(kK8J7JlwcxZY`P@Ev+te^=w9SeR3T9(-fRZeNFwxP{a+}x zams9ovgWIK>f)3+SpPEBlM9oO`|fJDCt%1u>v|Y$z)38_H^>~uJT|Y6I=KVN^V$5t z13Ypjn{9Z##04ftD&2tP&zA-^_EU+{OI0)_@?&EBa9IZQj16Z>Dawm3=|Pr4mXYR( z&BwNCQf~)r1HssJfNdn$rF@#gn?U}eehP1X3-uJ^a=Zm^#oKTN-hNeqEAbAz6ITWB zF7dM(@5X!ZUc8Td^~-xoa1Gv%YjNFGc~2>@s0S7{+yD&M;{yOU3{d#s+o%V`$8}Iw zh8yuAd|0Hn!}K)aX54~XaT`8zRpCZGn{Yesz@7Lg@Y@V+zV;RZc@!avjeB3Fdn#Dd;4oQ_ zJQv3O=si}NXWaKPC9P;rr9_q0FU>0+SW~|vUhj_A53Z@-Db?ql!9!2=GQ5VWDo*8j zhH7LD)vh?z=rdGj*HBeUR7G4|BJY~|-K-J&$d2KXfpU@$d##rD#w1ciU65ap+s1FE zDi)L|`{swbOH4peb7LNlvv-JX5R_im06XmDHo&ZM+W`m~?*)%PR)*&-oZC1jg~R6I4KL}+nF`(WNKQ~mHWDVv0uBz9*~K1utjLBu<~$$UVmN~)t~ z(Mc0jDEx{+`D(FRrSvN~vtP+yJC$OVNvqDGOQnA0ph}|Jt@5j|T-+pbOi#)$?SpT{ zIf|3}Rg?6k1MsC>tyMv7s~@Pu?;+r#kwoD6qbdAvRGM6_*{}5JKzi}PYN=bS$)pAi zQm2eMLE}%*XwRGwt=pgb`OlRpd;DswBUTUI`qj5VeA77TDlKtF6gGmOw?Qx!5e%&3 zoo7YcYm%s;_V4_%LF1}`YhF*u9j6!NmZnq{kk}xY@k_}xv0}R58cxcKgYVOo`L=RLP`W zt7YIh5Nlc`elWl>u!=Fv@nP1-xm_ojYL9c6-NRImi~>0492#TytP<6<)cIT6|9@H# zc@y+8p`eEvQMwlBCm`_#0luO-Rtrj|@cNvbPmqHM6S7U41X*K)pox#%CgX-+;ssf2 zyFimZ1!aucgjyO_%OA<^=ZMkxBhk%263j*i1Z8a8f^1||P>9176hkBZZkNas7_5o~ zLTKY1dIeRi*lSY@TJPwP+c)I)uwE_~xIzID);sR@4RGtAfMZChe2DdMIpi}BfK4ON zKK6XMkGnvMCI|%?q>mL0qb#hbr{5upA;>xBI7fmTK{htZmGi@cX#wjf*%gkFQJ;I* zJ;8DgZfuCOjy%CLp3>zA#Y!`96a>z3*Idq&G_#j6v+Bd-Rcrp#qiUf7fu%`KR(xiS#V zTsM!dlty#%=gY5jMRV3a&?pKlj{}s#oV=o^@`~pSq|1SbA$NY;_p7c|yfZ_K|@rZ{94OQOK{se^7%t=8$-7oysa&ObhFF zhUlGK8_eNX8_xHk<}!uRB=QBw4iXyau_DQ-obKi=Md}-#>t4A1v6xiFkmu>EvXxGovGW;cv+thb{gXa)rS}C zZ|~SJi<#aFgi6%h+6^0+x(i?v+^%-Q0IUwnjdCo&DyIEtpsBu~I-wVP3aVEXV~?F9 z0e@_J6J`4nG2D4lGDg5T0>%MMDB6z#k1rAYZWQ$ZG)=>LKn@PHfy)FM8!aefgAi!a zmw?;?a(PLV5FlnP=4X?<(wJlF@GYqRN1_g|0Rrlvr1VKsb@xme!Q%PsaHc(C+Vqf; zD|XD#(`dRps?$%Oy;u>|n`WIs6fC)-o%PR`hpa_mebIDXRIQ&apY594HCq-uJ8z!f zb}emQvtV8*Z(cZdy=q}&$W$3tSI*EYDqy&xrjn9px6KvwnZ{4 zLK)jby2=n;DQ>hqqD_kXcdq;9w>M|z>Uuz4?lb`|{>Lp<71szV0yqn-V$Xb%3Y(n! zGg426%gyq|t=}}gMbLK$5V5!UpR~3ZzX#v_@JCTyzud>Y!)LTrpFRa&0^K`;F9=Fl z+cD1LBC<)30ub{3=q`ZElwUTb^htFPM_`nQ#$b2L-jMervim_BAk2Z-Le8H(osvdl zIk9unW<}JcbZ~fb2q#kczi&%MgesNKI8tJiOEj>Vl@q_?TaUcxU^-#n!p3tBLoCU- zz40CDWrtXo4|c5>)|Ggfk#P^%*<+(jDQ5eeV?#b5j_vz+xEUvwIP$+dvgeS#<4BXE zt-Zdcd4E&AkW}B*S>Mvp*xKS~tLZ!_+U?kWRB&X%I1=fL!7R88(fpS|ftO4c!#B6* zmfryEZP@!>vgKh1swuT8q)NM&ylz?@HKx4o38fUxw}%X+A-!#?K59zm=h~0RxVD@3 z+Zm+bUW8V>wzE<30U$T{uFfr}jeoVXzzn7%wj719D6i;WN#bvC+xbVG>kLUg4b+Lx zbkfarM?XY?B&nc|dQIh$4NxxG$z)g+F-#*C#%GZz&XHKhNWwQ#{%zT%6q=gUO{x+U zz(fCv-X-fNT=FRDqJ9Q`{JL}0nvV4lq3a<+7kds6=4(Rq_;DEYfr0Mz9m!5abaSB`-dISklDy4!=<^Ivf|_!;aF z`-m^=$w4u~2tUz72>tu%E(_CxtN0w&uDlS%+BPtDawq}T@F znGrzA|9;~BmS_NpgH}&fsrB}=%l4vqM{Ha$o5P@tV$zgLjM zsgG+Rw9UNcL}B3H5cC(M-dKCIqqDWy(b3e}>1eF$;D}QZR5i7op!OX$Blm0aZIWz^ zBNi=sx!X|8{WSqUB_Kk;za@Z7kzgiH@Z6dJ&C7*JB{83ZcI*HQDJG^kncOF&?n3}x zBLHw{@#i$NFXr47;@G+0LH%tqpFu=XPxxP)c#Iz_sY3gw=BA)?(Y(0rM%qtW@0K5) z&6};`OG4(RsY3uBt0_y~pH#F}2~N|K9qg)cV_8#C+(H z9BG=+iX2F%npP-yJ=RnDEdKV%+}dbL-s3dAUb#Z)70M?MGpLLX^yHzIvNXd>2BcmR z{r=2mYLL@EtD*t^Jk!*mrhmRcMd0q@1_S?3C)XKOKqWep^Tc|oMBC*$r)3aePlYs#~=3^lQO@I`!$b_r0TIwaueKNkper`#|wU_p?oC%h_yyoJm=vgh{(i% z2?8Y=;p4 zpq?T1aurVIkDsooPr<42#88@qO)7oFev6M^iPLe$)%aPugDRkW`BH5m#jT%I@SmUF zYP0$kQhP*oQ&=^rcu{ewH})P!dgUhhHWNkZ7euXknb z{X0Mfb)R!+sMqN_V@np5ko%MnEE5pZL4t`YNSu|J65)ueNUFi=K1d#r?2b<_u?Md$U-j7}sUwGThUnpA9-2_CdVb=NEi`eaa){j^ zyqToe>JjX4tRWr)jx*v>CMH&jR_gpcU0jD0-|jC?hrOOA2?0A8Y!-PYRs28nm1ig? z;0^)nC%{f)gS5LR)qWMfufIgD8q6iS6iC9x%((J@9kL^Xc`+_)A;={T2;L@9J~iY} z;+hM-?T0j#m@tSWWZ1e^>sBKO`S`Q?*D0IS@c*-aYk&+r)`dnIA_6x5FTy}Tvq2Kp zuaPDJC$G&ofl4P$YF^Yp5-1~vtx@WfiG%#&Kx2R;7R3xk%snUc^FKhvL`4&L9Hy7` zxCS6QbcUSgAV~I%_!#fl=;#PXBILdi4#H+A7S)Qm0ABGvVJb$mL`d#G0&xEU3?b(Q zti`<%)=c!iHo1)ZobPjQOZ%MAk=THHKmh3+5-^AO8*b}~XEeQ)5Qs`1_qd{_L+-;p z%nJHZmg{r4M#jjQlKTLN1qBCRnENBa_3cmrboCrJ!g2qZ&l^nNPkIyNa9)K7K%9^N zO1gMR0P(z{)rgr3nQt@q-$;=tB{3Ub*l!S-j}7K)p8#frQF`CxuMHMgtuYEZQ&91y z=89&1{Ze|_;?W;>-{`(Iwp0OGiqxPgl-CtX>y8+ELMI#{qa&nt@P8O&Hl-+x)AALW zRbvFBG2gL}6E3Q}m2zwBgY2k%>yO;m-M6$6d+lR6O0GxKwfFhLGcPd7(CD$9(kIWW z7mtOLcSm%4LiC;|4>wV!L+Fdg+5CrRPT9Ab>Q(elX_`Re{?dBA>?6IV!KU~qrAE_G zsJK(8fR73DCKnuwd@O%(D95OSk-@;IM=yh)eZ?$#*M*@{{u`&Z!U1bxsD$?&-G*p_ zrOEuTpZo|;M{!Gp6Y^3*df@RlS;pWI}GkJ zW_}ds5LdTGXsuKOr%#0r>jyLVm%PaZS*zTrgzb^k4+sr!td8}FN-CA!#2$1|VERbD zPJEz)Bw|@43a}}s$>9}SNa<5Em>iqIz;j$X;`Fhjn0SaHQv^pbA0u`H;2a{?aeh^7 ziYC=itLC<@8LX9zPohq|6KQX8Y;xI}g-i~_g9_+cv*r}2@oR7j;pQupSgrbk)$-L- zpmtK@m#w9NbiVAO*|RrM2Ap-P>kiv#Qs>wCwc=pZeqA<cG zGx&rgoaHy*Y@BmdIcZFo>}fL`XLI2=yJiZdZHm*0-x80wlP153v-nMeWm3n9w?7_a zBFdAJU^KWXK_g*nY5mBzSt_3~R|D!Jc1(N*DUM2Q9|`)z2d^XZKe&l_VrP0kO3dmS8ivqF{St33;ou-CA31l5&*}gYmL|_nQla@X z=2cxRAa=*0nlalSkd9h4MFKbKR`#Ip;!amrRa@ta!F<}%<>S-5{;oJ{Qa$;2}POdmHg%s4&q ziyM-FwL=;!z{Ya1Zm*VvuJN=bCRr*cBvm0F`fO^}H{kS%ncM$EK33$$k~<7=242S5 z?{s^-u~t%QOgz2@Tu|x3jB^w&E(oEQ;n;D`?So`-?B2*OB*ll}AooFHE?2N73L&xP z_K4R`f(~vw$gL_lcTZHbN_P?rX(#q0CY}=wn1GXmKxM=?0NtJ)b8{@V3rg(t^g|Fp z4!E(L@fcterlX!S9+J@mNJ7GeH^KTKw;wt%0vR&j~6b!1I|UXgpC63l0*OMPHHIIs>Atz4*4) z1j`qki}dyWTWJydzSob0tu;@?)K@$A10owqzI>@mtaZtRu7vSA&q0g{dl90ul#5W0aAIGw>$eS#;pkO_487QdP^KLI( z$^`m3If7NOK;7yAH}P6M>YAQiS}qHkY$c2N9ROcUQE)JYOR(yh>SbM4SeF&b-nh^k z(UmXjD#N$QCS<->D_-!J*G{f0fVW&fvnwF^hyDVTSDulR0W zZP;43bhIlFIodM?my5}>O<_acLrSf%PovLt>A@qx^H+{98p1id@9E7UYvKHn`SaI~ z&o?h--m3es@q@;n_kDaa(r`RdeIm5$Wa#9nki`*lx|j5W_mZ=&9+*E6$=(=AF1xEQ zTanQkJMe&9*@3zBpXhT|bkNC)h0P?~67q(HX&#Fr9yoPG6?8!*q7=*b-e7RV6K@+QKT^J$2TyIzOz=pRZg}Z@35T zHiXrI;$?MNSY5W@Tj+b=yR@t6aTW0A_Pu)S+Z-5q5LmuTzku`r!;pVmwt4$`yd zg2xxq7Bq|Ikap{0_Y%GLo-XnZ&vFpaJ#j?=ewvgpWsN?vO{=`#Lz69Gz0S*7h zLp`P2{RE)?05Dclrrll=xBYB`rFpaZqr9Z%3fY|;L(5kB&ITIb-`GtpJLtdJs{;6O zmZoJl{c#Zu@ZVKzZ`rK))g~l|*Ue2Me?rOs1Z-mN5snNLM}wyGHD|M|3P7$W#Vo3# z&kd#tOzP;_CRrZh|LSa>vKZ2lq6_#7z-%hcMA}S~QX|2LWsMnHGx^lSh+<87 z!}uO_hi@9UDK|moWjX)C_|$jm6+g^V2`MMNA#a`%b@f;b7B_Ipe|6y{6j5<;~|_E5|3d7;m`jY>X*ri>4j zMX2gw8&abk4^#Q?y)c>mAXkU7=WACecrAFv*KOA$^0ku9@9~!f%8>GroRa0O$N*b8 zjVPJom5i4%Lb~h-%6TBmL=~hucZFapRq@(&5yU)@F=+2SfuMUl{p`Obl2!81o8wn1{h47>uzk6Nmx`;k~jY%aU1L19oHt zBomU>4RMyF5DamW$&++S>S>zSG*6mN(oRfFhvbekNi&&g(?1n~rb(vL^z4^SgFsXdpVaFpEsQmk~0{|xw0)dmz^~iY_EekP^wA`>r}8> zr_%k8lN?UW!!ub37sc(@P+XE;TBl??vFIoD6#po_jFeu<5A`ii@5q;NnA+qub?5l*8R=`Sg$SSg$)N`I^F6^|1 ztR)S^Mb?p9M+^b%nMc->MzVoy1b$T>gp$kTL|Ttmm7 z(n;=5I;SZuy@pa(>RSTSq?gIrB-jlxNAp3iXZq2SG=yunu8C2<*e^OmUvieyq#D+Z znT@B7xMb|<%pu2^T2m#>Y6KSD%%KfnrTBv(Z5(^=mf~_Nj{@4l!Oi}D`_0R`IH|#n zDe9^rXE;T=h4XLYu$Arf+>;aFWH$$m3bHGnW?Klxo$y0w+xx3&H*3A4uAmu4sC4drTza z00^foGh~;L#w=tDx94L`jBVZSvw2iVc=2UvJVT|C>ht=-k^Zrt4LZK-cx zuV%Vio4hS;uKG=Dnq0I37N^AkPKolPQwpFOLV>Uxr3-+LN&pUMlqV83&%3)X5F*~F z9E}DdVQ+L#i0%i*ZSV!rAP@&p!iaJTkA=r)%o%MP&$3jFXF3nMCUZ*J+MTV!Wi30CCjQR@iD>Tp%r4Mk~VGa*+c!I+z z4i5pSxqSb z+%%R7P>61k4L?x8zCUUbR{?#cXN8yz{nBux#1<2S4t@P}PfR3-{We_h$|L#u>ZCxY z1`SG4woN+6{cbaC^a*6b=>hAx$`>z+j+UpF9EK0`&WDC;z!x%z1;21dn)sn zaY3Gb2F$K)Elu@}%?{SGXT5O~pWnbv?qqV3_vFbE25Mu!AS)Q2uf*W3+L9Fmt(OLsKDs%{)knVOskvu5G1R`?xOgraZ z6vacfm@q61~7!3IUkyy<=|60@e5I*9Hm;5(Un=i0&HW=?I2z>TH~^g$yN z_AV+h3_$WFmI25qBvqgenqtOaF&iD4kq_TU?Mhin-xw@swLJxKl3AR3=nmr;4)Vw8 zs>UBgelqB@wBAIrM@0Pq7!940P}*cM1QD7$Jr<^317Y9QQPDn#%`9Wzk~jq5)E4G5 zJX=GfJa<1_D*n$7X|Rh!*Xh7uT1^kjtHy{2Rh6d1_~oWf5Yj_w&b=hN+XoM%dX2roF1%y6 zxRx2-si?QOY7OUV4KNtUUg#r)ow~D(H7&77bKydu(7Azqe`l2#Ls9nfP<>HsTHq2R z|J$l0|NF8J%rlFLQFkzm(oYw>6K2+WU{*OK3u1wyI^zs~v7zbGP##*F*{Zt>MmMK&Ra{yOBbJA zGhtnR(YoTi6>e|U!G@2mwn=l=c=n2y_FpispR~;!F^!rg>{Sys=eXI~_*rGu1*z!x z-1Ab!q-4J+m7bSMkKcAds{BH-jAtx88@M2?p0ws4yY=|3XZL^Hn=hEhqH`T5B(Ewcps>w>G*p+U`?6MG4DJ3JV#$|DB=S%LojQG^7n;8{WbIt zAp2}~(l`cxAOxlPQ;gi-R$Mc6zwn_zBRL1oqZ2SMfbqgJt5cYbVSTzuNcVDM-Is5upLoVfiOI5Q1p31F4-(;H~ zsu3FC-7g6^59)}4)B=A%R*`{RiP@k(sGEFz&IApn(V#z-(6DLPa%ZY`a_WtW5* zco}DxCpeX;O>t*UV88tz0tdAKiAaXad(<60)!xbE$<1jZ8RFi + AND|OR + [metric] + THEN + [action] + [action] + """ + conditions, actions = [], [] + state, pending_logic = "start", "AND" + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("//"): + continue + up = line.upper() + if up == "IF": + state = "condition" + elif up in ("AND", "OR"): + pending_logic = up + elif up == "THEN": + state = "action" + elif state == "condition" and line.lower().startswith("[metric]"): + parts = line[len("[metric]"):].split() + if len(parts) >= 3: + conditions.append({ + "metric": parts[0], + "op": parts[1], + "value": parts[2], + "logic": pending_logic, + }) + pending_logic = "AND" + elif state == "action" and line.lower().startswith("[action]"): + actions.append(line[len("[action]"):].strip()) + return {"conditions": conditions, "actions": actions} + + +def serialize_rule(parsed): + """Serialize a parsed rule to the DuckDB setting string consumed by the + C++ DynamicRule interpreter: RULE##.""" + cond = "" + for i, c in enumerate(parsed["conditions"]): + if i > 0: + cond += f" {c['logic']} " + cond += f"{c['metric']} {c['op']} {c['value']}" + return f"RULE#{cond}#{','.join(parsed['actions'])}" + + ACTIVE_OPTIMIZER_IDS = {"noopt", "opt1", "opt2"} BENCHMARK_OPT_ORDER = ["noopt", "opt1", "opt2"] @@ -595,11 +672,14 @@ def _parse_synthetic_sql(): # --- DuckDB helpers --------------------------------------------------------- -def _build_session_sql(opt_setting: str, body: str) -> str: +def _build_session_sql(opt_setting: str, body: str, preamble=None) -> str: """Assemble a complete DuckDB session script: extension load, preamble, - optimizer setting, then the body (EXPLAIN or SELECT).""" + optimizer setting, then the body (EXPLAIN or SELECT). The preamble is the + selected query's own setup (defaults to the synthetic query's preamble).""" + if preamble is None: + preamble = SQL_PREAMBLE lines = [f"LOAD '{EXTENSION_PATH}';"] - for stmt in SQL_PREAMBLE: + for stmt in preamble: lines.append(stmt + ";") lines.append(f"SET enable_extended_optimizer = '{opt_setting}';") lines.append(body + ";") @@ -647,6 +727,14 @@ def get_opt(opt_id, default_id): selected_optimizer_left = get_opt(optimizer_left_id, "noopt") selected_optimizer_right = get_opt(optimizer_right_id, "opt1") + # Custom-slot state for the optimizer builder UI. + used_slots = {o["id"] for o in OPTIMIZERS if o["id"] in CUSTOM_SLOT_IDS} + custom_slots = [ + {"id": s, "used": s in used_slots, + "name": next((o["name"] for o in OPTIMIZERS if o["id"] == s), "")} + for s in CUSTOM_SLOT_IDS + ] + return render_template( "index.html", queries=QUERIES, @@ -658,6 +746,7 @@ def get_opt(opt_id, default_id): selected_optimizer_right=selected_optimizer_right, rules_snippet=rules_snippet, definition_optimizer_id=definition_opt_id, + custom_slots=custom_slots, ) @@ -672,9 +761,14 @@ def api_query_plan(): if opt is None: return jsonify({"error": "Unknown optimizer"}), 400 - if query_id == "ranker_q1" and SQL_SELECT: + query = _get_query(query_id) + if query and query.get("runnable") and query.get("sql"): try: - sql = _build_session_sql(opt["opt_setting"], f"EXPLAIN {SQL_SELECT}") + sql = _build_session_sql( + opt["opt_setting"], + f"EXPLAIN {query['sql']}", + preamble=query.get("preamble"), + ) proc = _run_duckdb(sql) if proc.returncode == 0 and proc.stdout.strip(): return jsonify({"plan": proc.stdout.strip()}) @@ -691,13 +785,15 @@ def api_run_benchmark(): """Run the selected query under every active optimizer and return runtimes.""" query_id = request.args.get("query_id", QUERIES[0]["id"]) - if query_id != "ranker_q1" or not SQL_SELECT: - return jsonify({"error": "Live benchmarking is only supported for Q_Ranker"}), 400 + query = _get_query(query_id) + if not query or not query.get("runnable") or not query.get("sql"): + return jsonify({"error": "Live benchmarking is not supported for this query yet"}), 400 results = [] for opt in [o for o in OPTIMIZERS if o["id"] in ACTIVE_OPTIMIZER_IDS]: try: - sql = _build_session_sql(opt["opt_setting"], SQL_SELECT) + sql = _build_session_sql(opt["opt_setting"], query["sql"], + preamble=query.get("preamble")) t0 = time.perf_counter() proc = _run_duckdb(sql) runtime_ms = round((time.perf_counter() - t0) * 1000, 1) @@ -720,21 +816,78 @@ def api_run_benchmark(): return jsonify({"query_id": query_id, "results": results}) -@app.route("/add_optimizer", methods=["POST"]) -def add_optimizer(): - global ACTIVE_OPTIMIZER_IDS, OPTIMIZER_DEFINITIONS +@app.route("/create_optimizer", methods=["POST"]) +def create_optimizer(): + """Create or edit a custom optimizer from an IF-THEN rule. The rule is + serialized to a RULE# setting string that the C++ DynamicRule interpreter + evaluates against the query's metrics and applies as rewrite actions.""" + name = request.form.get("optimizer_name", "").strip() + rules_text = request.form.get("optimizer_definition", "").strip() + target = request.form.get("target_slot", "").strip() - text = request.form.get("optimizer_definition", "").strip() - if not text: - flash("Optimizer definition is empty; nothing added.", "danger") + if not name or not rules_text: + flash("Optimizer name and rule definition are both required.", "danger") return redirect(url_for("index")) - OPTIMIZER_DEFINITIONS["opt2"] = text - ACTIVE_OPTIMIZER_IDS.add("opt2") + parsed = parse_optimizer_rule(rules_text) + + # --- validation --- + valid_metrics = set().union(*[set(m) for m in METRICS.values()]) if METRICS else set() + for c in parsed["conditions"]: + if valid_metrics and c["metric"] not in valid_metrics: + flash(f"Unknown metric '{c['metric']}'. Known: {', '.join(sorted(valid_metrics))}.", "danger") + return redirect(url_for("index")) + try: + float(c["value"]) + except ValueError: + flash(f"Threshold '{c['value']}' for '{c['metric']}' is not numeric.", "danger") + return redirect(url_for("index")) + if not parsed["actions"]: + flash("Rule has no [action] lines.", "danger") + return redirect(url_for("index")) + bad = [a for a in parsed["actions"] if a not in ACTIONS] + if bad: + flash(f"Unknown action(s): {', '.join(bad)}.", "danger") + return redirect(url_for("index")) + + # --- choose slot: edit target, else next free, else cap reached --- + used = {o["id"] for o in OPTIMIZERS if o["id"] in CUSTOM_SLOT_IDS} + if target in CUSTOM_SLOT_IDS: + slot_id = target + else: + free = [s for s in CUSTOM_SLOT_IDS if s not in used] + if not free: + flash("All 5 custom optimizer slots are in use. Pick one to overwrite.", "danger") + return redirect(url_for("index")) + slot_id = free[0] + + opt_setting = serialize_rule(parsed) + description = ", ".join(parsed["actions"][:2]) + ("…" if len(parsed["actions"]) > 2 else "") + entry = { + "id": slot_id, + "name": name, + "description": description, + "plan_key": "baseline", + "opt_setting": opt_setting, + "custom": True, + } - flash("Optimizer2 has been added and activated.", "success") - return redirect(url_for("index", definition_opt_id="opt2", - optimizer_left_id="noopt", optimizer_right_id="opt2")) + existing = next((o for o in OPTIMIZERS if o["id"] == slot_id), None) + if existing: + existing.update(entry) + else: + OPTIMIZERS.append(entry) + + OPTIMIZER_DEFINITIONS[slot_id] = rules_text + ACTIVE_OPTIMIZER_IDS.add(slot_id) + if slot_id not in BENCHMARK_OPT_ORDER: + BENCHMARK_OPT_ORDER.append(slot_id) + BENCHMARK_RESULTS["latencies"].append([None] * len(BENCHMARK_RESULTS["queries"])) + BENCHMARK_LABELS[slot_id] = name + + flash(f"Optimizer '{name}' saved to {slot_id}.", "success") + return redirect(url_for("index", definition_opt_id=slot_id, + optimizer_left_id="noopt", optimizer_right_id=slot_id)) @app.route("/upload_optimizer", methods=["POST"]) diff --git a/Frontend/templates/index.html b/Frontend/templates/index.html index d41e2d5..58deef3 100644 --- a/Frontend/templates/index.html +++ b/Frontend/templates/index.html @@ -314,6 +314,8 @@ {% if opt.id == 'noopt' %} baseline + {% elif opt.custom is defined and opt.custom %} + custom {% else %} optimizer {% endif %} @@ -335,29 +337,55 @@ Optimizer Definition
-

- Definition authored using a drag-and-drop statistics & action builder - (rendered here as pseudo-code). +

+ Definition for {{ definition_optimizer_id }} (use it as a starting point):

+ 
{{ rules_snippet }}
+ +
-
-
- -
-
- -
-
+

Create / Edit Custom Optimizer

+
+
+ + +
+
+ + +
+ +
+ + + IF [metric] <name> <op> <val> AND|OR … THEN [action] <ActionName> + +
+
+ +
+
+ +
+
@@ -461,12 +489,6 @@ new bootstrap.Toast(el).show(); }); - // Force the optimizer definition textarea to use the server snippet - const defEl = document.getElementById("optimizerDefinition"); - if (defEl) { - defEl.value = defEl.getAttribute("data-rules-snippet") || ""; - } - // Fetch live plans for both panes fetchPlan("planLeft", SELECTED_QUERY_ID, "{{ selected_optimizer_left.id|e }}"); fetchPlan("planRight", SELECTED_QUERY_ID, "{{ selected_optimizer_right.id|e }}"); diff --git a/src/include/optimization/dynamic_rule.hpp b/src/include/optimization/dynamic_rule.hpp new file mode 100644 index 0000000..063aa6c --- /dev/null +++ b/src/include/optimization/dynamic_rule.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/optimizer/optimizer_extension.hpp" + +#include +#include + +namespace duckdb { + +// A single metric comparison parsed from a frontend rule, e.g. +// join_cardinality_ratio > 1e-5 +// is_or carries the logic operator that joins this condition to the previous +// one (false = AND, true = OR); the first condition's is_or is ignored. +struct RuleCondition { + std::string metric; + std::string op; // > < >= <= = == != + double value = 0.0; + bool is_or = false; +}; + +// A user-authored optimizer rule serialized over the enable_extended_optimizer +// setting as: RULE## +// conditions: " " joined by " AND " / " OR " +// actions: comma-separated rewrite-action class names +// +// Conditions are evaluated against MetricCatalog for the query's schema; if +// they pass, the named actions are dispatched in order. +class DynamicRule { +public: + std::vector conditions; + std::vector actions; + + static bool IsRuleString(const std::string &s) { + return s.rfind("RULE#", 0) == 0; + } + + static DynamicRule Parse(const std::string &rule_str); + + bool EvaluateConditions(const std::string &schema) const; + void Apply(OptimizerExtensionInput &input, unique_ptr &plan) const; +}; + +} // namespace duckdb diff --git a/src/include/optimization/metric_catalog.hpp b/src/include/optimization/metric_catalog.hpp index 6f90436..c3e4443 100644 --- a/src/include/optimization/metric_catalog.hpp +++ b/src/include/optimization/metric_catalog.hpp @@ -62,11 +62,36 @@ class MetricCatalog { return schema->Get(metric_name, default_value); } + // Resolve a metric trying the query's schema first, then the "_default" + // schema, then the supplied fallback. Used by the dynamic rule evaluator + // where GetQuerySchema() may return an unexpected name (e.g. "main"). + double GetMetricOrDefault(const string &schema_name, + const string &metric_name, + double default_value) const { + if (auto *s = GetSchemaMetrics(schema_name); s && s->Has(metric_name)) { + return s->Get(metric_name); + } + if (auto *d = GetSchemaMetrics("_default"); d && d->Has(metric_name)) { + return d->Get(metric_name); + } + return default_value; + } + private: MetricCatalog() { Initialize(); } + // Populate a schema set with the frontend metric names used by the + // optimizer-creator rules (must match the keys in app.py METRICS). + static void AddFrontendMetrics(SchemaMetricSet &s) { + s.values["join_cardinality_ratio"] = 0.0002; + s.values["feature_nonzero_ratio"] = 0.002; + s.values["feature_width"] = 150; + s.values["card_search_impressions"] = 500000; + s.values["card_listing_metadata"] = 200000; + } + void Initialize() { // ---- Define metrics for each synthetic schema here ---- // Example schema 1: synthetic dense features @@ -74,6 +99,7 @@ class MetricCatalog { SchemaMetricSet s; s.values["join_ratio"] = 5e-5; // big joins s.values["sparsity"] = 0.20; // 95% non-zero -> dense + AddFrontendMetrics(s); schema_metrics["synthetic"] = std::move(s); } @@ -82,9 +108,21 @@ class MetricCatalog { SchemaMetricSet s; s.values["join_ratio"] = 5e-5; s.values["sparsity"] = 0.10; // 10% non-zero -> sparse + AddFrontendMetrics(s); schema_metrics["syn_sparse"] = std::move(s); } + // Fallback schema: used by the dynamic rule evaluator when the query's + // resolved schema name is not found above (e.g. "main"). Carries the + // demo metrics under the frontend metric names. + { + SchemaMetricSet s; + s.values["join_ratio"] = 5e-5; + s.values["sparsity"] = 0.20; + AddFrontendMetrics(s); + schema_metrics["_default"] = std::move(s); + } + // Add more schemas as needed: // { // SchemaMetricSet s; diff --git a/src/optimization/CMakeLists.txt b/src/optimization/CMakeLists.txt index 08b38db..5ae8696 100644 --- a/src/optimization/CMakeLists.txt +++ b/src/optimization/CMakeLists.txt @@ -4,6 +4,7 @@ list(APPEND EXTENSION_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/replace_binding_rule.cpp ${CMAKE_CURRENT_SOURCE_DIR}/register_optimizers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_optimizer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_rule.cpp ${CMAKE_CURRENT_SOURCE_DIR}/catalog.cpp #All Actions diff --git a/src/optimization/dynamic_rule.cpp b/src/optimization/dynamic_rule.cpp new file mode 100644 index 0000000..0425daf --- /dev/null +++ b/src/optimization/dynamic_rule.cpp @@ -0,0 +1,166 @@ +#include "optimization/dynamic_rule.hpp" +#include "optimization/metric_catalog.hpp" + +#include "optimization/actions/MatMulDense2SparseRewriteAction.hpp" +#include "optimization/actions/MultiLayerUDF2TorchNNRewriteAction.hpp" +#include "optimization/actions/MLDecompositionPushdownRewriteAction.hpp" +#include "optimization/actions/MLFactorizationRewriteAction.hpp" +#include "optimization/actions/MLGreedyFactorizationOptimizer.hpp" +#include "optimization/actions/DecisionForestUDF2RelationRewriteAction.hpp" + +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +// Trim leading/trailing ASCII whitespace. +std::string Trim(const std::string &s) { + size_t b = s.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) { + return ""; + } + size_t e = s.find_last_not_of(" \t\r\n"); + return s.substr(b, e - b + 1); +} + +// Split on a single-character delimiter (empty fields preserved). +std::vector Split(const std::string &s, char delim) { + std::vector out; + std::string cur; + std::stringstream ss(s); + while (std::getline(ss, cur, delim)) { + out.push_back(cur); + } + return out; +} + +// Whitespace tokenizer (collapses runs of whitespace). +std::vector Tokenize(const std::string &s) { + std::vector out; + std::stringstream ss(s); + std::string tok; + while (ss >> tok) { + out.push_back(tok); + } + return out; +} + +bool CompareMetric(double lhs, const std::string &op, double rhs) { + if (op == ">") return lhs > rhs; + if (op == "<") return lhs < rhs; + if (op == ">=") return lhs >= rhs; + if (op == "<=") return lhs <= rhs; + if (op == "=" || op == "==") return lhs == rhs; + if (op == "!=") return lhs != rhs; + return false; +} + +using ActionFn = bool (*)(OptimizerExtensionInput &, unique_ptr &); + +const std::unordered_map &ActionTable() { + static const std::unordered_map table = { + {"MLDecompositionPushdownRewriteAction", &MLDecompositionPushdownRewriteAction::apply}, + {"MatMulDense2SparseRewriteAction", &MatMulDense2SparseRewriteAction::apply}, + {"MultiLayerUDF2TorchNNRewriteAction", &MultiLayerUDF2TorchNNRewriteAction::apply}, + {"MLFactorizationRewriteAction", &MLFactorizationRewriteAction::apply}, + {"MLGreedyFactorizationOptimizer", &MLGreedyFactorizationOptimizer::apply}, + {"DecisionForestUDF2RelationRewriteAction", &DecisionForestUDF2RelationRewriteAction::apply}, + }; + return table; +} + +} // namespace + +DynamicRule DynamicRule::Parse(const std::string &rule_str) { + DynamicRule rule; + + // Expect exactly: RULE## + auto sections = Split(rule_str, '#'); + if (sections.size() < 3 || Trim(sections[0]) != "RULE") { + return rule; // empty rule => no conditions, no actions + } + const std::string conditions_section = Trim(sections[1]); + const std::string actions_section = Trim(sections[2]); + + // --- conditions: walk whitespace tokens as [metric op value] triples, + // with AND/OR keywords in between. --- + auto tokens = Tokenize(conditions_section); + bool next_is_or = false; // logic for the upcoming condition + size_t i = 0; + while (i + 2 < tokens.size()) { + RuleCondition c; + c.metric = tokens[i]; + c.op = tokens[i + 1]; + try { + c.value = std::stod(tokens[i + 2]); + } catch (...) { + c.value = 0.0; + } + c.is_or = next_is_or; + rule.conditions.push_back(c); + i += 3; + + // optional logic keyword before the next triple + next_is_or = false; + if (i < tokens.size()) { + std::string kw = tokens[i]; + for (auto &ch : kw) { + ch = (char)std::toupper((unsigned char)ch); + } + if (kw == "AND" || kw == "OR") { + next_is_or = (kw == "OR"); + i += 1; + } + } + } + + // --- actions: comma separated --- + for (auto &a : Split(actions_section, ',')) { + std::string name = Trim(a); + if (!name.empty()) { + rule.actions.push_back(name); + } + } + + return rule; +} + +bool DynamicRule::EvaluateConditions(const std::string &schema) const { + if (conditions.empty()) { + return true; // no conditions => always apply + } + const auto &mc = MetricCatalog::Get(); + bool result = false; + for (size_t i = 0; i < conditions.size(); i++) { + const auto &c = conditions[i]; + double v = mc.GetMetricOrDefault(schema, c.metric, 0.0); + bool pass = CompareMetric(v, c.op, c.value); + if (i == 0) { + result = pass; + } else if (c.is_or) { + result = result || pass; + } else { + result = result && pass; + } + } + return result; +} + +void DynamicRule::Apply(OptimizerExtensionInput &input, unique_ptr &plan) const { + const auto &table = ActionTable(); + for (const auto &name : actions) { + auto it = table.find(name); + if (it == table.end()) { + std::cerr << "[DynamicRule] unknown action skipped: " << name << "\n"; + continue; + } + it->second(input, plan); + } +} + +} // namespace duckdb diff --git a/src/optimization/register_optimizers.cpp b/src/optimization/register_optimizers.cpp index ebd4366..46c7a98 100644 --- a/src/optimization/register_optimizers.cpp +++ b/src/optimization/register_optimizers.cpp @@ -5,9 +5,10 @@ #include "duckdb/optimizer/optimizer_extension.hpp" #include "optimization/add5_rule.hpp" -#include "optimization/replace_binding_rule.hpp" -#include "optimization/dynamic_optimizer.hpp" +#include "optimization/replace_binding_rule.hpp" +#include "optimization/dynamic_optimizer.hpp" #include "optimization/metric_catalog.hpp" +#include "optimization/dynamic_rule.hpp" #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" @@ -132,6 +133,17 @@ struct CactusDynamicOpt : public OptimizerExtension { in.context.TryGetCurrentSetting("enable_extended_optimizer", setting); std::string opt_setting = setting.ToString(); + // Dynamic rule path: a frontend-authored optimizer serialized as + // "RULE##". Conditions are evaluated against the + // MetricCatalog for this query's schema; matched actions are applied. + if (DynamicRule::IsRuleString(opt_setting)) { + auto rule = DynamicRule::Parse(opt_setting); + if (rule.EvaluateConditions(query_schema)) { + rule.Apply(in, plan); + } + return; + } + if (opt_setting.find("1") != std::string::npos) { if (join_ratio > HIGH_JOIN_RATIO) { MLDecompositionPushdownRewriteAction::apply(in, plan); From 7d58edd0cfc335b3e3d2057cf38b37c283cd657a Mon Sep 17 00:00:00 2001 From: jrt1899 Date: Thu, 28 May 2026 16:31:38 -0700 Subject: [PATCH 6/6] Completed the frontend with new queries --- .gitignore | 8 + CMakeLists.txt | 10 + Frontend/__pycache__/app.cpython-311.pyc | Bin 44091 -> 71236 bytes Frontend/__pycache__/app.cpython-312.pyc | Bin 39670 -> 52877 bytes Frontend/app.py | 582 ++++++++++- Frontend/templates/index.html | 195 +++- README.md | 63 ++ baseline.sh | 28 + .../greedy_factorization_pass.cpp | 913 ++++++++++++++++++ expedia.json | 34 +- plugins/opt_passes/.gitkeep | 0 plugins/opt_passes/_manifest.cpp | 13 + plugins/opt_passes/include/.gitkeep | 0 queries/expedia_query.sql | 62 +- queries/flights_query.sql | 90 ++ queries/forest_narrow_baseline.sql | 26 + queries/forest_narrow_optimized.sql | 27 + queries/forest_query.sql | 6 +- queries/forest_stress_baseline.sql | 26 + queries/forest_stress_optimized.sql | 27 + queries/uc01_query.sql | 74 ++ queries/uc03_query.sql | 75 ++ queries/uc04_query.sql | 46 + queries/uc08_query.sql | 97 ++ queries/uc08_query_factorizable.sql | 63 ++ src/cactusdb_extension.cpp | 3 + src/include/ml_functions/kmeans.hpp | 44 + src/include/ml_functions/one_hot_encoder.hpp | 12 +- src/include/ml_functions/relu.hpp | 22 + src/include/ml_functions/text_functions.hpp | 29 + src/include/optimization/metric_catalog.hpp | 65 +- src/include/optimization/optimizer_pass.hpp | 70 ++ src/ml_functions/CMakeLists.txt | 3 + src/ml_functions/kmeans.cpp | 135 +++ src/ml_functions/ml_functions_scalar.cpp | 8 + src/ml_functions/relu.cpp | 90 ++ src/ml_functions/text_functions.cpp | 205 ++++ src/optimization/CMakeLists.txt | 7 +- ...ecisionForestUDF2RelationRewriteAction.cpp | 132 +-- .../actions/MLFactorizationRewriteAction.cpp | 117 ++- src/optimization/dynamic_rule.cpp | 2 - src/optimization/optimizer_pass.cpp | 84 ++ src/optimization/register_optimizers.cpp | 60 +- 43 files changed, 3286 insertions(+), 267 deletions(-) create mode 100755 baseline.sh create mode 100644 demo/uploadable_passes/greedy_factorization_pass.cpp create mode 100644 plugins/opt_passes/.gitkeep create mode 100644 plugins/opt_passes/_manifest.cpp create mode 100644 plugins/opt_passes/include/.gitkeep create mode 100644 queries/flights_query.sql create mode 100644 queries/forest_narrow_baseline.sql create mode 100644 queries/forest_narrow_optimized.sql create mode 100644 queries/forest_stress_baseline.sql create mode 100644 queries/forest_stress_optimized.sql create mode 100644 queries/uc01_query.sql create mode 100644 queries/uc03_query.sql create mode 100644 queries/uc04_query.sql create mode 100644 queries/uc08_query.sql create mode 100644 queries/uc08_query_factorizable.sql create mode 100644 src/include/ml_functions/kmeans.hpp create mode 100644 src/include/ml_functions/relu.hpp create mode 100644 src/include/ml_functions/text_functions.hpp create mode 100644 src/include/optimization/optimizer_pass.hpp create mode 100644 src/ml_functions/kmeans.cpp create mode 100644 src/ml_functions/relu.cpp create mode 100644 src/ml_functions/text_functions.cpp create mode 100644 src/optimization/optimizer_pass.cpp diff --git a/.gitignore b/.gitignore index 9d67701..b0da62b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,11 @@ archived_tests/* queries/synthetic_query/*.csv queries/synthetic_query/*.db *.zip + +# Uploaded optimizer-pass plugins are runtime artifacts; only the baseline +# (empty) manifest and the dir placeholder are tracked. +plugins/opt_passes/*.cpp +!plugins/opt_passes/_manifest.cpp +plugins/opt_passes/*.rejected +plugins/opt_passes/include/* +!plugins/opt_passes/include/.gitkeep diff --git a/CMakeLists.txt b/CMakeLists.txt index b00bbb8..02e96d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,16 @@ add_subdirectory(src/ml_functions) add_subdirectory(src/util) add_subdirectory(src/optimization) add_subdirectory(src/cost_model) + +# Uploadable optimizer passes. Files dropped in plugins/opt_passes/ by the +# frontend upload flow are compiled directly into the extension target (NOT a +# static archive) so their REGISTER_OPT_PASS static initializers survive linking. +# CONFIGURE_DEPENDS re-globs when a new upload appears, so no manual reconfigure. +file(GLOB OPT_PASS_PLUGINS CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/plugins/opt_passes/*.cpp") +list(APPEND EXTENSION_SOURCES ${OPT_PASS_PLUGINS}) +include_directories(plugins/opt_passes/include) # uploaded .hpp headers land here + message(STATUS "EXTENSION_SOURCES include ${EXTENSION_SOURCES}") build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) diff --git a/Frontend/__pycache__/app.cpython-311.pyc b/Frontend/__pycache__/app.cpython-311.pyc index 2e983343c2d759d09e0c97f0e743e6b438b8ea9c..5eaa430fa0b9e2dfad249af196dac403bff2765a 100644 GIT binary patch delta 25778 zcmch=d30OXeJB1N_8nZnRa_pmk)SCq+Np&iB~mLzNhGzHk_Oa`<^ zu_>8}C_THPZ7HFbIC8AiO4P+|r-@@b9=DmafQr;8TAix-O-`De&Ivt^r}}w1&3wN1 z0pKFp^T(W-r^heuyZ3wd?YrHpKTD`SxT&#y+iEp&c$%&joV@h2%eF$*SB#gdjzvUH zRQv2bcJ`aqlcnN3yeIn&p5xxapYrtNh#F6hD>wZuYI_{OGo8#!`*<9p?xy|?6=LB} zd3es_@Z_A#Pk)PsO>iPsLE2}!kduEtl53mqC;!ukW4>1rd8s_b@;9C-k#9a@%j#J) zFG>l8FZL`Fjq^*;Q~Gc;(*m@zhohMnpp`!y&9VSZco40|BZ$@oXceNZXR&A()%ces zW?$3vEJ2&vPcD_eXU>oGEW?NSoLX$Zn-))%@Zfqx{zNtYu-V9}>2%XT4Y^mU|k8cu^25uBpYv;u3Kwb1f5>i}m8gwsOZ?dp3AB zOmD=P$}89)Zk%6y4C-{)CQO~9RW9=*diWB`dcI7WBA#I z;qSumZ$;oy1h$Efi*E6R*vq197e&z{_KE%Cfapc$C&g2u5B~;5zZkft6NkhnM2YzX z#gI6BO)s9lrtaAxo)OQA=fp7FJH;o(5pndIs%IB+_{yt1V1&WrW9;WC2A^i|KQMTn z!LI^p#AmK)dv=T85})<#5y$a&!Lz4NCtf_S?P(XA*|+uN-t;i((}>TV*YtFFa;Epu z$lN?3eyyLs`StmuOuTfSn?I9#_KTCLv^C;oq;MdeLgkhD9VyBW#O=)BKED9>g$HoE z(zq*rPrUk&l1)7jqdODh#fQdt>46vr(=jSvUQnKE58xh3_2kQbuA=i*Q81w{_dau9NH!OO>1$CjpAqqL+@%Ik6P7Hf}Vy!RGpVW9o zpC`$^sY&XBA$Mpvm^4ycm)||;N$P{XK*;44llnnVFzD|0gc~~BcI@ivZtHaI+u!Zl z-_q4}ELeT4>R2#RU3Ii&MUQ($*mXSRs9(|OI$rGzFMbR-OYoN}3|vPyEk0g#^s&WF z#~*PX1#}{Nak%%Ccp_{?A-#j%ush@p`0K*fL7%H{*xyTE!Md=yHxLZD1_PqUSJy;T zMN&QJ9!jc%5l<+obxV@_TvFRR=nf4ewNDIthCNAbXm|)U)cKHTD45g@dIkg1IjXJX z_4g-rr#-!)fRxnt21A(VeTu{D4<+?|K=HT-(F$qDaLDWPhR!7oUX+4`8K|@YzuVWD zG`fBL0m&O07)+YdC3gFg)?t4@5WxQ5=vBVl-DM|@|8S! zizz#zlyv^Wr%awDTmu)>oKc-sKgylqdG1k;$3L1@PFwh1r{@g1(IY%KpX$(ses7;A z7^>|Z8WMs5q2230wQnf2A5&((GX1KYg5>E#e@Fbio}fv<<2mh-&I!H4lH~D+eCJjS zNuD5FsSGNEf*W%(APK>9gC_#MTA_Qu8%!mFGz5H{8E}V$Qy$MyDnWPeKq|HnIB^mq zSn&FV(11svIqdBfd?;2#s0w;KLTWmBB(^5h4h#)BYr`5Uf7m*&Ivu~FK_z7a^v)Sx zoCY4N@b~FrfIVoZtK0~8+%(FI+z5}(6l+wylt^? zI^Y$l5NuUZD(VVrs+=259dB}Ps*h3*NC3&!DcqH^~$kpo(hBmQ^S2tBHf1vVCs=QQ>Qb-K|sk%$6$zu(` zmsGip@V5XcRi5EgxjF_Con2AGUR2(&S9xdD>7ZGpS(mgHX$8C{~tt;YpK;HOxaN&XcC_Vq#ZiK}jmzl)|Y9rocv1 zxY&qN`vd72in-G74WjWqqA6T7YBG%q6uuy9Q<6X9v?o=8U{W*0HVAJ}^h!z7{`P}A zc6GQ~cXeV{2uL9h_B$Waqd)^1Ab19gdQuY@^7tc3&B=h*pEPH-!Ok{B^0=j5U|>fO zlQe=6DXHz30>eW|En7>R`lQY)V$KJXsy=^`?@RK7Nv-JV^`S6WE&xvi#B)K~0c1d*|NsA-_;w>pLurl>CqQ@VtOI27iKuUw)$>P z;q{zr#oyW;E2@hZ)x~n^<2m&+oX%Ly!1&fgZr;_6S2j+aj^!?o=PsY&cx%y(#kcHt zs<%X|wyp4~V`=aKC zTdgtk`Y7I=eV>@M#sPlD!kZU=V$~ZL&!{=A@iPZ!vVHBuME@nN%-NyF~SfYX06GG>D)eMpW2y zbl25=*lR|(X>FQ3ae(3(7*&TX=>#}#S`S(vU;Zzl99@u~C?CL&zaFUo0jvC;}803S+pSRj8LfRj-QYP#~44p@KP0 z;Vqq>(6Hnesxstsjo^`_faI(dGG4T|cLe+%Z15hx5=>s83~QaLj_}5=z_5h1bHE*> zmAD<*QH(Ig0Cb9TVxF4O^R z45O)BLDmdx&-VZ|NW13!)TVFS9<^=1!H=K**e=|$3)9&#`;xeQ2|O3PY|9JhwR(eY zAGR`&C}g&*#o^_Pg{r`CD2PRxZGJ;ifXJ#k>;h;SpwountaV1r5>;K=MqoQYQYCqk z#=c>n4@?I(0aI`YECO{vFlpM=v90}JYnuz3gjNa8ZE0`ccet(9)z#hV+S%69+Sb{X zRJ;A>l7;Dzbfo>AZQFMpY3p+8l9~`GDrpb0i%1U2R6`jgRo+4AAU=aS0NOKzW@XpN zeMsv`ZXg{ZYy+n4{W0zrhTM-0#di$FQ>$YJA#M=HS`uaptsz}}!rmFRcP6s3uj;Sp zr})dJ=S^d~?wYL=kxkLUsvFLjd3D^pns5&QH#eHMf4UJM_1gDG4f_*Wj;r=7_E=U$ zJgZ`cGio2>6*#WBYc2TLT6)J?Iwi)eOXAigt!d5*9}yJNIhgmBOit%Zr|0 zG_FgSvwxOX6*beFutXep&9=E6`?D2o_>9@M#{tdTbCGpd=7E!P^#t@Gl`|Gu`&G@?mM4JogPq#Xf|CGsSH13Q#?=B`|# zX8w8!hDbYi9XrY7`V>{vin@MPC?j#_xDhQ_zRYtnvxtiN5%o>O8+jOEZ_y}Y&!|o` z(m!GXYe#e^t?2;Kq;QvkCG+c;4hM4vW>Ot0V?1rmsNR2Q2^SL5!5p`glXi_7LW^g8 z)^I!*!agONBjpul(K2FqnY*sKxj;w`<}FjALyzKP)bJI9XoQ%Axz6xsIEB|6(MyLz z&e`Ngz{6mlL7%;q={IEf1qJB0jc}r!?Lb*09G3p<>)M++3u7xqnco*wz%K3!_)dh( zZ!a1Fn?ludib;%C?)3#5;;3%ELIXtE*Cs9PyS8@k>)f`Jc!jX(D|Hc{_XLDTy#wLw;DB55)Isue z%2U@(UebEBXi`HSNxkIpdE7xyw^N%`_nrZhfPDqq4f_RAAU#I7YLEYPheS(3Qp+fX zaGp|qpBnCRH!1=F5(F4)B@bBxwh(BhAmi49yV_e_?fZ6Mvm8EwotBzD zsg{QQNz4AueY@MXb-OzE?dwkJuu1Ma*qtA1WiV&GzqrE$m7abv<< zG+h)mSL2P`Eh)RU>ffi2RyAyLyfX`HGB({$IG{ep9K!()BY?jpFy1<&Mm!doflnxz-x8Shdzgzc zDG$NQ;cPh*u4(>`wIXV5{nm-s`@ijf)&GZ&#g@0mm$#0q##Q(4nj?-GPOY`i0#FvT z{da@ZAb8hO@Uf-rj-_n6Kf12-qbC69wUosyU2#iS)Y1h8D}$}X0e=b~@K_7{=cb!> zM+iw_vmT0l&tMzZrT+o%{)`^QrC$?gFwJmQ5Jh(li>7u=KYB|OGpval);A)X1*xF3; zivO2GR>g-zi7ZIvGZTlgm*Q}5IHQ$K${Vw7iQBe}tBHbKcd6-O)6|lfxh!rjqlT=% zYss0|Go_yDn%)>It&Lgg;+DForS7hwBxWd!8_J@FvQ(2038^xo%~JguLB}R4kNx?s zXyq~G%{0VgtRW}@Zv{j{q$4XyP(u*TLVqRni(slD(n&?W2rqaZv-s()xvd87y}X)j zR`vV51@8B)nr+$o_p{aHcJSoRTM73E22mhph|C|d(c zMP$yW|3;iUXxj_Kv4TA(no3lGDD4 z8PQ1jBbsRyao_p#i4dOi7ZcGU*FICWc|^T1P&6`;vWY=6gO=<15S2UiT;Lc9Y=W+0 zZF$uNZea=+xR5Q4%W)vi579SexA;NixwMODEn;Y8qM@IkOE%9#vdR0OE0p(tEia-9 z(To$-JAy1v;VU!7I2~t|EH~|8HNy16H<$<)HXpb{q2rY$BKmbR- zWK%{i&{P}Fo)d}uHwZ}sB|)6u=bC*v!0#qOW-tihqRJ~uPs?9jS!LDrf}sx#CN-zL zeo_97w__W~g|DNSiop5JH1i!`yV^rJ?+5or;=HxfNaR!bf68kkBzf!zZ-R_<+ruhG zWfH0ssj)|dY(k;P#1&q@({y-eTL<>Q@Z+^iWKo4C@;l*y#Gq0e?!`v$_6hyNZb>w4 z+qrLFR~ylHeqqq-r*?zNPKXfoN0~r#!es7Hzp*9;c>=wlsb&BE_D7LL`Rsg%TW2b5aA7Z=dWQ za^xo+E!(|2^LBoo z4@3M6;>+Tx%9v*HXBJLdB>(8eljf+V6ffhQZ$GWSuU1Kea`}}aSvH?o$}}5q@<}Z^ z)H{UXr-*xkbM+kZAqqHH@>fG( z2Gww&zaOFcGj7R`esEf(-7KYE`TN$!-I9x?&At07^{ zhd$QMtK|QtuhoevXj4@r+2qyoe>AKx{t4J~&8U7v2fyV8NJTxPh7kiH$bV+AYeYR; z=T_*!_DQQpj0j-K<(WoIQt^n1!qX3vU@+xxEp)O-6w5b1(Kn(O4G@#6M|7jwiD&R- z%qW$=1|6<$RwyeACav~$>;<}%RyOC6KZPJeOEOY${&QApaQ=0p=DBO_Nu&Jx#xjd& z29Iu5cx?G~(+Zts_`jfha@^GL2;^qM!EFLf+;akkhR_ENsg^N(T{QNBLi@g!R>lv> z#qP3Gq`W#22n`6})v$CZERO)yDhP2H9e7wwXtYv9IV>oM;iL|PQkE9=pm2Cs_s-hd z+OD>Cuqna`U!eCC^tVCAS1RQoMNVyp9GT3g*-)qG`QAj4C9<2FI|-!k!XsRf8Zg6b zq{#1HazySC_U{v*rB-^I0O@7f%#d!8i#B1WO8Optk{T*hQbTo>z7LOZ>9)YIPh?6u zHW;aqsCkxQ7g>e{j!i3N3jHA^Msq~^BLaU+AVLcrn;_CV(2@NfB;Qq>0JUpB0!TydXK>S1ioa#Sm)JW`U{S$;QkJAf^QkTdsp6a`k zy)>G=6bq;P=k`Ok4ezwRTM;v~#tp4eL#te#wJc(96}ZcJE7fmU;P}2~t^SYHaQq3s zR{y8lmcjLbU}`H;e^9-&&7uCWjfeZk4lQ6fKecbv4focsacfy)5$&6UYWk_>w?4d4ma?^Eo2JkqKS!d^_>ISE5+GT`kW5Vp7P+s|e< zZOh>`U9_MJ5Tv^XhkcN6NC6NUq>sgQ^_y2v#T?bDks@)5RZ zu%v0d_?#CcHRS`DDzsb#pta({+RlURZA(7q@z$UWG+2^G4^sw#I%0b(8ECOg#iXV; z;1{J;NH59v1~sYDDMQExd5_oa1AWViq?dA$+D}#T`KC9EX&f~X`keM-&G{R7>y|1D z$gRwPkWX&890}`It*=?%xHhbg{LwYBRSg{@)js+yJc^-jH1Mh&-!Ksf<{Km~!t1eO3`I{qb9G)f=_ zFnkWZC|}-QBMZ&da!IHCwx{`DOLA%INWUiVUkT|E(>x@bEbQ@A2VBg23|4qG0tv5b zgn zJYFMYq6-j!dRzlA^aCLb0=i0s7>!8sSqx{5aJa#B22#Hop%P{cK~JSqkh2avCU0&z zAzy91X!m(t5PHGJgUkhm&2oKPBdh8}tI<9*H%JiVr`n3-f9$l%f4yHXmv^{Q&0>*% zzr9+n-=~+)w;xn7w0!xI&TRRgJ8h*Yu^@zfLY2F}AJo1tm?|Q}%(ZpXlDa`i6+*Um z1aev9P*YW{t6wM3Sa8bk?$4JuZ9m91de(}99BZRqSt}29XtpQ~sssovJ9a?h3b|qG zdv1$#9o$=mc16SN-PKWb#^X69y3aY~>;uPBO;WySw&OOVb<&+kiCmK_nrrw;IlF6x zJh}Uk+g01m{7OMcRYAcFo$#@2QHe}FQV7HW!ai+K7TVXcMjkwHL_7T7%nuHpqqAj5 z51v$AGJVU&i(3?3r%zWc@p<|%JeEjUdqWuvwDZrOnS_lEBddey;r4&O`yKMDEjjWp zJMFTqT?cobJke&dveroL0abEETfQ9KZ_G}6WV#BYEYGX7+{1=D{Ogf&vEpfOBO zrW93~bx0YoG+XjgL)3!7Qcc4&J#glZVr6alvPnd(Xqp|p|19 zwfa9(oSzc-D*}H_AhUi;cj)tP2+UvDotC6((8owJsh*a928cp?=;u$M;hYl!g2F+O z@(4iB#z_SLDCplG?{U!Hi08Rx#ZfrfjDU5IEsHb|;u-?XwGys=hC1?N9B-iK`R_GL zs|aB=fYYKRaPJ$blqUZ)17l;X=J1)_7C|VA1k&?FTszI zk5UI*x9u*iihoph) zw6I~TP%SwR@_W(_l<*(a^7q_Rpr!VmDC65nM$ zYwta7y*(|iqK7b9 zw?G(dWd7<0gv2H>D;0T>**0!Nsj^Y397^mcM}z2R87TXP(i5fvN9}Be0gZ zU>VVkm~ZC30jbPeB$;6kN)XaYVi{b(A^&X3*ZDxc zS1I#@7!eEPuvCV2BWUFy6)$b+RD2*>d|*2JR?98%4|YVi?7y>me{}W!sRn@Ammw)) z1DR6)8bGf~ernPo9~=0$nuiu4sZ)x%uoV82wJN(t{=S#S>O&jus=WVXVY=Z6JzFDx z<78t!H5?lEn^Xc~JEiN2$D^pLbQB<|MwTqU#~;Tuk;_jlRarE0+o?fhMS{fdj;*c& zk4h{hekh?g6+xmzjW%D3BHIFGDu+hi>8ne{_FQgR=0Y}*LfW?-?CRdP*VWa&ubV`) zlAGF_Mgz%N<$v{6sfsjm#o$Ur?H>GxI?9nTv5LSvJw#Yn-K4B(vd$W-$u57Dszf7S z@ULUFW^x)tqGXa9=)WPDhcm@~r`v&zx|{NhKh=2OWmPEF$hCoia%#M=3F3Rt+1i27 zpbzpIS|1r_CT->AKZG52&YUsk4T6(_2Hm2F{+TvAh;2>Fir|*|pLTQB z0)E1<%_$N(R~n*_;}nt&*4}&CLz6+#eFP;R1r8E;f&k3{sgr;gAZSMkggH-hbF;Ei zYgwyEVJooye{@9f*KpjI3!eC!y`LGl{9;Uy%=@jAaK?%c#NpRYs;b)iM)!LBC7?q zzyQA`KssBUGQAAQx)*<$CnL*!3BFf3d3w!q$0$$BdioK0ta)1b2hSACe`&~)j~H^x zRZl&MRZ|@X>o=-~g_q_OH9s~n*d?hIjg)-k<*c&cas;Zw1q;v?%}3L{yFsMWmgCIl_1PA zgb)+8DFyQg5Z8i*R`4PzM211<7_ewVl+e#YiC6%a=?}TFlEbeO_AEZQU8g+fDp_h| zWOK&tJH-rzs9e+xew+lr=^)6X&aODY7&+FA2!oKB4iG?Ng|kC!)X1NCvm#aK>x>nx zl}uE@3%yWt{e|=KqF}~F5q);BpVUlVW8INKL;#aAI z(vP*9d;$4jy*;vlHX7OvQx>&`rl_ImLTLPn$%-4B-)V_HvKjAqw(^^T42R8w4rXV} zw%1`A>OX&ef{Ce}Iz7pfX-iiOR(cO>slh zyP8kbsvKPeMgTdwvE9&z8FHff(tBCOQwJ{F#9O|D~;JJ0k6ahZQJYilqoFMR zzstVRb$*j<8g3|G60)KZQjeG&T9kIfyi;@1EW?g8$IH(rPFXYd>oTDk zGOGbe(m_bj547f3t->Yk*pn3HvSRHfR7t&?mx)f&k^HI#sfCx$Ly`abY@u$FGI&n@ zuV)uUo&ieI&+Jn9debrH!Cw zke&nXcA7ytmHbcVuGCqzyZOnyt3_9erZ&W~m&dc0WA}sKjWusKey8!)!Pp}kSl|_Cv{6yE!sC&n+@;*K$A3UIqA}*0jA*{h?jh zzFPg^YVG#M+yDM#v!;0dNSo99C*>n!8&&6dS-4UNT5?SZQG?Rs1l#}_`j1-cyYjz} zRjUFTx%#QP1|q%`){%ya^i?2x{}~g3t#TdGvy=iO=+vx%VRdowsk#DcYen<%b#l=V z1;v+?qfa?-Rs=q?1A;Uw%2*(}i;V@?_e1%Zv_M7Or^slbq$Wl&sbvPJY?vf1or;pO zjSi70!-BE9G&bf88p<2zp+gch3)DQ_(nTu3B?89?(CkR6LiG|GFRZ*YXdb3#$fmKv zj1=kA_@G-lCH)?f@Mv!T@zdoNK3o3y>wT&)FK@ZDYZ>FGc!`w9s5&EX_fWEFN*ivp z^gm5HRWEAfzrD2F!djZ?COBR;OfEG@e!@}l^2*5?6;vUQPd>IJmFj8ano2dvyC0I~ z8yeYixf&$=y32u5*4MPjKG@eDUXID-RyukOtCXgj82F#czqzdN4I7-4onpR_;tYx6 zmj;m3{AGz%M0<{l0pQi*I*>!2ib)a@owKRBqc3`vY7=yo92c+%pnR;h94Wd;1)Jrwvo*zzc<2vXFn$iafnIAkWltK`DpIG!Cq! z23Fo4Wj->U;hxOkC~DyK6>NQsao~g839R5yW$IQN%2E;34~h z8Rq>t*R>SEffIdNF_*lqt8fzB0luaLvYSN(mQZ=B+3+)NhFm8PavcgQUb?W2Mztea zSdnN)b^cvUFqMDaAU9nsk*6-^@&%O7F&LK=&h`YZ0$4Je`pKNMJG4CQLeCZ5ES}#o z^Up7wbfj^wOyutl)ug{LRH%ZB9ynxS2-VHvj2ISPX9S}hSkg=vKGfK8w*Tb@v8J+g zKJhh5CN~F0OtVLZq}?N?X?BZM{@gPt^U7Q@x3-}*bIOK1b##71 zK>fdDq)*eOSb3HD3+54>SjO6*9A`L$H?Zuyg+JxN`4u+EX((^rnD)gnlCrcr)RcCC z+|(g8zE6$5sF;5`W?Eo}1i_6KqgZnUlj35Q^^)sqaVe(N&WLka_})R~T@xwsORaQ`7RNAwT(mfhLMe0bsNuCOse<4H zsoO_KR?g8;7N|r~N;=$f*o~55OP~Ux)+D414%9Nso(}0nN|EM9SUVi*ThX~*`U8C6 z=%xE+VZpU}?t3YCj?zIn118kofLe)>4ToP(%U@;A-JxT|jdL=$m3 zl~yWc9e=xw?m?MdRZ={*limNJQ8t?iB+IsL9A0G`=sYWmI+%qU9)ii7)cp_XbGJC# zHOHoO6D810U3nWTNkVIE~9O!{DS*IU$BgnACN%!(!|x50hj#tsr7(Y7^H$ z#=b*LjG8|3B}vKbv;nXSbCZd372@V?YDWXkXTc8{?50c)5}@J7h|=aDkrgTaj(*Z| z(kx{Z%%O!oLSlbIL-H>H?%yBda8|>TD@U&HTVc$l12nY^+%@MD_nCwNZ2J zT}yT}XZJ_DVwOX3%b}>{kQ|%JjnFX`1%Ec?CzQr-GYPSPgt-m`kVnZ+N5TQry!JH?JRSy_;8c!x77?jpx-q)iJhfJebhqOpXdSW*DsFy_4%> zhLX6UBx)#;AD_x!m(b&!5dtT7P8;tyoKc4pXi?jv(?04G=SQ#qo9U zQOK)`=uV0vT~w1t@~R@b;iEW_vqXMrs@l^2(Ll^{IBq!{wH!u2l}B)POshY?=Vw`q zZ*)gBD`T3Kam`9NloMhj7e}t|h7ozy7UhlEo8$K8v7L8~*71QUb<9{2Hmu=V*JSKF_&|EyxwAG+Q>^nok3=3soy!IB;P07Ui# z)wA)HRB(z2X`x3l>CVlAjt4CWP!13+@Kn&KvutsV<1R+@nOnwpR!5gNf8?M%=`Ip2 z^}AO0OP%>?&r|*9`^Wl!p)p2HU6bbkQt!c|G0m~K=2%p7?5;WYg&p$pZ)8V;w0Hh! z@#d|K>h~HoTQ}?9=Xk)sT%Nzjp#F(Lv&W+UiJb@hY4MT{qx!FmnhvY}ud;Z+aCRL8 z0Cj9Nb}6fIFA~c<#7Ppv@F^`W^REG6{1k=B>Q^hOM!68}6sbq#XZPOBePktsLC17T zaBY|7rgk9-FY4$YojAuWAfr4E`=UF=b<}R!!^EBqnU#5d|6$16Y(y z{D?mba?Clr9!$^GbO)GUlQ|{MipoQl6kjNXu_t<8}?%f&=3Xg#5gUY}s z1p3%UmOA<(LGjSfC@;{Gk@SU12X_$_HWcvr6qEH90j7vJ=K`v=GH|MrE&{Gq0GkpdLv;ZU{MoO4jVOAq|#`u?40d|h-<<{ z>q$LURouL^?#jBUhHq|sY2%BVuWi1Od&6x_|!{<{Yd_yp}Dk3P>0%*o<(|kQK4+6+&q;ULDiq-8JV-uDQ@O(G=Ab z1$Ux*##>Y^`RX6#Yg&r+KPu%3E_H0f+OocW>uU9TRTjA4Tdmo;UjN>DHMuwPN#`fAEkUuAeS5&+;-h**#szhvn$C;s_D%kPo3_IqA>fRwP*ekz5NU zScLK}1rSj;hirQXpGh4!#y)R91c&bv3MDXCbcnkc8Gp$RgX7@(9fVgRQ~7(-?=f9| zIE$@+idhn^f5BcPn|VlvrpDi=B)OAozuKIGb&fn!kEjkANGB5EybG>Ycu))D)P%RE zE)1fKFds?VNxYg{=!c%duN-oL0tF}Q=p13{k~SadFj&#RmH9nBXRV?gAd?(N9?7QDwo3L{?Es4`{S1MtX^Igcj47t$vI;VUm+8jV7}H*XXuxpU zasBW&kG^#DM&YfMvC_5i(zVgtbs)A3ptho#_26JauPifB53*Jpy8YTK%XnJ|-|1(X zmhQZWX5ppCq~ykhDB5e<`FtH#U|ok|9MOVuDkfviGUekd$U_lEwvp>~ z0<0OR_Au%uiK;}2ByDodYmKYPQsu$x*mBqslop1&V2|yV{vMbcsfFH?Uwmz{`)=Oy zkMpYUL-;w=tf#F;TEMk-sF7w`|6&bHIcE;1f<~(f^hJKDBzr;3f?&+;0dS zvkP1|E{B5>Ug|`#oJZw>*PFZ82tP!{93w#OmNdW&c!+TqKSC*_Nh&{5A>Q2OVoCwr zvEYGVG(?JPiNrwC4+t|xV1xYG>q6u-dA1PPL!M!B-6lXiF8z@F*y2qKIu04(z!O{L zaZ##ru2Z=X=}|&HPT(j3Fh`KS6SzPiLf``eZAdR^BFjDH2$dB13wS53WRjUV-N!!H z&;MNI!i7l8)GujPkl7Ke6zaS%#AIPi1D8~}o}fku!90a7YLJGhMt7+!q*ah2*U8Cb z)r>o2Lh)P*VWR6-D3pyBHeW6vAZeyEG>^KNI!+m4&l5hG3?{R-p;df1R3PqGF1DCe z?G_j&zQ%N5JivFIb)Vp*;6tkyAWA}eRscYiGFKs@Wf znDt(1_0;hjn`5Po@zTZ^S3G7Ne=_k9KNuj`ixXL!6Lzp>Wm%;FWu-Iv#ai9W?MvS- z=M5IVa)twt|LDfrNJ4KMUwZ!RK zUI_z)M)<6`U`!M(PX+y-M7}?xv5KKOShjRCD@tuDbY3g3wP|K-8mnf;q6f&)g-a5PtLGq;B#MhFa4ecI2vihNZ2?fxO|s!T zizBDiLb0SmZ7LSyX9wTL+kruF%skVnQfDD%1{gF{kPI-XX=Tp<1DD&*M|1Yzy=Rc3 z>`i>J1C`Fv*=Nf0@{l0_ChN>v`OJ4tMpp4i7T^;#r>cna!i-p7V&FlH~>nkEaS5&gIFHp1(t=tZ<*l$K2gg}Zyjsd z%rBqe04UzdINva%BA;eeCSqBPFOT!(Gb%e@jH8%*F<~O9ln?%=inrG9Dvj~Tv}{JD s<{Qycd?O(tTFS?gNll3HsA4}-R*s^{SX{H zb38kVlg*gi+0)4$cj6`)6B6dcEJ@5~Hfv&Hew$5`Fe~$BV$5gGBr~(K?Itthn%Pg) zZ6JcqzVj&R)vc;~Z&lr@y8YUS^3sHy`B+AVO@im4g1o+m|N6en0(q0=zOvh>EU84z zI!PqyN%%`WUME#lr!RZzo0`2jz-Q~5GeuEzsAVMWqzqE za`Er~qF9+iG7*{LIg!uFidJ^7iwYPH*9i@gf{ws1{P|%*nhrT%XN0i*4?; zY(t>UK8tN0bsUkr#jwfZld!fYr`9Lk2HW$__qu5Y%~ah{LbIszxQS-d96Co}xzt7T z1eQ+=XraK0=nZtPz~<3nI$vOJx_~YeSV;+8L>CLRlrEuVi1k{%3)BUhrQ0`q7peg>Fv;rJ;GnC0JgeFbdNmtQoT0>VKH`6tAEv=>N=z8(Bfz}Pt zjr3cz9zT0c-X*kwZlaBJ^Kp~63{*6M3J+}th2`|y0GIBT=$1!_cNvUrhOr9TLT{v7 z1%D;jzLjpHZM2~LE^2VJ7_2EqB}v~a*(;{VFK{41iV?mt+$AOUIA|fXri|r zSG+6eK6<;llHLJ-K6Pb}h3-0H_O7CS@$KlVo-$RBLc5PB-Wt_;Vl{fBB0y;`8Bu3i zo%S4&W(JIR&6LEF-kH{=yRYG0JIx!M#oKocZ|xLs#U2{E&Pw{Pkytk^5x%ZOE?vfF)SbJgxhzgp$jVl1eum3~?Y zXr)SHReenNhp8H>+8hbQ`qi+mRke3?ZBoO5-Bryj64uo)t@1~smC=K|ta`QC0}3@B zs9urzC5FKjvfcn-8{m&scMKUyB&vpuoX%U?y0$cRwKV!Vx3~H_8oIU^3RC^mkRzI* zME&}1L+OhI!-k_{NBfOUja|MS?d@Gg8dIYoe?Wy@L_tW5_3YE%zK!iEV*h9wl~?R5e}+I~L^2gAJ{he3L{RM=x-Z;Q`#@TvY znN>R&)_1FVFu;F*$J5!vvLLrHtO*_yo5O$NE6l<)e#N?>R}HHNqHH67&o^JboA9Ks zc3nR5$)H<3pc`r1cQ);4Y3lS4whibS7MfiQr45^}SJi!cV=6nywgcfV!n=1Zw+Qic z{=lwre$;sp8s6G}8sq9`e41Y~{B0TIu<8L$p zn6xEgUd|(B{Ef(c=GZ~l5I=nXT7G74@%`VikKVoa>$%Qbk;#vM0`S=Q&mQ$OdmzC2 zL#z(T4^M-?_=yAH#UCK&F$DJ^!N&jL$6gu!Wu06+^azrFj6h(I0@e$Ek~H{weG2*E z)Ab%(3cdKFN!qa&Cz0ssN#wxGlgPevlgQtlsb`qUQ=&J&Q!l1TeJy{wepoTm_QtBg zXsTw|MZEg-gHiafMc}CRgCUt_SVQ21Z~%Ci4|EwycZ41st$WNYlLFp2ykw{dsNf%m zNF6(x8n$g}Z89>OZtiMo>uhOn6X`?*T*{MXp@xls6kCA`&8fIu1-OPDLx)a!G{@H+ z3x=pqQ#B1zhfmuZV*dy%k3cwSXwJh@l3y6USDYt9_fVv$efT zENkp)dEWnAnfwcx$DdmXqGv+6{K9i@)?yuG*!q4`^^k%yGj#tf0|I5BBna)s0lqhp z>!_SAixB+(dwNY9T8Uwj2MdT2tHdBhiwQikFcW(xH7YiUGZ;2x@oAxmuBB$HpLADh zx4)G6qtEZ2kD3Q;V(QA>dVk1cV~+z#ln3^N#FxIXzsC`XX?mnTrORVym^=){&wAmk zLj`6XC?I^5wymjaM@wU;;hJ`uZ*x;~OIu49WT;NDNH&h_SW%2L^g7Tz8O)Hl^F`OD zS8(iA1ZNRkK=3XABikR)gZoqr(aD6#eu;x0BDjd)41%9aeC$uILeU_KQ9_)lK^fWQ0FsRGx6WWK{;xg-Jj3PDnGm@O=y%deJ`%cB*qt|eyopnO8cK9dif z%`w}izn(nnB!A#PJ^S~hfqP$DkyWN+vYLEoHqe%r2g%Hqwo}qEe@{MrnN!$_ES$ic zGpO`Ax%2fxetaZjIg!F^N+f;Z6q6(^l34j6^Z%!U-}!o_X%XxYG9G{X_0Nj2TbfK7 zfEkWy%cT+koKny16P}9C-vIlp_~hSyqg1{u^T->uLl`k_hFN7S!UnQ;hWCUc`@`;O z?Oz-NGqYx#Toh}pWdrF=H+Qr)w6wWJC4%E93X%d%ZFnHJ75wT()UD|h)UIR7he~5iO*sdZHK^Hm2$^77j$HYFJ=pkyxtnX64dtz`YbjmH`lh zux#eR4>Rl&0y;gFv;ReqQXMK<1A2TYl58!YVu?)HIvqZ-7Q<5ynJoLQT>E7pUBaNf zO!)G*?yqu~ZzRWD@xuEHA6Pe*T{@m!3Xu!6$7`OdeWLc%&atKI$Cs|h7`~AtWjnw0 z*50Pal@a^-vfB5{YRAgfkC&}a%-=AUUpJm#mvGj7=*YU~#`BKi_Z`KF`8B8XXYM>x zeWvkD_4t}CV~&<_M@vGymp|mszWx0hGMg3Y^}Gsz=M-D>TGP2qck?RKxmD)owWGJb zQ?C@wjJ2!0|2zKdd+W)+PbTD=xtLN}0>O+ViiA9VYoc5(H1Y2xs#b`EhoK`Ph5a6| z;O9_um=0K5a2duJrdP;^XhP`cf1Rk>nnLmWYXrYT23dFluTWYnq-!46bwgo0@{1_I zJPJ|3{bTt<*pwJ%7Kw!^`%lCjJ7QtTKWY;OKl=(<%MhUJAua%ju)v886WlmG)>It) z5r_Va03$*i3`5qJvp?aNSSyCa%x<5#<^c^7Uo_WRi$W`f;knMQNppYI@r zy!t|q{JF%BUuY@G1_8M1nlx31ePbywNdK%#XZy$_H=20ON2Ly7I3Cl*eHgX0MRW{-gw^t@&hYO$_*AK$AW> zd{QKbE{|pMwufqNM<5c1Xs<1)qQR7S5gs>rZ7NyvH*<{S4bfmWJR6f=?0H!Od{X z593|lvdTwx?#Hs$5J6FqPKh;#qUM%9R#8@TVhd)w>%NA zpCS2e0Q~uiN|$p}Q(NPfZ4EoN`dS+{HMMqnGT}hgeO54Z6j+Q5m^(;){ZQ$H0ge3) z(oEg(qlMDLR5f+WK;uG92%%*3!nGZ%bbiSJcNkt+|-;mnjv%qXRSPxuReoI7O0g(;eUpUT`Qv;YAbUYu~l z0{7ycVDB*eKp_@U1&Xdd5r#7s1;5a5S`}6eEQfGU{{&hNOTV(`+*5bnKKFh5+{Cz4Gwp342yjlFjA3@^W>&+e|j(uX`XpVOy7apKlpcn#Yyqgwo7k{&Sw0VceZw zC6liSDU4Ssr1+MZdmWh503A~#?Ea>|cQ0WgapKll1%j2c*Pb!CcI>tyVYUXwkDB7? zX7U=zkH^x;Vt&YBB^ypZ3MknHHy0CJu9V}->}JPfZcFu%!)P~gJRSqglg8XQ@7+k9 zj~E7FDkc};C#GOv!lT57rFgHE)Kz24ui@F4jGa{E+rlQ}GEa zIkspbf8j)4$%Jch(rV6y4iCU3$?Qn`3ITkY(^!7|Asfjvdq|t-Tk)UTNc}c9%w)(+ zWv3BI*VX>0@biC*PoEEe@$&gnv0&3w2w>f}M?EM4%t-I|?@?)xX>0{%x8y1A$N9;|neC&Zz_ zHbj&gaV$j~i8$Zkm!A!H)HJ%hO8 z9T~(G-Q8eKwfH#?@|}A=o=f@~DhSL3;F3v_7mO2kQbx)y0_oy=h2Jeq*yoOsdE;bWQZ6DjNeKXQ7Ee>w z#V2yffqW~P6#x#d949N1GExfS_qoVgw|k6$>IF$ThrrPQfLsfwDdq8;JW{EkqyG;< C1W7^w diff --git a/Frontend/__pycache__/app.cpython-312.pyc b/Frontend/__pycache__/app.cpython-312.pyc index 0ec662420142d170b40aace17bd736cc14858df7..8b45f4edaf5fc1db984014b55873ca8761f3d241 100644 GIT binary patch delta 18006 zcmb_@3v?9ObzoKhKWeqqYM~E86bO(SP>atne54TwB!oZ+ki<`Xnxocc#tq-feQ!529G9oVkREP*d%-AcyqGMY$n6T$0x0ka?q5OWbB$89AaM(C_{^FH+J@?6!(bvdrXMGT_Lm#4mQ95Hsdb@FDuV4mnErgzQX*Qq5u zC%Q#f&r12G#hs{uJki$U)f#1fdsb)CTI&C~=31?%)UOeK#bP67ge+AuxO9?+g{xbVq_|4zHCbUWcUwtEsYqCgz!hKopc$713n zdicvBxz`LGWw7(ko*nY(wZ)c>rSM|!X_haB-HG&BC|5JJ$z^^?Ub;>Nad&vSIg;{%Sbs(RS*wZ&QdGLN>a&EUQ$I?vD9j^hOA|&7sxvDBBr{m z9c#oj_1v)yFyvZsZBH<>5G_U4lMNcG5^%PkU1=lHYn7f`XHyrKsdHhIfn!4p>d>UR z2u39HPgQW7#9zn)jI0w^Yq%8GlJ#(v5*(?4qNeCyJ8Y#@8cUb2t0UekAMgdW3{>tM=~a)E-U`!H`H^ znboEx)8^1@6q7Hfhj?{X^*v%Pc@0l7&4Y$-&u*cnfmeJ*Ysqti`a!+tocyoG82^qu zXnHddbM=M#yMkeIjF;-9&?d`*olVUi~?K{~Y3=wZA(xn)J zaBdXt9fM+$qW);Vq*$@7FYNCX6|)qIM14V`n0rM@@^^``)vXPUO>ONBt-d{b+kJcM z+S-mv%a2wZl@iMPus0a%?HFN z`d)vZqL&h4R5AJ~^$#k>K(9aAtr$=Di~XWvjQ01z3QZyCG%6{kUa>bq2XVDD817O` zXT(4>LKSmBiV`Ars)=AYs+c>WisQ7!7dp_;I(@qUX3PTo*SapFXj6OVp%ozP8=; zCsC`G7fp#39M{8y5iMW+#kO*opO*#1`5PQJ#K-v_ha9_5Qb3Jyp6J*C<1Z9PovC&( zqOd9V$xiSsGl@8_kpXQw>$o9Z)WzD!3#|o)xUQ!Fd0v{$4IxW}Kdm9pCG>;Jr)e`p6uACt>OS2JW+U>0#w1F3kN$zDe4XM z^$Ai$Xby%??dgl|g-gCyB^VVRfr_1QLc;-3vI+2rXGA(E1o|oDMMHy?eF-W`kW39w zE(w01A`vP`gS{ssA+ONh9h5Si2=Fq{?T-qlM6oZ`qd(A{YAr-g_Q0JJf?*-rEec3= zf&n1}(-JBqQ4~`5U!;tPdAs}iJl;gifb)+zo>)%H-{W1NxHp0Ht$-H;{71R%Ay89p z7a;Kp7w3+_Z5iST7w2K``Rn@M;ji$3-Yf$*sQZB)@TiY16wgLQ8Wt9BTP&Q31PM-s z5kr51E!P(_RCuawE$?ve=oFoImBI%l6y62lv2#0(grj1ZRQdb*ynTa@tuuUrBm)L z6M6f-c!+EJa*m7t|FYnXR@@vcC^>xBT=x*u@5JkxY*Thdm4d*AJCxBI6oyYDCT zkog6D5i0d}0H~@G>h`zqscdX$X=tr$Z>See4zknKnc4;chG*jUwu<6u(Ak^iom2p&>O7N!DN79 z06ZBXgOV9G1c(McZ1)xSvkpAn1J!T?rFOWh!`wX!mz^`bea37(H}J~9xuI8vE_Y0s zg&9}jHP>6kZx(-d=VVdUq-)g!&SYJl7}+|Vmw%=DQuWm{Q+dm7EWYXdc=_h(oWhyh z{L3e=bY1GY61o)n?xq`ba^9NB+_m>O-m&`T=^6LJyY6KZ?q%brzjx;CGv7Ho>0Wh_ zf24=PUpjN9bLL%fTyk9Xzt#0-*SP;%J%5$sx#xh0A}}j0S7PeH-`EHa=-z zeY5^!`-bF>&ii(wwVS_}#oHI(cbKh<@98a68%f4`OtZTGuEHoOrwrM;unZ5qK zyh-azun6P4+qu&g>)3YlsQIGdf&->*oX7GXmBS*~G9N!$re!{s3IV|%IF~f6G=7-t zYS^IvaHX+fo%O>F#_ha0z{_tRac>9#`p+4!a8anLK0P3Q%zfgV#-&h(koZAfPDTsl z)0j~#@C64$W)lVjD8V4InGbj~nE-*JHt`_!n$bMo-$(W8we29EB7Sx_a>ct zla{^HIqoaYOU}uhMGrWu@i;$Xm~j-`bu5^0EVxQ09c3fBPn@~aS?=lFyeoy53a>7b z=Z~1C?YUF=6%+OfD44NlPiAkQs%)47Xy1-ioq5TE%FQ=AZyZXlJDSWtmUJHb*mCS% z9`=9F&1L0ZBqN)XhT?~?6fQ8E=Aa^j?+;%$$ji;ex*h{ih`g%Q(FARRkL%CyAU~mu z!JeM9j-=&)-jgf4<-c5O%N{b!t|(tHmoNN{4M9Wt$0PHqH6dQR5}+{|;xIPaiqhHu zmmx%;p`^N9C%@mnC_#+G)TN8g(`FPmrj#Ve=t06f=ON~}{$0!WQ!5!V5i9;_eS*$p z>OpC6&O>ad!=2@68PCmL<}-{u&(c1krCNX5GKb7@Gp&hwwEQ@T$UwkT`vZe4)=JiK z@U}#I>nYb#>Mt|0knA`|oQ&S*#5thAxz~;Fx-y;3UE`qji8VagBfRxEp335)%e6cv z%jaEJxebm#^zU`|EPc$XVS*M#E48klWqOZW<5uEE^)RzZQIvhzvqZCy{U(&jt9soD zlAk$5fTDTV_3x&w%3!mda|{Npb5*~K`~|nF{FQN0AvHX=gri%AEWgEW$s2p^36d4J ze2crT`%NQ|ouGSmhXz4#29ca-N@O6?+ZPOp5~z_Yv0B)Mq|^^~qBEdov4&|y zQb047fIapHK*g23LTjQwj7=(ofdK~NO z#qgPy1Vz%J7@5!!%U72l@`ItPTj=z|!pJ7!9UVm>hay2ybdgh!`SMsvxsX~pa5$K< zwqQs}!eibGT$;53p0751Y*p20)Sj(gDx&Hwl32o}4yle+4ML45#PYp@Y7!87BWD2B ze&noPPmy9E!9bLv14c2!?1P|W(M|Yf0}U6Z(J#R|s&U9V1Y5Akx^;h3bG@&5Pb09` z{*%C)5%r3m_J@@$V3T`xHf(G6weH!|u9$$Y?%Ch2*g)+BsVo52J;fxR1>P+w*$s!< z8(P|$_OyVtv3-YPZ8#ecnaxeHX(MdgrP$SUTRSL&imAVkz#&lbd>VQf4t#hFC{lC9 zl#sg7mbc$=Dz1SZwD*BUt zA|WHgbErNlU(fNul2wbV&o2#z1N1z4Bu+t9iGx!Bs*ow)==|Ko^vaDFtyjGhIi49u?q$!_wn@jLdz{6ZJ>E8xx8O?crP{=FZo$=}WKP+P zTbM2=o+(I8pkg||XxhEtbBEcTJyQQj4^8g5xNO&$Z`3!=PdFBR?5O|l$?tW& z-Sxfj+u=VtKDn%ZM0Y=nv)E=V^RG6JAHHdrvaAD90+b}{VSXm7;BHpwL{{lIx!E}F zS$^Y%8_SZbn(rLElW0orKaeax2s)5s;pd>O9R{@wn}GfV@N*jnOHAuTH#|4y-)u^j z*G^_{9?_$G1A5q4?P%@QvMGD%OqOeG*XXXR`m1f@)e{T6Q(09rmXaw;X=)zFXU(H) z+Ee`}P487FlG~3Y%a5jqKhA~+IR%D?^g3AheMH!=K|jmNf57RDPSzW5ZJ*s+a%0P# z({}>N^{vUQ(y6RA(9fX4BM#wx1|NTB$^P{{GqivFC_A&j#N$UU97!GTv2+4X!)VxV4VZw*eI$- zpyeSyAUaUr)D$Rg#B|LJI$BB)3n*%sQH_)dk_O4&QLNqS=)) zjFvcx zz)a9PQF#(r0J9yU8$$#mDRQ5IAXs{YXk?oJj+3&Qw31otjpfd2h54HV#e#jHy71V5 z)!WhdBS~P9(FF;8Rc>8X;V=b2IgIowhEu^Xk^iuA-ywA5sRp@sG+iyiXxcLswWmqr zGxW4^=<=Z~zAhvl;y%6VDC94h3;C8<4H&DoJ!`QzPoYA!zKH~MUzJaj%3#=IJGi5v z1xRD;gqNwr6@a#|2Qr{D@Tc?#fcpDGLRY_^65F;Nd-k+7AlDBIy}?8nkqHhP#@TTY zFhVn1CHP(#+JR?*Q^;1gcW?7y7-C_H25Ub!gcyxN51ZHyJmzhRH3YjC4g}Fr2F@Z1 z36EVdfY~;x*Z}WMyPG;d)uSL;uH`72DV6+mwyz!(y_TkS;7o0byKY;1(}4zbim8o! zP4#VZ=Q-~ZeFzM2Ia?cAw(Zzm*SgEsT(`BMxy@tDFdO>n*GJsDB{*>037oB}P7lK? zMeq&yFMS38w7+{M&N&ZMx9OtgH#Xi_kt}#|+Fd$bwBTy+>XBrDXW9jY^X@w>@}Hhx zx8|OKx4NcX^DoW^BD28tDW;Mo)$alIyL}QUl65ElIar{Iua-|37Jr__8H;58t9vXT z8y3)$Fq~X_Vcx9@^Buj5hT-)wU(F2d^jlc=YummxDw}t;9d2pg(a_$s&DXZCnek{^ z4^^17z=Q-Aka07lChVAh<=uWKr*Y?f3s1K_d z10Yp}6=NToBIpQYNvIVduF{XnYTdx+sU-S<%YsM!t8^t?9 zUj=aMnCl;kO+UfcF?o084_DFO!SpHsX-mgx#e&CT5ImqyUm!?>QT)z}vYS%A& zk)qR?-bB!b;7164jNtbW{15>iX^)kOrSuVI{7Qaj{lb?R)^+`T^gltKgr3K_cvjJz z7O+9W35oPypkh#W%ebM2H?R}5dF{MgM>f{;3D!4o12{zj{b%^44-tHh;GY1*jIH~d z8_FK@@NXsk736qyidAG5FSw%Y1fh$V4NDY5AQC2YEmTwZ0J9R|{8SWYgD4vGhk)>^ zlQIGNb(eHmwg>y$(@hs1qfKGgqQ!RWB1Nd$moI=9CzraTUfFBw(3V)-Z9YYkgwR) z6BixqLt9iD!xFs)rR)gM7D(N?a^y3f{{uOGtUQ4?60noS?q6#|-R`Z;4fJKKj{B=3 z&zNaS@z5(qX6>SX0%=Bi*bb{Gbftt<2p~W`jNPGSF!POoI;kHTFyp-?CuuI2$y+!g zS03*+A!_Ldx4w0}gl|SIhvLFK28z|`832l{e*d;z^;>;gn_4Im1GpZDJa#=&duL~o zvg&l9D`3P%y=*(Nnm;MmoG8mfW{;cq@EX8#j&sg)Idb2L{f?ZBZ+s-@*2gE@eBpWL zs54_&2H`+QEaxFd{>95Bi$|R#9c>1dqO@au#0j#E9qh>2Alod+H%4<|F!scaU~X}P z7+^$`bRtU=42T0wG1?m#aI#C%#@a_Y%E7l(y~@!$Ub+7ZX6=-i6PSRiZ+Bg^w)J4@?oGX zFchR~z%(7_9I1&e;<)pUQTU5zo#AF7#{tN3B62ucC%WW?0b3&HnjYp|2Hj;3IidwA zWP!;i;~9^p&0b(TLA#PIZjT#AopDn<8_Yr}+kYEOpNIcLmVu{Jjseb-*TP6C)0r7* zM8mh-*tcu_T%t{rFbyRageRuwswNv3gi9E1gSLF@~k5Z({*+E&3J!AY>b0HFOgIMGr%=b)7f?*Hhl!wL}-x%b~6Y zxQ<);isYYm)m5drlqzTS2TPp;l|=gh?F{gXupl7@Nm!=4Wt=2uSVFZ#U5k!8BaIgF++9u#p!ZTuGx zY@BO1{OAmBlT->*`m-fD+cz8jY@Gq%hxs{;2E&KT%m_CuYqZ{ae5%4wI)Tg4Az+)D zm0mwypBX%t&$&8nxNcYwtb3TP6wc^>>|3V0te5|5-;#Pfqb(HeB(dE6;ZxzrKsXZ| z^7j{^G5slyifn|jx0t2j(B9^{rWS$mxOaH^6M6UPU8YM|;e!0_)AJJtp_O9k1m_Ui z{XBW>(9lNgYX^e#nPI1dB6DDp8bt=kcoH4Nylwg-9P3?bb@3Z}V!4wrkRfryix^L)jy+O|!i=RaN zEM%OL7oM$Ny&t2-vPRr@3AGg5{vQf!L`M~L9hJk*{T9HiLH)fCZZoC z+X;E%tUECWPm@@Sp$M;(N=y(>;Q0zd^k7z>NIQK2Fs*_J$8e_@nW2@TP|0eA0;t!3 zepeN5OkP3c(F|5`0ubL9r5@XH6_rPDUOohj&>&{n8Blc5Rg__s%^XqtOoLo5qLkU> zFcz}cPfyWK=;QmhJ{bJeX2)Bm3d0ZOAH25IOd~j-f0FOMR>}WLHh$$reoU_U%C4#h zkll}m_m{)J&jj~&kLkH$JC*SoEJ%}QdIcvp=jd;Ey`so$fWh9x5%Kh^LLU;tnEZ{` z%RI=WV7D06NYf@yX^Qf7Mrf~kif_ulcs(!iFW5s9E(e7U#n>MP>Ingl{!qUNqzOx3 zVg(Q+;|GoR02Zjb(TuIwrtZR&3Bm7S1I8;7m|}Z1j;Xg1{4Rp;AownV-$xUt$HmSU z*mXfYkbLM4RZQSeM&hi;E_Xpy#Q_rvLU3e?<{F_u;r(5~^JC%}v~8R|?2q2Ir}wR_}W^$tA5xEl}MW)LjGCpO4m zyU@E9B~JD*#}|lj>O)M3Phw`Vl%9-?L@dNA@B7k1`I9$t*93ILhJTtIham`7Zf;&w z>yzto$z$x9hA;|5|#%{*^;Tt`>Nb)(s*)&pM4GnGgVbL-`fW`oblb={b0(g^_~ zMv{jG*L5)I5^Y?NER6wOS~@kxjl_+(%QzuZ(rGF`C7qgvjL}6}A0ycydFF%US%M}< z5In6d%wwsKn};mn&6%!XFl$PPh3y;yUtQdE1|nB3SmIWYQ|%z9+91~^&008_msv$3 zZX*RaJCKdbw9$K(XsOI^beWbWh3^(=_@3?c>#5LcWPW>AWYStHx>8F4?kYJ_oN{GC zE(rNmT4CHy=4-=1TxnK3t7kRF%H)Dk^~pm9(Y!AtO0>-$a>O09HSTzpvz%^+{I%K$ zncs~6@V8z>8XeJf*bnkB4hKaaK79GX9&P&C)Bsy>yi#SUU&wpTjeFL&Kg2wm4GPr) zb{tE=g}U(jKs)i7jg@*eBU;T$0QA&osmyP*R!he%&?(=khm939r*%&q#x@ti?2t;Z zhb&^FF21fOW!k+-%=MLC)LribM-H^#JfV;$h0stqAe&-9dX7rvzCdzQpL-LFP9 z`UMbGJ1fBq67_?Tn4keP@wDMtwZ*c)-xW-9!a#L_MbxgeA$4Sw7Yr;yFuGD8G|~qN zw5q0~6lZ-)P_+_57FDMqdS%mYT8L``KV&4lUGTF$yMRQ81+}7PW5#a5Kgp&IdYS)Z zXjYi7P=O|HHCD>wmH+XLB?&cFIi6iI7!8ndzFMz2q_F`RXAPCFyhtPRD zh)!Mb%fT$r&wJ1hBLiQ=3BamQg^;?vOj2v1LG0EI8yYkAM>{Jw&_9GU1U;D&Ag)BH z6m!=@tRwh)!Rd@vZS)C3T-JheVbM!kxK}3HODooudt>HCKmr-8O6#iXu>puj6mQ+b zLjY#QXaTunvHI+~Ab*|pkhF<`Ko}R&SmZn`Mh+tSkz%ESw?Gvi)U#M-gUTe|fHF1* zh~;9E!4n!9E`5#+0y(j={!VtvGMw@mj)9G^5DBSQO1rhRUEFtr|Do|6N}N_Df#)VkX=f@hnn>E+bA!Vz>ph z0@8R$D6W%YTn`0tf}=wSzJwrx6Jlxu5+;}xgsXwug8H#ABX2P&Ya`aMU|<2n^h$~= z74fS@nn7s7Eb}^2HM#{sBQ`TgesF7}D~rYt2h&Vp;^ar0zh#iY}dw0J%-J1_1S zw|?yQK%HdHlJP_1JCojBcbe{0uSgy^l*~SyJQA3)kf++&ozuC6SBtLNlg=e02Km>c zWu-HPB|o>cU0Xh7X-n=u{7|neG9?Z9_f4FoXxddKd(JOTG~el-$~ySO;*(jG;1s%e z`cnCb83GK9=5xDV*)^53_(uC*8P;e~1b~NYJJFwX^Ki0a^Q3dj@QxX)W2F14e$raP zf{n*{3__kAgZ!DpJ$7vL*j-2IgroH8(4?bsII;Z`v*Y5jD^*{wnzm$**+=b{w_a(y z)Oe-&QuEZJwSVNhec&VC`@YF_`zPH8rYr|%424sMVwjc94bcL|z5I+8+c>`bJ?owI z(@SgL8+`B79e1*<1^o2Z`4A>x-HU#DsEdoqzH4w#7~C_kfde-{@i5-_)!gS z-M!NAk=+3BznEORSL=UTR=T@Z|FdG_?j`1*tv2qiHvg>F*kU%va^WkHU=<@PK9#JX zx2*;F-0Zk)rgorO!k#EnH}+^k)DV}+BRn#K)CXP#n2)~j6YznN;a#I9eSu_T!oOMX zh9TZMOR|X~&0vo<2!6;!49{}L!{<#-6pa$ZOiaaTCIuha%o!xk!w0w)L|ex3o{2AS zdrB6%aNs$T&{8cwFnF0eo0sXk_QM%AtBPq=y}Q~ zO$Bg0GU)&X8kPJA`#~W&Dcpfep*?c3S(oP1wujmiVDKS@Y@vlKvxQv<7*|D8_@~s70Q|D zQ7(Y#M8fzdEeH-Htb_o#!coWpX~wIDd13GoO`v9~I%h+WY6(vN+rRyrVu8i^aMcjl z#2C17ynJA&on<)r0Jq%ni6!eEd|u&r zETI~6ZtYg5{(Yl!YdQD6yKL)x{rl@Jkp94F+*)Yp$Pfu034yAGk;CmCwdkyjj zm%h!PyygG8m*09FI~B0)HB2ppl*g>vbHG27GVi>NMa=jEHXhad2bEGyu$fsYynpY8Q!Y<4pOR?BK zv;QP~;{~5cK?m3+bTEMR>kuplg2kyW)F0qGk}mo>cET?AhG z80E(Cbr%-=3#OQP?rltc2Lam`X7}sBD&NICf~jQyl$>oa*^mf9pnKJj%#39gDBV7+ z2R7y~M>{czHZmrNv0Kf~48@IPK3(vaW4zL2pc;%dT{LM=+UbylNB|S zj+cgQ)A;lOMoy0PjP@jpc28!vV1861pkY*Rpm2w5|>1^O}X$;Mtg zavCRr6hg@KG_;}Z3AD5nx@oFyLwnAiCJjxSW;Z3J-83(4KHKbe+wP_`bi0Z7obKK` z&k}~vXX)zBy?5@Nxxbm`6JN0W<=aZpD}{wl37#*N&p3GY;ZsHB%FnE)7VHZs5|wH3 zMHzXL6!Z|4dO{`UNRo&(SPBx(gY`Me&0ktdY?tg86_X-Mw3K?d=Uda9#IYV|FlixQ z_E;G&Iaj9Md5}Fbe4XGt{fX%UxdNQ9ZKw@7(u$opkOZ(S=$$50%!L9t%$+8k3S(sZO z_$xtHT97t6|gB%cxZtnY4UM5Y_pbnb5W%8hz~zlU_DtzmXj6aF0zuWI%6fP$r^GuS(_p2#Be>?KsJ(l z$R-^1S%UM)X3|8qkmfU%-~v$L2Ngcj0ty$BtpFGGOJv(QIS9)q&A?kiTFJd+yO3`H z&$p2sBtY6p2ibYX8kK{Mq?2@!Zn6vXErmIsI4c9(eKv$Yd&FR`fI$I23D83Jol%3! z$NX^^!W-6xtITT@jV$sjYSjGuts7uoG`4ZFw-?drdlDy*gN%^Xm0FJ#%W_%OS7m> z8^WXk&<0AlvN{bVN(?m-j??V?6~CWlTLdh&iq8x@v+{z%R>3H6 zLGoD)rD3TdA2u9^)07Q`qC~TZC6BQc@H!xq{ei&p?!jbRc!;uH%`KgDF#FB+ZSoAm z(G%9FaJ#(JXBAswtNGE|1=-4-M-|sz(7{3oda}=Sy=9SO{IT5&mEXzyd%Npk@Xqdf zzv0wUv8awpG*$(&K+rLEgHgSo&Be)Hq*`J$!RFww8i23BP<7Zb8$!x3U%zMWhDH=q z(kZqK5Ul_}KI*0mrkMdtLOR8x!<2||zQ4Cw&IsLj zdQHe)b_9M|8o>a9K>)+n&RuQ(nh6;`i`g(xAP@yK0hz%D=E$1{%NHeX8!yQep0U94Yd~w^s9T?& zvrCF}TMpN;!-;3aF(Qq~BLySQ5$lLLQh2HOqRZrT!5(Dzj@k!5j=2XVpC=%?DZ^GE zdRzp0Cp3!F7RMa-aTLBm)&=qM!r^IZyxA%0momW3RM|n}FfAWzAFsmb57C~J(B5!5_O~=#Q}}0(*4o-Y)Mw#O*UwnYc8F&0r3Z6W+xe<>&wi-a-7PsMYm=C}$hu@^5-ctBq zkF8pWQw=q(Q6dx!JEL$`A8F{<<1wF;eHAre6J{4Ae&3^m9qve4)06R#7E9`y&&{wo z7;2dHK`p~s$O0%U_G91S@9u2f(q&A~Z!Fa8Z)pv*c0&j1Vn0F2m-&)U*BmNlKSed3 ze1PMiDH_q(^C%*lWoA834nHTQZj^T)=*Ttnc&d z-}_B@kL~L;G-Pm4pnvdE5bLw!qO_P;cq=r>&9dW7*@(yl5sb$sNRf*#9@T}f7s#Q)6urMHQ*K9`-03C@fN;4vmd4K zL>n#*B)zFvIN=Qsgrl)=PmDHn`xN%?z%v}Z;aIFE9697G6X9h4g413{kdLMrE$pV^ zp2M)=i#}{P8P(G)5lIqi*fgCa)4I5Euo@`?GPNjJ2S6}k18g1+tgzfbm(OALfL8>` zyhh=s;2sO-16vNLSfc43yqt-1$eRmaQ8*QyYzDYF_D7KT6UK2E;`rfM_JPkNt+Q^8 z3A^dJ-*D&W*DZSwji3{S>)5{m7X1%6MwkXzAi5YdiNb)Id>5rw^MTJdW=s^ve?ssZ z0N_qEW!M?jp!zl6j5~U>9p!~1%>uw7dK0c*(Kt2SDav|7kz^XK#mw}umBEOx-=UDZ z6ByBWocb{p*dK6qH3IZ1xEz4+I4;PrXgY%$77q9?WPN}DONXJyMgACyzLdR>Be4;5 zx#k2g>1gU3EuU!nhYKd5YrzZ^?3v@Afalq zq4g$N+$@1A-IF^PR8QxGK+mj5AD#O|vONMpB z`3dBiL3z{`N)phu#b9!;oy%p&tIU6@t50=XX3**DuSn=MPyitH~g{MC~E<_lku z`5WJ^>FkZhs5jTx8`%G$F_A!ph^a7RqI|wLodP!zO1xS+f)~TybSyU1z^uT&P-Mo$ zikdVC3jgWl$_(bHp=#j)%F0nu1p@TEC<4Olj{xv;2kkW!IFw^W!g; zuVPhTQtXk^={e&(1uk z>?l%;5ySK`v#A7!1*XvCRKESC4NG4Fl7^WL<&KZ(qEhUxJ@}d)53@rdNXjs41PxDG z_~n<*RN}r2M<^5zM-!otPd2QC_N4oKa=>>FXFuAs1LI;SjI!?tNu9=1A{I9$C*nJN zC}hUVsZq!#J#>f;F-#1@5f2}tB+4{agNv@<)*pK__;O$d`7&b9Aeeyw|3AR~9)RJ* zH=Ztk8~o!#oL+_!It*_dRG++*;j2Yo zhJ=6r2QksOJzJ9TRE$EkcbIQAH`_LpvB-5inAk@MutXT9nuw*GlRMZJ+VfiQd|ZHHHtr0W)0mfx1TUzfUHle&K`Eqqfd5TBAarDFbtmoF(FIJ3X~ s$)}a#C9>^?MN+D-D}Y@;z#sVMk>Ucm9%;+3D}Y@u=ZjvwPrRQ0KOPhW%K!iX diff --git a/Frontend/app.py b/Frontend/app.py index 45604cb..72911e2 100644 --- a/Frontend/app.py +++ b/Frontend/app.py @@ -2,8 +2,10 @@ from werkzeug.utils import secure_filename import os import re +import shutil import subprocess import tempfile +import threading import time app = Flask(__name__) @@ -24,6 +26,154 @@ PROJECT_ROOT, "queries/synthetic_query/synthetic_query.sql" ) +# --- Uploadable optimizer-pass plugin flow ---------------------------------- + +PLUGIN_DIR = os.path.join(PROJECT_ROOT, "plugins/opt_passes") +PLUGIN_INCLUDE_DIR = os.path.join(PLUGIN_DIR, "include") +BUILD_LOG = "/tmp/optbench_build.log" + +# In-process build status, polled by the UI after an upload. +# status: "idle" | "building" | "ok" | "error" +BUILD_STATE = {"status": "idle", "pass_name": None, "slot_id": None, "message": ""} +BUILD_LOCK = threading.Lock() + +# Extract the registered name from REGISTER_OPT_PASS(, ...). +REGISTER_RE = re.compile(r"REGISTER_OPT_PASS\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)") +INCLUDE_RE = re.compile(r'^\s*#\s*include\s*[<"]([^">]+)[">]') + +# Uploaded C++ is compiled into a privileged extension; restrict #includes to a +# project/std allowlist as a lint guard (trusted-author-only — see plan §Challenge 4). +ALLOWED_INCLUDE_PREFIXES = ("duckdb", "optimization/", "ml_functions/", "cost_model/") +ALLOWED_STD_HEADERS = { + "algorithm", "cmath", "cstdint", "cstddef", "fstream", "functional", "iostream", + "memory", "queue", "sstream", "string", "unordered_map", "unordered_set", + "utility", "vector", "map", "set", "limits", "array", "tuple", "optional", +} + + +def _regenerate_manifest(): + """Rewrite plugins/opt_passes/_manifest.cpp so LinkOptPassPlugins() references + every currently-present plugin's anchor symbol. This reference is what keeps + each plugin object in the static link (see optimizer_pass.hpp).""" + idents = [] + if os.path.isdir(PLUGIN_DIR): + for fn in sorted(os.listdir(PLUGIN_DIR)): + if not fn.endswith(".cpp") or fn == "_manifest.cpp": + continue + with open(os.path.join(PLUGIN_DIR, fn)) as f: + m = REGISTER_RE.search(f.read()) + if m: + idents.append(m.group(1)) + + decls = "".join(f"extern \"C\" void optpass_anchor_{i}();\n" for i in idents) + refs = "".join( + f"\tkeep.push_back(reinterpret_cast(&optpass_anchor_{i}));\n" for i in idents + ) + body = ( + "// AUTO-GENERATED by the frontend upload flow — do not edit by hand.\n" + "#include \n\n" + f"{decls}\n" + "namespace duckdb {\n\n" + "void LinkOptPassPlugins() {\n" + + ("\tstatic std::vector keep;\n" + refs if idents else "\t// no plugins registered\n") + + "}\n\n" + "} // namespace duckdb\n" + ) + with open(os.path.join(PLUGIN_DIR, "_manifest.cpp"), "w") as f: + f.write(body) + + +def _lint_plugin_source(text): + """Return (pass_name, error). pass_name is None when error is set.""" + m = REGISTER_RE.search(text) + if not m: + return None, "Source has no REGISTER_OPT_PASS(, check, apply) line." + pass_name = m.group(1) + if not re.fullmatch(r"[A-Za-z0-9_]+", pass_name): + return None, f"Pass name '{pass_name}' must be alphanumeric/underscore." + for line in text.splitlines(): + im = INCLUDE_RE.match(line) + if not im: + continue + header = im.group(1) + if header in ALLOWED_STD_HEADERS: + continue + if any(header.startswith(p) for p in ALLOWED_INCLUDE_PREFIXES): + continue + return None, f"Disallowed #include \"{header}\" (outside the project/std allowlist)." + return pass_name, None + + +def _run_build_async(slot_id, pass_name, name): + """Rebuild the extension so the freshly placed plugin compiles in, then + activate the optimizer slot on success. Runs in a background thread.""" + try: + # `make release` reconfigures from scratch and needs Torch discoverable. + # Honor an existing LIBTORCH_DIR, else fall back to the conventional path. + build_env = dict(os.environ) + if "LIBTORCH_DIR" not in build_env: + default_torch = os.path.expanduser("~/libtorch") + if os.path.isdir(os.path.join(default_torch, "share/cmake/Torch")): + build_env["LIBTORCH_DIR"] = default_torch + with open(BUILD_LOG, "w") as log: + proc = subprocess.run( + ["make", "release"], + cwd=PROJECT_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + env=build_env, + ) + ok = proc.returncode == 0 and os.path.exists(EXTENSION_PATH) + except Exception as e: # noqa: BLE001 - surface any build launch failure + with BUILD_LOCK: + BUILD_STATE.update(status="error", message=f"build launch failed: {e}") + return + + if ok: + _activate_pass_slot(slot_id, pass_name, name) + with BUILD_LOCK: + BUILD_STATE.update(status="ok", slot_id=slot_id, + message=f"'{name}' compiled and activated as {slot_id}.") + else: + # Move the offending file aside so the next build is green again. + bad = os.path.join(PLUGIN_DIR, f"{pass_name}_pass.cpp") + if os.path.exists(bad): + shutil.move(bad, bad + ".rejected") + _regenerate_manifest() # drop the rejected plugin from the manifest + with BUILD_LOCK: + BUILD_STATE.update(status="error", + message=f"build failed; see {BUILD_LOG}. Plugin moved aside.") + + +def _activate_pass_slot(slot_id, pass_name, name): + """Create/replace a cost-based optimizer slot wired to OPT#.""" + entry = { + "id": slot_id, + "name": name, + "description": f"cost-based pass: {pass_name}", + "plan_key": "baseline", + "opt_setting": f"OPT#{pass_name}", + "custom": True, + "kind": "pass", # distinguishes uploaded passes from RULE# optimizers + } + existing = next((o for o in OPTIMIZERS if o["id"] == slot_id), None) + if existing: + existing.update(entry) + else: + OPTIMIZERS.append(entry) + OPTIMIZER_DEFINITIONS[slot_id] = ( + "// Cost-based optimizer pass (uploaded, compiled-in)\n" + f"WHEN {pass_name}.check(plan) // structural guard\n" + "CHOOSE plan minimizing the pass's internal cost model\n" + f"APPLY {pass_name} // setting: OPT#{pass_name}\n" + ) + ACTIVE_OPTIMIZER_IDS.add(slot_id) + if slot_id not in BENCHMARK_OPT_ORDER: + BENCHMARK_OPT_ORDER.append(slot_id) + BENCHMARK_RESULTS["latencies"].append([None] * len(BENCHMARK_RESULTS["queries"])) + BENCHMARK_LABELS[slot_id] = name + def _parse_synthetic_sql(): """Return (preamble_stmts, select_stmt) parsed from synthetic_query.sql.""" @@ -51,7 +201,7 @@ def _parse_synthetic_sql(): break if first_tok == "SELECT": select = stmt - elif "enable_extended_optimizer" in stmt.lower(): + elif first_tok == "SET" and "enable_extended_optimizer" in stmt.lower(): pass # skip — we override this per-optimizer at runtime else: preamble.append(stmt) @@ -64,6 +214,59 @@ def _parse_synthetic_sql(): SQL_PREAMBLE, SQL_SELECT = _parse_synthetic_sql() +_PROFILING_KEYWORDS = ("enable_profiling", "profiling_output", "profiling_mode") + + +def _parse_uc_sql(path): + """Return (preamble_stmts, select_stmt) parsed from any UC query SQL file. + Strips LOAD, PRAGMA, profiling SETs, and SET enable_extended_optimizer — + all are either injected at runtime or irrelevant to plan visualization.""" + try: + with open(path) as f: + content = f.read() + stmts = [s.strip() for s in content.split(";") if s.strip()] + preamble, select = [], None + for stmt in stmts: + first_tok = "" + for line in stmt.split("\n"): + clean = line.split("--")[0].strip() + if clean: + parts = clean.split() + first_tok = parts[0].upper() if parts else "" + break + if not first_tok: + pass # comment-only chunk — skip + elif first_tok in ("SELECT", "WITH"): + select = stmt + elif first_tok in ("LOAD", "PRAGMA"): + pass # LOAD injected by _build_session_sql; PRAGMA not needed for EXPLAIN + elif first_tok == "SET" and "enable_extended_optimizer" in stmt.lower(): + pass # overridden per-optimizer at runtime + elif first_tok == "SET" and any(kw in stmt.lower() for kw in _PROFILING_KEYWORDS): + pass # profiling directives not needed for plan visualization + else: + preamble.append(stmt) + return preamble, select + except Exception as e: + app.logger.warning("Could not parse UC SQL %s: %s", path, e) + return [], None + + +UC01_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/uc01_query.sql") +UC03_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/uc03_query.sql") +UC04_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/uc04_query.sql") +UC08_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/uc08_query_factorizable.sql") +EXPEDIA_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/expedia_query.sql") +FLIGHTS_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/flights_query.sql") + +UC01_PREAMBLE, UC01_SELECT = _parse_uc_sql(UC01_SQL_PATH) +UC03_PREAMBLE, UC03_SELECT = _parse_uc_sql(UC03_SQL_PATH) +UC04_PREAMBLE, UC04_SELECT = _parse_uc_sql(UC04_SQL_PATH) +UC08_PREAMBLE, UC08_SELECT = _parse_uc_sql(UC08_SQL_PATH) +EXPEDIA_PREAMBLE, EXPEDIA_SELECT = _parse_uc_sql(EXPEDIA_SQL_PATH) +FLIGHTS_PREAMBLE, FLIGHTS_SELECT = _parse_uc_sql(FLIGHTS_SQL_PATH) + + # --- Demo in-memory data ---------------------------------------------------- QUERIES = [ @@ -387,6 +590,30 @@ def _parse_synthetic_sql(): _q["runnable"] = bool(SQL_SELECT) _q["schema"] = "synthetic" _q["preamble"] = SQL_PREAMBLE + if _q["id"] == "q_uc01": + _q["runnable"] = bool(UC01_SELECT) + _q["sql"] = UC01_SELECT or _q["sql"] + _q["preamble"] = UC01_PREAMBLE + if _q["id"] == "q_uc03": + _q["runnable"] = bool(UC03_SELECT) + _q["sql"] = UC03_SELECT or _q["sql"] + _q["preamble"] = UC03_PREAMBLE + if _q["id"] == "q_uc04": + _q["runnable"] = bool(UC04_SELECT) + _q["sql"] = UC04_SELECT or _q["sql"] + _q["preamble"] = UC04_PREAMBLE + if _q["id"] == "q_uc08": + _q["runnable"] = bool(UC08_SELECT) + _q["sql"] = UC08_SELECT or _q["sql"] + _q["preamble"] = UC08_PREAMBLE + if _q["id"] == "q_expedia": + _q["runnable"] = bool(EXPEDIA_SELECT) + _q["sql"] = EXPEDIA_SELECT or _q["sql"] + _q["preamble"] = EXPEDIA_PREAMBLE + if _q["id"] == "q_flights": + _q["runnable"] = bool(FLIGHTS_SELECT) + _q["sql"] = FLIGHTS_SELECT or _q["sql"] + _q["preamble"] = FLIGHTS_PREAMBLE def _get_query(query_id): @@ -401,6 +628,7 @@ def _get_query(query_id): "Mul2JoinAggRewriteAction", "MultiLayerUDF2TorchNNRewriteAction", "MLDecompositionPushdownRewriteAction", + "MLFactorizationRewriteAction", "MatMulDense2SparseRewriteAction", "TreeModelTeePruningRewriteAction", ] @@ -431,6 +659,13 @@ def _get_query(query_id): "plan_key": "optimizer2", "opt_setting": "2", }, + { + "id": "opt_factorize", + "name": "Optimizer3: ML factorization (high FLOP queries)", + "description": "Split mat_mul across join boundary when ml_flops > 5B", + "plan_key": "baseline", + "opt_setting": "4", + }, ] # --- Optimizer definitions shown in the Optimizer Definition pane ---------- @@ -457,6 +692,12 @@ def _get_query(query_id): [action] MLDecompositionPushdownRewriteAction [action] MatMulDense2SparseRewriteAction """, + + "opt_factorize": r"""IF + [metric] ml_flops > 5e9 +THEN + [action] MLFactorizationRewriteAction +""", } @@ -515,13 +756,14 @@ def serialize_rule(parsed): return f"RULE#{cond}#{','.join(parsed['actions'])}" -ACTIVE_OPTIMIZER_IDS = {"noopt", "opt1", "opt2"} +ACTIVE_OPTIMIZER_IDS = {"noopt", "opt1", "opt2", "opt_factorize"} -BENCHMARK_OPT_ORDER = ["noopt", "opt1", "opt2"] +BENCHMARK_OPT_ORDER = ["noopt", "opt1", "opt2", "opt_factorize"] BENCHMARK_LABELS = { "noopt": "NoOptimizer", "opt1": "Optimizer1", "opt2": "Optimizer2", + "opt_factorize": "Optimizer3", } METRICS = { @@ -531,7 +773,39 @@ def serialize_rule(parsed): "join_cardinality_ratio": 0.0002, "feature_nonzero_ratio": 0.002, "feature_width": "150", - } + }, + "q_uc01": { + "customers": "7,071", + "orders": "367,695", + "lineitems": "2,306,263", + "order_returns": "134,204", + "features": "2 (frequency, return_ratio)", + "kmeans_k": "5", + }, + "q_uc04": { + "inference_reviews": "2,687", + "training_reviews": "10,746", + "vocab_size": "18,925", + "spam_ratio": "48.6%", + "model": "Naive Bayes (Laplace smoothed)", + }, + "q_uc03": { + "cardinality": "38,896", + "nnz_ratio": "0.025", + "zero_rows": "0.0", + "zero_cols": "0.0", + "ml_flops": "280 M", + "num_parameters": "7,297", + }, + "q_uc08": { + "cardinality": "2,122,088", + "join_cardinality_ratio": "0.0003", + "nnz_ratio": "0.021", + "zero_rows": "0.0", + "zero_cols": "0.0", + "ml_flops": "6.79 B", + "num_parameters": "3,200", + }, } @@ -656,15 +930,186 @@ def serialize_rule(parsed): └───────────────────────────┘""", } +# Per-query static plans keyed by (query_id, optimizer_id). +# Used by api_query_plan before falling back to the generic PLANS dict. +QUERY_PLANS = { + ("q_uc01", "noopt"): """\ +┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ customer_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ customer_id, │ +│ kmeans_predict( │ +│ min_max_scaler( │ +│ [frequency, │ +│ return_ratio], │ +│ uc01_scaler.csv), │ +│ uc01_centroids.csv) │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ r.customer_id = ├──────────────┐ +│ f.customer_id │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ AGGREGATE ││ AGGREGATE │ +│ AVG(orders_per_year) ││ AVG(ret_price/ │ +│ GROUP BY customer_id ││ row_price) │ +└─────────────┬─────────────┘└─────────────┬─────────────┘ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ COMPARISON_JOIN ││ HASH_JOIN │ +│ li_order_id=o_order_id ││ li_order_id=or_order_id │ +│ (+ LEFT JOIN ││ li_product_id= │ +│ order_returns) ││ or_product_id │ +└─────────────┬─────────────┘└─────────────┬─────────────┘ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ Table: lineitem ││ Table: order_returns │ +│ (2,306,263 rows) ││ (134,204 rows) │ +└───────────────────────────┘└───────────────────────────┘""", + + ("q_uc04", "noopt"): """\ +┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ id, │ +│ CASE WHEN │ +│ log_spam_score │ +│ + log_spam_prior │ +│ > log_ham_score │ +│ + log_ham_prior │ +│ THEN 1 ELSE 0 END │ +│ AS predicted_spam │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ CROSS_JOIN │ +│ token_scores × priors │ +└──────────┬────────────────┘ +┌──────────┴────────────────┐┌──────────────────────────────┐ +│ AGGREGATE ││ SEQ_SCAN │ +│ SUM(LN(spam_prob)), ││ Table: uc04_train_ │ +│ SUM(LN(non_spam_prob)) ││ preprocessed │ +│ GROUP BY id ││ (10,746 rows) │ +└──────────┬────────────────┘└──────────────────────────────┘ +┌──────────┴────────────────┐ +│ HASH_JOIN │ +│ ──────────────────── │ +│ Join Type: LEFT │ +│ token = token │ +└──────────┬────────────────┘ +┌──────────┴────────────────┐┌──────────────────────────────┐ +│ UNNEST(tokenize_text( ││ SEQ_SCAN │ +│ clean_text( ││ Table: uc04_model │ +│ stem_token(text)))) ││ (18,925 tokens) │ +└──────────┬────────────────┘└──────────────────────────────┘ +┌──────────┴────────────────┐ +│ SEQ_SCAN │ +│ Table: review │ +│ (2,687 rows) │ +└───────────────────────────┘""", + + ("q_uc08", "noopt"): """\ +┌───────────────────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ o_order_id, date, h1 │ +│ mat_mul( │ +│ list_concat( │ +│ [qty, scan_count, wd], │ +│ one_hot_encode(dept) │ +│ ), W1_wide, 'dense') │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ li_product_id = ├────────────────────────────┐ +│ p_product_id │ │ +└─────────────┬─────────────┘ ┌─────────────────┴──────────────┐ +┌─────────────┴─────────────┐ │ SEQ_SCAN │ +│ PROJECTION (agg cols) │ │ Table: product │ +└─────────────┬─────────────┘ │ (707 rows) │ +┌─────────────┴─────────────┐ └────────────────────────────────┘ +│ AGGREGATE │ +│ SUM(qty), MIN(weekday) │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ o_order_id=li_order_id ├──────────────┐ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ Table: order_tbl ││ Table: lineitem │ +└───────────────────────────┘└───────────────────────────┘""", + + ("q_uc08", "opt_factorize"): """\ +┌───────────────────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ o_order_id, date, │ +│ list_add_vv(#left, #right│ +│ ) ← replaces mat_mul │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ li_product_id = ├────────────────────────────┐ +│ p_product_id │ │ +└─────────────┬─────────────┘ ┌─────────────────┴──────────────┐ +┌─────────────┴─────────────┐ │ PROJECTION │ +│ PROJECTION │ │ mat_mul( │ +│ mat_mul( │ │ one_hot_encode(dept), │ +│ [qty, scan_count, wd], │ │ W1_right[47×64]) │ +│ W1_left[3×64]) │ │ (707 rows — amortized!) │ +│ (2,122,088 rows) │ └─────────────────┬──────────────┘ +└─────────────┬─────────────┘ ┌─────────────────┴──────────────┐ +┌─────────────┴─────────────┐ │ SEQ_SCAN │ +│ AGGREGATE │ │ Table: product │ +│ SUM(qty), MIN(weekday) │ └────────────────────────────────┘ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ o_order_id=li_order_id ├──────────────┐ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ Table: order_tbl ││ Table: lineitem │ +└───────────────────────────┘└───────────────────────────┘""", +} + # Benchmark results (latency in ms) — hardcoded for the full suite display BENCHMARK_RESULTS = { - "queries": ["Q1", "Q2", "Q3", "Q4", "Q5", "Q6", "Q7", "Q8", "Q9", "Q10"], - "query_types": ["ml", "sql", "ml", "ml", "sql", "ml", "ml", "sql", "ml", "sql"], - "optimizers": ["NoOptimizer", "Optimizer1", "Optimizer2"], + "queries": [ + "Q1", "Q2", "Q3", "Q4", "Q_UC01", "Q_UC03", "Q_UC08", "Q8", "Q_UC04", "Q10", + "Q_UC03", "Q_UC08", + ], + "query_ids": [ + "ranker_q1", "q_expedia", "q_flights", "q_credit", "q_uc01", + "q_uc03", "q_uc08", "q_uc10", "q_uc04", "ranker_q1", + "q_uc03", "q_uc08", + ], + "query_types": [ + "ml", "sql", "ml", "ml", "ml", "ml", "ml", "sql", "ml", "sql", + "ml", "ml", + ], + "optimizers": ["NoOptimizer", "Optimizer1", "Optimizer2", "Optimizer3"], "latencies": [ - [85000.0, 12000.0, 60000.0, 15000.0, 20000.0, 18000.0, 40000.0, 22000.0, 30000.0, 5000.0], - [3674.0, 7000.0, 9000.0, 8000.0, 12000.0, 9000.0, 12000.0, 15000.0, 11000.0, 4500.0], - [1976.0, 6800.0, 6000.0, 7800.0, 11500.0, 8500.0, 8000.0, 14000.0, 9000.0, 4400.0], + [85000.0, 12000.0, 60000.0, 15000.0, 4800.0, 18000.0, 40000.0, 22000.0, 620.0, 5000.0, 864.0, 1030.0], + [3674.0, 7000.0, 9000.0, 8000.0, 4800.0, 9000.0, 12000.0, 15000.0, 620.0, 4500.0, 864.0, 1030.0], + [1976.0, 6800.0, 6000.0, 7800.0, 4800.0, 8500.0, 8000.0, 14000.0, 620.0, 4400.0, 864.0, 1030.0], + [1976.0, 6800.0, 6000.0, 7800.0, 4800.0, 8500.0, 8000.0, 14000.0, 620.0, 4400.0, 864.0, 924.0], ], } @@ -678,7 +1123,10 @@ def _build_session_sql(opt_setting: str, body: str, preamble=None) -> str: selected query's own setup (defaults to the synthetic query's preamble).""" if preamble is None: preamble = SQL_PREAMBLE - lines = [f"LOAD '{EXTENSION_PATH}';"] + # Use the bare extension name so DuckDB resolves it relative to its own + # binary directory (build/release/extension/cactusdb/) — same as the SQL + # files themselves use when run directly from the command line. + lines = ["LOAD 'cactusdb';"] for stmt in preamble: lines.append(stmt + ";") lines.append(f"SET enable_extended_optimizer = '{opt_setting}';") @@ -715,6 +1163,11 @@ def index(): definition_opt_id = "opt1" rules_snippet = OPTIMIZER_DEFINITIONS.get(definition_opt_id, "") + # Cost-based passes (OPT#...) aren't editable through the IF-THEN form; the + # definition window shows them read-only and points the user at re-uploading. + definition_opt = next((o for o in OPTIMIZERS if o["id"] == definition_opt_id), None) + definition_is_pass = bool(definition_opt and definition_opt.get("kind") == "pass") + def get_opt(opt_id, default_id): if opt_id not in ACTIVE_OPTIMIZER_IDS: opt_id = default_id @@ -746,10 +1199,25 @@ def get_opt(opt_id, default_id): selected_optimizer_right=selected_optimizer_right, rules_snippet=rules_snippet, definition_optimizer_id=definition_opt_id, + definition_is_pass=definition_is_pass, custom_slots=custom_slots, ) +def _extract_explain_output(raw: str) -> str: + """Return the DuckDB EXPLAIN plan tree, stripping the 'Physical Plan' title box.""" + lines = raw.splitlines() + # Strip the "Physical Plan" title box (always the first 5-line box DuckDB emits). + phys_idx = next((i for i, l in enumerate(lines) if "Physical Plan" in l), None) + if phys_idx is not None: + # Advance past the closing └ of the title box, then drop leading blank lines. + end = phys_idx + while end < len(lines) and not lines[end].startswith("└"): + end += 1 + lines = lines[end + 1:] + return "\n".join(lines).strip() + + @app.route("/api/query_plan") def api_query_plan(): """Return the DuckDB physical plan for a query+optimizer pair. @@ -771,8 +1239,9 @@ def api_query_plan(): ) proc = _run_duckdb(sql) if proc.returncode == 0 and proc.stdout.strip(): - return jsonify({"plan": proc.stdout.strip()}) - app.logger.warning("Live plan stderr: %s", proc.stderr[:200]) + return jsonify({"plan": _extract_explain_output(proc.stdout)}) + app.logger.warning("Live plan failed (rc=%d): %s", + proc.returncode, proc.stderr[:300]) except Exception as e: app.logger.warning("Live plan failed, using fallback: %s", e) @@ -892,17 +1361,88 @@ def create_optimizer(): @app.route("/upload_optimizer", methods=["POST"]) def upload_optimizer(): + """Upload a self-contained cost-based optimizer pass (.cpp + optional .hpp), + validate it, drop it into plugins/opt_passes/, rebuild the extension, and on a + green build activate it as a selectable optimizer (OPT#).""" file = request.files.get("optimizer_file") if not file or file.filename == "": - flash("No optimizer file selected.", "danger") + flash("No optimizer .cpp selected.", "danger") return redirect(url_for("index")) - filename = secure_filename(file.filename) - file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) - flash(f"Optimizer '{filename}' uploaded successfully.", "success") + with BUILD_LOCK: + if BUILD_STATE["status"] == "building": + flash("A build is already in progress. Wait for it to finish.", "warning") + return redirect(url_for("index")) + + name = request.form.get("optimizer_name", "").strip() + expected = request.form.get("pass_name", "").strip() + + text = file.read().decode("utf-8", errors="replace") + pass_name, err = _lint_plugin_source(text) + if err: + flash(err, "danger") + return redirect(url_for("index")) + if expected and expected != pass_name: + flash(f"Declared pass name '{expected}' != REGISTER_OPT_PASS name '{pass_name}'.", "danger") + return redirect(url_for("index")) + if not name: + name = f"Greedy/{pass_name}" + + # --- choose a custom slot (shared with the rule-based builder) --- + used = {o["id"] for o in OPTIMIZERS if o["id"] in CUSTOM_SLOT_IDS} + existing_for_pass = next( + (o["id"] for o in OPTIMIZERS if o.get("opt_setting") == f"OPT#{pass_name}"), None + ) + if existing_for_pass: + slot_id = existing_for_pass # re-upload of the same pass overwrites + else: + free = [s for s in CUSTOM_SLOT_IDS if s not in used] + if not free: + flash("All 5 custom optimizer slots are in use. Free one first.", "danger") + return redirect(url_for("index")) + slot_id = free[0] + + # --- place the sources where CMake globs them --- + os.makedirs(PLUGIN_INCLUDE_DIR, exist_ok=True) + cpp_path = os.path.join(PLUGIN_DIR, f"{pass_name}_pass.cpp") + with open(cpp_path, "w") as f: + f.write(text) + header = request.files.get("header_file") + if header and header.filename: + header.save(os.path.join(PLUGIN_INCLUDE_DIR, secure_filename(header.filename))) + _regenerate_manifest() # make the static link retain this plugin + + # --- kick off the rebuild asynchronously --- + with BUILD_LOCK: + BUILD_STATE.update(status="building", pass_name=pass_name, slot_id=slot_id, + message=f"Building extension with '{pass_name}'…") + threading.Thread( + target=_run_build_async, args=(slot_id, pass_name, name), daemon=True + ).start() + + flash(f"Uploaded '{pass_name}'. Building extension — watch the build status.", "info") return redirect(url_for("index")) +@app.route("/api/build_status") +def api_build_status(): + """Build status + a tail of the compiler log for the upload UI to poll. + A terminal 'ok' is one-shot: reported once, then reset to 'idle' so the + client reload doesn't re-trigger. 'error' persists so the user can read it.""" + with BUILD_LOCK: + state = dict(BUILD_STATE) + if BUILD_STATE["status"] == "ok": + BUILD_STATE.update(status="idle", message="") + tail = "" + try: + with open(BUILD_LOG) as f: + tail = "".join(f.readlines()[-40:]) + except OSError: + pass + state["log_tail"] = tail + return jsonify(state) + + @app.route("/upload_action", methods=["POST"]) def upload_action(): file = request.files.get("action_file") @@ -916,6 +1456,13 @@ def upload_action(): return redirect(url_for("index")) +@app.route("/api/query_stats") +def api_query_stats(): + """Return statistics for a given query_id (used by the live stats panel).""" + query_id = request.args.get("query_id", "") + return jsonify({"stats": METRICS.get(query_id, {})}) + + @app.route("/api/benchmark_data") def api_benchmark_data(): queries = BENCHMARK_RESULTS["queries"] @@ -932,6 +1479,7 @@ def api_benchmark_data(): return jsonify({ "queries": queries, + "query_ids": BENCHMARK_RESULTS.get("query_ids", []), "query_types": query_types, "optimizers": active_labels, "latencies": active_latencies, diff --git a/Frontend/templates/index.html b/Frontend/templates/index.html index 58deef3..62b0f93 100644 --- a/Frontend/templates/index.html +++ b/Frontend/templates/index.html @@ -149,21 +149,21 @@
- - +
SQL + ML
- 
{{ selected_query.sql }}
+ 
{{ selected_query.sql }}
@@ -246,22 +246,24 @@

Data / plan statistics for this SQL+ML query (used as input to optimizer decisions).

- {% if metrics %} -
- - - {% for k, v in metrics.items() %} - - - - - {% endfor %} - -
{{ k }}{{ v }}
-
- {% else %} - No statistics available for this query yet. - {% endif %} +
+ {% if metrics %} +
+ + + {% for k, v in metrics.items() %} + + + + + {% endfor %} + +
{{ k }}{{ v }}
+
+ {% else %} + No statistics available for this query yet. + {% endif %} +
@@ -338,12 +340,20 @@

- Definition for {{ definition_optimizer_id }} (use it as a starting point): + Definition for {{ definition_optimizer_id }}{% if not definition_is_pass %} (use it as a starting point){% endif %}: 

{{ rules_snippet }}
+ {% if definition_is_pass %} +

Cost-Based Optimizer Pass

+

+ This is a compiled-in cost-based pass — it is not expressible as an + IF-THEN metric rule. To change it, upload a revised .cpp + via Upload Cost-Based Optimizer Pass and rebuild. +

+ {% else %}

Create / Edit Custom Optimizer

@@ -386,6 +396,7 @@
+ {% endif %}
@@ -411,13 +422,33 @@ action="{{ url_for('upload_optimizer') }}" enctype="multipart/form-data" > - + + + + +
+ +
- - + +
+ + Compiles the pass into the extension and activates it as + OPT#<pass_name>. Rebuild takes a few minutes. + + + +
window.location.reload(), 1500); + } + } + // ── Constants ────────────────────────────────────────────────────────── - const SELECTED_QUERY_ID = "{{ selected_query.id|e }}"; - const SELECTED_QUERY_LABEL = "{{ selected_query.name|e }}"; + let SELECTED_QUERY_ID = "{{ selected_query.id|e }}"; + let SELECTED_QUERY_LABEL = "{{ selected_query.name|e }}"; + + // All queries embedded for instant SQL display on selector change + const ALL_QUERIES = {{ queries | tojson }}; + const QUERY_SQL_MAP = {}; + ALL_QUERIES.forEach(q => { QUERY_SQL_MAP[q.id] = q.sql || ""; }); + + // ── Query selector change ────────────────────────────────────────────── + async function onQueryChange(queryId) { + SELECTED_QUERY_ID = queryId; + const q = ALL_QUERIES.find(q => q.id === queryId); + SELECTED_QUERY_LABEL = q ? q.name : queryId; + + // Update SQL display immediately from embedded data + const sqlEl = document.getElementById("sqlDisplay"); + if (sqlEl) sqlEl.textContent = QUERY_SQL_MAP[queryId] || "(no SQL available)"; + + // Sync hidden query_id inputs in plan-selector forms + document.querySelectorAll('input[name="query_id"]').forEach(el => { + el.value = queryId; + }); + + // Refresh plan panes + const leftOptId = document.querySelector('select[name="optimizer_left_id"]')?.value || "noopt"; + const rightOptId = document.querySelector('select[name="optimizer_right_id"]')?.value || "opt1"; + fetchPlan("planLeft", queryId, leftOptId); + fetchPlan("planRight", queryId, rightOptId); + + // Fetch and render statistics + await refreshStats(queryId); + + // Clear any stale live benchmark result from the previous query, + // then re-render the chart so it shows the selected-query slice. + liveSelectedData = null; + renderBenchmarkChart(); + + // Update URL without reload so back-button works + const url = new URL(window.location); + url.searchParams.set("query_id", queryId); + window.history.pushState({}, "", url); + } + + async function refreshStats(queryId) { + try { + const resp = await fetch(`/api/query_stats?query_id=${encodeURIComponent(queryId)}`); + const data = await resp.json(); + const stats = data.stats || {}; + const keys = Object.keys(stats); + const tbody = document.getElementById("statsTableBody"); + const empty = document.getElementById("statsEmpty"); + const panel = document.getElementById("statsPanel"); + + if (keys.length === 0) { + if (panel) panel.innerHTML = + 'No statistics available for this query yet.'; + return; + } + + // Rebuild table + const rows = keys.map(k => + `${k}${stats[k]}` + ).join(""); + + if (panel) panel.innerHTML = + `
` + + `${rows}
`; + } catch (e) { + console.warn("Could not fetch stats:", e); + } + } // ── Plan visualization ───────────────────────────────────────────────── async function fetchPlan(preId, queryId, optimizerId) { diff --git a/README.md b/README.md index 32760ee..c800c00 100644 --- a/README.md +++ b/README.md @@ -78,3 +78,66 @@ Different tests can be created for DuckDB extensions. The primary way of testing ```sh make test ``` + +--- + +## Frontend (OPTBench Web UI) + +The frontend is a Flask web app that lets you explore query plans, compare optimizers, run benchmarks, and upload custom optimizer passes — all from a browser. + +### Prerequisites + +Python 3.9+ is required. Install the Python dependencies: + +```sh +pip install flask werkzeug +``` + +> **Note:** The frontend shells out to `./build/release/duckdb`, so the extension must be built first (see [Build steps](#build-steps) above). + +### Starting the server + +The `Frontend/app.py` file expects to be run from **the project root** so that relative paths to `build/`, `queries/`, and `plugins/` resolve correctly: + +```sh +cd +python Frontend/app.py +``` + +Flask will start on `http://127.0.0.1:5000` by default. + +If `LIBTORCH_DIR` is not already exported in your shell, set it before launching: + +```sh +LIBTORCH_DIR=~/libtorch python Frontend/app.py +``` + +### What the UI provides + +| Section | Description | +|---|---| +| **Query explorer** | Browse the built-in SQL+ML queries (Expedia, Flights, UC01–UC10, synthetic ranker) | +| **Plan comparison** | Side-by-side physical plan diff for any two optimizer settings | +| **Benchmark runner** | Execute a query under every active optimizer and compare runtimes | +| **Optimizer builder** | Write IF-THEN rules using the metric catalog to create custom optimizers | +| **Optimizer upload** | Upload a `.cpp` pass file; the server rebuilds the extension and activates it live | + +### Uploading a custom optimizer pass + +1. Write your pass as a self-contained `.cpp` file that calls `REGISTER_OPT_PASS(, check, apply)`. +2. Open the **Upload Pass** panel in the UI and select your `.cpp` (and an optional `.hpp` header). +3. The server drops the file into `plugins/opt_passes/`, regenerates the manifest, runs `make release`, and activates the pass under the slot name `OPT#` — no manual rebuild needed. +4. Build progress is shown in real time via the **Build Status** indicator in the UI. + +> **Allowed `#include` headers:** standard library headers (`algorithm`, `vector`, `string`, …) and project-internal headers under `duckdb/`, `optimization/`, `ml_functions/`, and `cost_model/`. System or third-party headers are rejected at upload time. + +### Production / non-debug mode + +For demos or multi-user use, run Flask behind a production WSGI server: + +```sh +pip install gunicorn +gunicorn -w 1 -b 0.0.0.0:5000 --chdir "Frontend.app:app" +``` + +> Single worker (`-w 1`) is required because the optimizer state and build status are kept in-process memory; multiple workers would not share them. diff --git a/baseline.sh b/baseline.sh new file mode 100755 index 0000000..850c45f --- /dev/null +++ b/baseline.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Resets the plugin directory to a clean baseline so the upload flow can be re-tested. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_DIR="$SCRIPT_DIR/plugins/opt_passes" + +# Remove any uploaded pass sources and rejected files +find "$PLUGIN_DIR" -maxdepth 1 -name "*.cpp" ! -name "_manifest.cpp" -delete +find "$PLUGIN_DIR" -maxdepth 1 -name "*.rejected" -delete +find "$PLUGIN_DIR/include" -maxdepth 1 -type f ! -name ".gitkeep" -delete + +# Reset manifest to empty baseline +cat > "$PLUGIN_DIR/_manifest.cpp" << 'EOF' +// AUTO-GENERATED by the frontend upload flow — do not edit by hand. +#include + +namespace duckdb { + +void LinkOptPassPlugins() { + static std::vector keep; + (void)keep; +} + +} // namespace duckdb +EOF + +echo "Baseline restored. Plugin directory:" +ls "$PLUGIN_DIR" diff --git a/demo/uploadable_passes/greedy_factorization_pass.cpp b/demo/uploadable_passes/greedy_factorization_pass.cpp new file mode 100644 index 0000000..69ff7b3 --- /dev/null +++ b/demo/uploadable_passes/greedy_factorization_pass.cpp @@ -0,0 +1,913 @@ +// Uploadable optimizer-pass plugin: greedy cost-based factorization. +// The full MLGreedyFactorizationOptimizer algorithm plus one REGISTER_OPT_PASS +// line at the bottom. Upload via the frontend; it self-registers as +// "greedy_factorization" and becomes selectable via OPT#greedy_factorization. + +#include "optimization/optimizer_pass.hpp" +#include "optimization/actions/MLGreedyFactorizationOptimizer.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/assert.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/value.hpp" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" + +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/operator/logical_comparison_join.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" + +#include "ml_functions/matrix_multiplication.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +// --------------------------------------------------------------------------- +// JoinTreeNode +// --------------------------------------------------------------------------- + +struct JoinTreeNode { + LogicalOperator *plan_node; // pointer into the live DuckDB plan tree + int side; // 0=left child of parent, 1=right child, -1=root + JoinTreeNode *parent; // nullptr for root + std::vector children; // up to 2 for a binary join + + // Populated during tree walk + idx_t num_rows; // estimated cardinality + int64_t num_features; // total input feature dims still un-reduced at this node + int64_t new_features; // model output width n (first-layer neuron count) + double cardinality_ratio; // num_rows / parent->num_rows (selectivity from this side) + int depth; // absolute depth (root = 1) + int max_depth; // deepest node in the whole tree (same for all nodes after BFS) + double cost; // current cost estimate + + // Assignment phase + int label; // 0=unassigned, 1=push mat_mul here, 2=combine children (list_add_vv) + bool processed; + bool subtree_processed; + + // Weight-split info set during apply phase + idx_t row_start; // start row in the original W for this node's feature block + idx_t row_count; // number of rows (= sum of k_i for tables on this side) +}; + +// --------------------------------------------------------------------------- +// Expression helpers (shared with MLFactorizationRewriteAction pattern) +// --------------------------------------------------------------------------- + +static bool IsMatMul(const BoundFunctionExpression &f) { + return StringUtil::CIEquals(f.function.name, "mat_mul"); +} + +static bool IsListConcat(const BoundFunctionExpression &f) { + auto name = StringUtil::Lower(f.function.name); + return name == "list_concat" || name == "list_cat" || name == "array_concat"; +} + +static bool AreConsecutivePartIndices(const std::vector &indices) { + if (indices.empty()) return false; + for (idx_t i = 1; i < indices.size(); ++i) { + if (indices[i] != indices[i - 1] + 1) return false; + } + return true; +} + +static int64_t GetStaticListLength(const unique_ptr &expr) { + if (!expr) return -1; + if (expr->return_type.id() == LogicalTypeId::ARRAY) + return static_cast(ArrayType::GetSize(expr->return_type)); + if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) + return static_cast(ArrayType::GetSize(cast.child->return_type)); + } + if (expr->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = expr->Cast(); + auto name = StringUtil::Lower(bfe.function.name); + if (name == "list_value" || name == "array_value") + return static_cast(bfe.children.size()); + } + return -1; +} + +// Collect every table_index referenced by BoundColumnRefExpressions in expr. +static void CollectTableIndexes(Expression &expr, + std::unordered_set &out) { + auto owned_copy = expr.Copy(); + ExpressionIterator::EnumerateExpression( + owned_copy, [&](Expression &sub) { + if (sub.type == ExpressionType::BOUND_COLUMN_REF) + out.insert(sub.Cast().binding.table_index); + }); +} + +// Recursively flatten list_concat(list_concat(A,B),C) -> [{A,k_A},{B,k_B},{C,k_C}] +struct FlatPart { + Expression *expr; + int64_t length; +}; + +static void FlattenListConcatImpl(Expression *expr, std::vector &out) { + if (!expr) return; + + // Unwrap implicit cast DOUBLE[N] -> DOUBLE[] inserted by DuckDB + Expression *inner = expr; + if (inner->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = inner->Cast(); + if (cast.child) inner = cast.child.get(); + } + + if (inner->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = inner->Cast(); + if (IsListConcat(bfe) && bfe.children.size() == 2) { + FlattenListConcatImpl(bfe.children[0].get(), out); + FlattenListConcatImpl(bfe.children[1].get(), out); + return; + } + } + + // Leaf: compute static length and record + // Use original expr (not inner) so GetStaticListLength can unwrap the cast itself + unique_ptr dummy; // we only need length from it + int64_t len = -1; + if (expr->return_type.id() == LogicalTypeId::ARRAY) + len = static_cast(ArrayType::GetSize(expr->return_type)); + else if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) + len = static_cast(ArrayType::GetSize(cast.child->return_type)); + } + out.push_back({expr, len}); +} + +static std::vector FlattenListConcat(Expression &expr) { + std::vector parts; + FlattenListConcatImpl(&expr, parts); + return parts; +} + +static unique_ptr BuildConcatFromParts(const BoundFunctionExpression &concat_proto, + const std::vector &parts, + const std::vector &indices) { + D_ASSERT(!indices.empty()); + auto expr = parts[indices[0]].expr->Copy(); + for (idx_t i = 1; i < indices.size(); ++i) { + vector> children; + children.push_back(std::move(expr)); + children.push_back(parts[indices[i]].expr->Copy()); + expr = make_uniq( + LogicalType::LIST(LogicalType::DOUBLE), + concat_proto.function, + std::move(children), + nullptr); + } + return expr; +} + +// --------------------------------------------------------------------------- +// Weight file helper +// --------------------------------------------------------------------------- + +static std::string WriteSplitWeightCSV(const std::vector &weights, idx_t n, + idx_t row_start, idx_t row_count, + const std::string &base_path, + const std::string &suffix) { + std::size_t h = std::hash{}( + base_path + "|" + std::to_string(row_start) + "|" + std::to_string(row_count)); + std::string out_path = "/tmp/matmul_greedy_" + std::to_string(h) + suffix + ".csv"; + { + std::ifstream probe(out_path); + if (probe.good()) return out_path; + } + std::ofstream f(out_path); + if (!f.is_open()) + throw InternalException("MLGreedyFactorization: cannot write weight file: %s", out_path.c_str()); + for (idx_t row = row_start; row < row_start + row_count; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) f << ','; + f << weights[row * n + col]; + } + f << '\n'; + } + return out_path; +} + +static std::string WriteCustomWeightCSV(const std::vector &weights, idx_t k, idx_t n, + const std::string &base_path, + const std::string &suffix, + const std::string &cache_key) { + std::size_t h = std::hash{}( + base_path + "|" + cache_key + "|" + std::to_string(k) + "|" + std::to_string(n)); + std::string out_path = "/tmp/matmul_greedy_" + std::to_string(h) + suffix + ".csv"; + { + std::ifstream probe(out_path); + if (probe.good()) return out_path; + } + + std::ofstream f(out_path); + if (!f.is_open()) + throw InternalException("MLGreedyFactorization: cannot write weight file: %s", out_path.c_str()); + for (idx_t row = 0; row < k; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) f << ','; + f << weights[row * n + col]; + } + f << '\n'; + } + return out_path; +} + +// --------------------------------------------------------------------------- +// Expression builders +// --------------------------------------------------------------------------- + +static unique_ptr BuildMatMulExpr(ClientContext &ctx, + unique_ptr input_expr, + const std::string &W_path, + const std::string &mode_str, + const std::vector &W_flat, + idx_t k, idx_t n, + ML_MatrixMultiply::MulMode mode) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = catalog.GetEntry(ctx, DEFAULT_SCHEMA, "mat_mul"); + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 3 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1].id() == LogicalTypeId::VARCHAR && + fn.arguments[2].id() == LogicalTypeId::VARCHAR) { + chosen = &fn; + break; + } + } + if (!chosen) + throw InternalException("MLGreedyFactorization: mat_mul overload not found"); + + auto bind_info = make_uniq(W_flat, k, n, W_path, true, mode); + vector> children; + children.push_back(std::move(input_expr)); + children.push_back(make_uniq(Value(W_path))); + children.push_back(make_uniq(Value(mode_str))); + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), std::move(bind_info)); +} + +static unique_ptr BuildListAddVVExpr(ClientContext &ctx, + unique_ptr left, + unique_ptr right) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = catalog.GetEntry(ctx, DEFAULT_SCHEMA, "list_add_vv"); + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 2 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1] == LogicalType::LIST(LogicalType::DOUBLE)) { + chosen = &fn; + break; + } + } + if (!chosen) + throw InternalException("MLGreedyFactorization: list_add_vv overload not found"); + vector> children; + children.push_back(std::move(left)); + children.push_back(std::move(right)); + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), nullptr); +} + +// --------------------------------------------------------------------------- +// InsertProjectionOnSide +// --------------------------------------------------------------------------- + +static idx_t GetLogicalWidth(LogicalOperator &op) { + if (op.types.empty()) op.ResolveOperatorTypes(); + return op.types.size(); +} + +static bool InsertProjectionOnSide(LogicalComparisonJoin &join, int side_idx, + Expression &extra_expr, idx_t &new_col_idx, + idx_t &new_table_idx) { + D_ASSERT(side_idx == 0 || side_idx == 1); + auto &child_ptr = join.children[side_idx]; + if (!child_ptr) return false; + + auto side_tables = child_ptr->GetTableIndex(); + if (side_tables.size() != 1) return false; + + const idx_t table_index = side_tables[0]; + const idx_t child_width = GetLogicalWidth(*child_ptr); + if (child_width == 0) return false; + + vector> select_list; + select_list.reserve(child_width + 1); + for (idx_t i = 0; i < child_width; ++i) + select_list.push_back(make_uniq(child_ptr->types[i], i)); + select_list.push_back(extra_expr.Copy()); + new_col_idx = child_width; + + auto new_proj = make_uniq(table_index, std::move(select_list)); + new_proj->children.push_back(std::move(child_ptr)); + child_ptr = std::move(new_proj); + + // If the join already projects a subset of child columns, explicitly expose + // the appended column so parent expressions can bind to it. + if (side_idx == 0) { + if (!join.left_projection_map.empty()) { + join.left_projection_map.push_back(new_col_idx); + } + } else { + if (!join.right_projection_map.empty()) { + join.right_projection_map.push_back(new_col_idx); + } + } + + new_table_idx = table_index; + return true; +} + +// --------------------------------------------------------------------------- +// Join tree construction +// --------------------------------------------------------------------------- + +static idx_t EstimateCardinality(LogicalOperator &op) { + if (op.estimated_cardinality > 0) + return op.estimated_cardinality; + // Fallback: product of children cardinalities (pessimistic cross-join) + if (op.children.size() == 2) { + idx_t l = EstimateCardinality(*op.children[0]); + idx_t r = EstimateCardinality(*op.children[1]); + return std::max(1, l * r); + } + return 1; +} + +// Walk the DuckDB plan tree and collect all LogicalComparisonJoin nodes, +// building a parallel JoinTreeNode tree. Returns the root JoinTreeNode. +// all_nodes is populated in post-order (children before parents). +static JoinTreeNode *BuildJoinTree(LogicalOperator *op, + JoinTreeNode *parent, + int side, + int64_t new_features, + std::vector> &storage) { + if (!op) return nullptr; + + // Recurse into children first (post-order) + if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + auto node = make_uniq(); + node->plan_node = op; + node->side = side; + node->parent = parent; + node->new_features = new_features; + node->label = 0; + node->processed = false; + node->subtree_processed = false; + node->row_start = 0; + node->row_count = 0; + node->depth = 0; // filled by CalculateDepths + node->max_depth = 0; + node->cost = 0.0; + + idx_t card = EstimateCardinality(*op); + node->num_rows = card; + + if (parent) { + node->cardinality_ratio = (parent->num_rows > 0) + ? static_cast(card) / static_cast(parent->num_rows) + : 1.0; + } else { + node->cardinality_ratio = 1.0; + } + + JoinTreeNode *raw = node.get(); + + // Build children recursively + if (op->children.size() >= 1) { + JoinTreeNode *left_child = BuildJoinTree(op->children[0].get(), raw, 0, new_features, storage); + if (left_child) raw->children.push_back(left_child); + } + if (op->children.size() >= 2) { + JoinTreeNode *right_child = BuildJoinTree(op->children[1].get(), raw, 1, new_features, storage); + if (right_child) raw->children.push_back(right_child); + } + + storage.push_back(std::move(node)); + return raw; + } + + // Not a join — recurse into children looking for joins below + for (idx_t i = 0; i < op->children.size(); ++i) { + int child_side = (op->children.size() == 2) ? (int)i : side; + JoinTreeNode *found = BuildJoinTree(op->children[i].get(), parent, child_side, new_features, storage); + if (found) return found; // return first found (handles single-path plans) + } + return nullptr; +} + +// --------------------------------------------------------------------------- +// Depth computation +// --------------------------------------------------------------------------- + +static int CalculateDepths(JoinTreeNode *root, + std::vector> &all_nodes) { + int max_depth = 0; + std::queue> q; + q.push({root, 1}); + while (!q.empty()) { + auto [node, d] = q.front(); q.pop(); + node->depth = d; + if (d > max_depth) max_depth = d; + for (auto *child : node->children) q.push({child, d + 1}); + } + for (auto &n : all_nodes) n->max_depth = max_depth; + return max_depth; +} + +// --------------------------------------------------------------------------- +// Feature-part attribution: which flat parts belong to which join-tree side +// --------------------------------------------------------------------------- + +// Returns true if the given LogicalOperator subtree contains table_index t. +static bool SubtreeContainsTable(LogicalOperator *op, idx_t table_index) { + if (!op) return false; + auto tables = op->GetTableIndex(); + for (idx_t t : tables) + if (t == table_index) return true; + for (auto &child : op->children) + if (SubtreeContainsTable(child.get(), table_index)) return true; + return false; +} + +// For a given JoinTreeNode, return which side (0 or 1) the table lives on. +// Returns -1 if not found in either child subtree. +static int TableSideOfNode(JoinTreeNode *node, idx_t table_index) { + if (!node || node->plan_node->children.size() < 2) return -1; + if (SubtreeContainsTable(node->plan_node->children[0].get(), table_index)) return 0; + if (SubtreeContainsTable(node->plan_node->children[1].get(), table_index)) return 1; + return -1; +} + +// --------------------------------------------------------------------------- +// Cost model +// --------------------------------------------------------------------------- + +static double ComputeCost(const JoinTreeNode &v) { + if (v.num_features == 0 || v.depth == 0 || v.max_depth == 0) + return 1.0; // safe fallback above threshold + + double var2 = static_cast(v.num_features); + double var4 = static_cast(v.new_features); + double var5 = static_cast(v.max_depth); + double var6 = var4 / var2; + double var7 = var4 / (var2 * v.depth); + double var8 = static_cast(v.num_rows) * var4 / var2; + double var9 = static_cast(v.num_rows) * var4 / (var2 * v.depth); + double var10 = var2 - var4; + double var11 = static_cast(v.depth) / static_cast(v.max_depth); + double var12 = v.cardinality_ratio; + double d4 = std::pow(static_cast(v.depth), 4.0); + + return -4.62412178e-04 * var2 + - 6.99311355e-04 * var4 + - 4.17964429e-03 * var5 + + 1.57416551e-03 * var6 + - 3.69161585e-04 * var7 + - 2.94571978e-05 * var8 + + 6.07897302e-05 * var9 + + 2.36899161e-04 * var10 + - 4.63010544e-02 * var12 + + 2.12526468e-03 * d4 + + 1.40041493e-04 * std::pow(var11, 4.0) + - 3.86043424e-02 * var11 * var12; +} + +// --------------------------------------------------------------------------- +// Greedy algorithm helpers +// --------------------------------------------------------------------------- + +static void ProcessSubtree(JoinTreeNode *node) { + if (node->subtree_processed) return; + std::vector stack = {node}; + while (!stack.empty()) { + JoinTreeNode *cur = stack.back(); stack.pop_back(); + cur->processed = true; + cur->subtree_processed = true; + for (auto *child : cur->children) + if (!child->subtree_processed) stack.push_back(child); + } + if (node->parent) node->parent->subtree_processed = true; +} + +static void UpdateParentFeatures(JoinTreeNode *parent) { + int64_t total = 0; + for (auto *child : parent->children) + total += (child->label != 0) ? child->new_features : child->num_features; + parent->num_features = total; +} + +// --------------------------------------------------------------------------- +// Greedy assignment +// --------------------------------------------------------------------------- + +static void PerformGreedy(std::vector> &all_nodes) { + constexpr double COST_THRESHOLD = 0.5; + + // Prefer deeper joins first. This avoids selecting a parent early and + // marking its entire subtree processed before child joins can be labeled. + using Entry = std::tuple; + std::priority_queue, std::greater> heap; + + for (auto &n : all_nodes) { + n->cost = ComputeCost(*n); + if (n->cost < COST_THRESHOLD) { + int depth_priority = n->max_depth - n->depth; // smaller => deeper + heap.push({depth_priority, n->cost, n.get()}); + } + } + + while (!heap.empty()) { + auto [depth_priority, current_cost, v] = heap.top(); heap.pop(); + (void)depth_priority; + + if (v->processed) continue; + if (current_cost != v->cost) continue; // stale entry + + if (v->children.size() >= 2 && + v->children[0]->label != 0 && v->children[1]->label != 0) { + v->label = 2; + } else if (current_cost < COST_THRESHOLD) { + v->label = 1; + } else { + break; + } + + ProcessSubtree(v); + + if (v->parent) { + double old_cost = v->parent->cost; + UpdateParentFeatures(v->parent); + double new_cost = ComputeCost(*v->parent); + if (new_cost != old_cost) { + v->parent->cost = new_cost; + if (new_cost < COST_THRESHOLD) { + int parent_depth_priority = v->parent->max_depth - v->parent->depth; + heap.push({parent_depth_priority, new_cost, v->parent}); + } + } + } + } + + // Final pass: promote nodes whose both children are labeled to label=2 + for (auto &n : all_nodes) { + if (n->children.size() >= 2 && + n->children[0]->label != 0 && n->children[1]->label != 0) + n->label = 2; + } +} + +// --------------------------------------------------------------------------- +// check +// --------------------------------------------------------------------------- + +// Returns true if the plan contains a projection over a multi-join with +// mat_mul(list_concat(...), W) where list_concat has 2+ parts. +static bool HasGreedyPattern(LogicalOperator *op) { + if (!op) return false; + + if (op->type == LogicalOperatorType::LOGICAL_PROJECTION) { + // Check if any child is a comparison join + bool has_join_child = false; + for (auto &child : op->children) + if (child && child->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) + has_join_child = true; + + if (has_join_child) { + auto &proj = op->Cast(); + for (auto &expr : proj.expressions) { + if (!expr || expr->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &f = expr->Cast(); + if (!IsMatMul(f) || f.children.empty()) continue; + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) continue; + // Check that the argument is a list_concat with 2+ parts + auto &arg = f.children[0]; + if (!arg || arg->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &lc = arg->Cast(); + if (IsListConcat(lc)) { + auto parts = FlattenListConcat(*arg); + if (parts.size() >= 2) return true; + } + } + } + } + + for (auto &child : op->children) + if (HasGreedyPattern(child.get())) return true; + + return false; +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// Public interface +// --------------------------------------------------------------------------- + +bool MLGreedyFactorizationOptimizer::check(OptimizerExtensionInput &input, + unique_ptr &plan) { + return plan && HasGreedyPattern(plan.get()); +} + +bool MLGreedyFactorizationOptimizer::apply(OptimizerExtensionInput &input, + unique_ptr &plan) { + if (!plan) return false; + + // Bottom-up: rewrite children first + bool changed = false; + for (auto &child : plan->children) + changed |= apply(input, child); + + if (plan->type != LogicalOperatorType::LOGICAL_PROJECTION) return changed; + auto &proj = plan->Cast(); + if (proj.children.empty() || !proj.children[0] || + proj.children[0]->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) + return changed; + + // Find the mat_mul(list_concat(...), W) expression in the projection + BoundFunctionExpression *mat_mul_expr = nullptr; + unique_ptr *mat_mul_slot = nullptr; + for (auto &expr_ptr : proj.expressions) { + if (!expr_ptr || expr_ptr->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &f = expr_ptr->Cast(); + if (!IsMatMul(f) || f.children.empty()) continue; + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) continue; + auto &arg = f.children[0]; + if (!arg || arg->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &lc = arg->Cast(); + if (!IsListConcat(lc)) continue; + auto parts = FlattenListConcat(*arg); + if (parts.size() < 2) continue; + mat_mul_expr = &f; + mat_mul_slot = &expr_ptr; + break; + } + if (!mat_mul_expr) return changed; + + auto *bd = dynamic_cast(mat_mul_expr->bind_info.get()); + const idx_t n = bd->n; + const std::string &W_file = bd->weights_file; + std::string mode_str; + switch (bd->mode) { + case ML_MatrixMultiply::MulMode::Dense: mode_str = "dense"; break; + case ML_MatrixMultiply::MulMode::InputSparse: mode_str = "sparse"; break; + default: mode_str = "dense"; break; + } + + // Flatten list_concat into ordered parts + auto parts = FlattenListConcat(*mat_mul_expr->children[0]); + if (parts.size() < 2) return changed; + const auto &concat_proto = mat_mul_expr->children[0]->Cast(); + + // Validate all part lengths are known + for (auto &p : parts) + if (p.length <= 0) return changed; + + std::vector part_row_prefix(parts.size() + 1, 0); + for (idx_t i = 0; i < parts.size(); ++i) { + part_row_prefix[i + 1] = part_row_prefix[i] + static_cast(parts[i].length); + } + + // --------------------------------------------------------------------------- + // Build join tree + // --------------------------------------------------------------------------- + std::vector> node_storage; + JoinTreeNode *root = BuildJoinTree(proj.children[0].get(), nullptr, -1, + static_cast(n), node_storage); + if (!root || node_storage.empty()) return changed; + + // Set num_features on each node (total k = sum of all part lengths) + int64_t total_k = 0; + for (auto &p : parts) total_k += p.length; + for (auto &node : node_storage) node->num_features = total_k; + + // Compute depths + CalculateDepths(root, node_storage); + + // --------------------------------------------------------------------------- + // Run greedy + // --------------------------------------------------------------------------- + PerformGreedy(node_storage); + + // Check if anything was labeled — if not, nothing to do + bool any_labeled = false; + for (auto &node : node_storage) + if (node->label != 0) { any_labeled = true; break; } + if (!any_labeled) return changed; + + // Conservative safety guard: if any join already carries a projection map + // (i.e., columns have been pruned/reordered upstream), skip this rewrite. + // The greedy rewrite currently appends columns on join children and assumes + // stable pass-through bindings. + for (auto &node_ptr : node_storage) { + auto &join = node_ptr->plan_node->Cast(); + if (join.HasProjectionMap()) { + return changed; + } + } + + // --------------------------------------------------------------------------- + // Apply push-downs + // Iterate nodes in storage order (post-order: children before parents). + // For each label=1 node, figure out which flat parts belong to each side, + // build a mat_mul for that side, and insert a projection on the join child. + // + // We track, for each node's plan_node (join), the column indices of inserted + // partial results so the parent can reference them. + // --------------------------------------------------------------------------- + + // Map from plan_node pointer -> {left_col_idx, right_col_idx} of inserted projections + // -1 means not yet inserted on that side. + struct InsertRecord { + idx_t left_col = idx_t(-1); + idx_t right_col = idx_t(-1); + }; + std::unordered_map insert_map; + + for (auto &node_ptr : node_storage) { + JoinTreeNode *node = node_ptr.get(); + if (node->label != 1) continue; + + auto &join = node->plan_node->Cast(); + + // Determine which flat parts land on left vs right side of this join + std::vector left_part_indices, right_part_indices; + for (int i = 0; i < (int)parts.size(); ++i) { + std::unordered_set refs; + CollectTableIndexes(*parts[i].expr, refs); + + for (idx_t t : refs) { + int s = TableSideOfNode(node, t); + if (s == 0) { left_part_indices.push_back(i); break; } + if (s == 1) { right_part_indices.push_back(i); break; } + } + } + + // Helper lambda: build mat_mul for a set of consecutive part indices + auto build_side_mat_mul = [&](const std::vector &indices, int side_id) + -> unique_ptr { + if (indices.empty()) return nullptr; + + idx_t row_count = 0; + for (auto idx : indices) row_count += static_cast(parts[idx].length); + + std::vector W_slice; + W_slice.reserve(row_count * n); + + const idx_t row_start = part_row_prefix[indices.front()]; + for (auto part_idx : indices) { + const idx_t start = part_row_prefix[part_idx]; + const idx_t end = part_row_prefix[part_idx + 1]; + for (idx_t row = start; row < end; ++row) { + auto row_begin = bd->weights.begin() + static_cast(row * n); + W_slice.insert(W_slice.end(), row_begin, row_begin + static_cast(n)); + } + } + + std::string suffix = (side_id == 0) ? "_left" : "_right"; + std::string W_path; + if (AreConsecutivePartIndices(indices)) { + W_path = WriteSplitWeightCSV(bd->weights, n, row_start, row_count, W_file, suffix); + } else { + std::ostringstream key; + key << "parts"; + for (auto idx : indices) key << "_" << idx; + W_path = WriteCustomWeightCSV(W_slice, row_count, n, W_file, suffix, key.str()); + } + + unique_ptr input_expr; + if (indices.size() == 1) { + input_expr = parts[indices[0]].expr->Copy(); + } else { + input_expr = BuildConcatFromParts(concat_proto, parts, indices); + } + + return BuildMatMulExpr(input.context, std::move(input_expr), + W_path, mode_str, W_slice, + row_count, n, bd->mode); + }; + + InsertRecord rec; + + if (!left_part_indices.empty()) { + auto mm_left = build_side_mat_mul(left_part_indices, 0); + if (mm_left) { + idx_t col_idx = 0; + idx_t table_idx = idx_t(-1); + if (InsertProjectionOnSide(join, 0, *mm_left, col_idx, table_idx)) { + rec.left_col = col_idx; + } + } + } + if (!right_part_indices.empty()) { + auto mm_right = build_side_mat_mul(right_part_indices, 1); + if (mm_right) { + idx_t col_idx = 0; + idx_t table_idx = idx_t(-1); + if (InsertProjectionOnSide(join, 1, *mm_right, col_idx, table_idx)) { + rec.right_col = col_idx; + } + } + } + + insert_map[node->plan_node] = rec; + join.ResolveOperatorTypes(); + } + + // --------------------------------------------------------------------------- + // Replace the top-level mat_mul with a tree of list_add_vv over + // BoundReferenceExpressions pointing at inserted partial results. + // Walk label=2 nodes (and the root join) to collect references. + // --------------------------------------------------------------------------- + + auto &top_join = proj.children[0]->Cast(); + top_join.ResolveOperatorTypes(); + auto top_bindings = top_join.GetColumnBindings(); + + auto find_top_index = [&](const ColumnBinding &binding) -> idx_t { + for (idx_t i = 0; i < top_bindings.size(); ++i) { + if (top_bindings[i] == binding) return i; + } + return idx_t(-1); + }; + + // Find all nodes with inserted projections and build list_add_vv chain. + // Resolve each inserted column through the top join's current binding map, + // then reference the matching position with BoundReferenceExpression. + std::vector> partial_refs; + + for (auto &node_ptr : node_storage) { + JoinTreeNode *node = node_ptr.get(); + if (node->label != 1) continue; + auto it = insert_map.find(node->plan_node); + if (it == insert_map.end()) continue; + + auto &jn = node->plan_node->Cast(); + jn.ResolveOperatorTypes(); + auto node_bindings = jn.GetColumnBindings(); + auto left_child_bindings = jn.children[0]->GetColumnBindings(); + + if (it->second.left_col != idx_t(-1) && it->second.left_col < node_bindings.size()) { + const auto node_binding = node_bindings[it->second.left_col]; + const idx_t top_idx = find_top_index(node_binding); + if (top_idx != idx_t(-1) && top_idx < top_join.types.size()) { + partial_refs.push_back( + make_uniq(top_join.types[top_idx], top_idx)); + } + } + if (it->second.right_col != idx_t(-1)) { + const idx_t node_idx = left_child_bindings.size() + it->second.right_col; + if (node_idx < node_bindings.size()) { + const auto node_binding = node_bindings[node_idx]; + const idx_t top_idx = find_top_index(node_binding); + if (top_idx != idx_t(-1) && top_idx < top_join.types.size()) { + partial_refs.push_back( + make_uniq(top_join.types[top_idx], top_idx)); + } + } + } + } + + if (partial_refs.size() < 2) return changed; // need at least 2 to combine + + // Fold into list_add_vv(list_add_vv(...), last) + unique_ptr combined = std::move(partial_refs[0]); + for (idx_t i = 1; i < partial_refs.size(); ++i) + combined = BuildListAddVVExpr(input.context, std::move(combined), std::move(partial_refs[i])); + + *mat_mul_slot = std::move(combined); + return true; +} + +} // namespace duckdb + +// --- plugin self-registration ------------------------------------------------ +REGISTER_OPT_PASS(greedy_factorization, + &duckdb::MLGreedyFactorizationOptimizer::check, + &duckdb::MLGreedyFactorizationOptimizer::apply); diff --git a/expedia.json b/expedia.json index cce837e..b53a797 100644 --- a/expedia.json +++ b/expedia.json @@ -2,13 +2,13 @@ "total_bytes_written": 0, "total_bytes_read": 0, "rows_returned": 79756, - "latency": 0.619100106, + "latency": 0.594917065, "result_set_size": 2871216, - "query_name": "SELECT\n s.prop_id,\n s.srch_id,\n tree_predict(\n list_transform(\n list_concat(\n /* 1) NUMERICALS — per-column scaling (exact order) */\n min_max_scaler(list_value(CAST(s.prop_location_score1 AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'),\n min_max_scaler(list_value(CAST(s.prop_location_score2 AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'),\n min_max_scaler(list_value(CAST(s.prop_log_historical_price AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'),\n min_max_scaler(list_value(CAST(s.price_usd AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'),\n min_max_scaler(list_value(CAST(s.orig_destination_distance AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'),\n min_max_scaler(list_value(CAST(h.prop_review_score AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'),\n min_max_scaler(list_value(CAST(h.avg_bookings_usd AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'),\n min_max_scaler(list_value(CAST(h.stdev_bookings_usd AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'),\n\n /* 2) CATEGORICALS — one-hot, exact order of your Python */\n -- stringEncoderSpecs (string→int): TRUE\n one_hot_encode(CAST(s.position AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', TRUE),\n one_hot_encode(CAST(h.prop_country_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', TRUE),\n\n -- encoderSpecs (int→int): FALSE\n one_hot_encode(CAST(h.prop_starrating AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', FALSE),\n one_hot_encode(CAST(h.prop_brand_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', FALSE),\n one_hot_encode(CAST(h.count_clicks AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', FALSE),\n one_hot_encode(CAST(h.count_bookings AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', FALSE),\n\n -- stringEncoderSpecs (string→int): TRUE\n one_hot_encode(CAST(r.year AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', TRUE),\n one_hot_encode(CAST(r.month AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', TRUE),\n one_hot_encode(CAST(r.weekofyear AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', TRUE),\n one_hot_encode(CAST(r.time AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', TRUE),\n one_hot_encode(CAST(r.site_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', TRUE),\n one_hot_encode(CAST(r.visitor_location_country_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', TRUE),\n one_hot_encode(CAST(r.srch_destination_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', TRUE),\n\n -- encoderSpecs (int→int): FALSE\n one_hot_encode(CAST(r.srch_length_of_stay AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', FALSE),\n one_hot_encode(CAST(r.srch_booking_window AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', FALSE),\n one_hot_encode(CAST(r.srch_adults_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_children_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_room_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_saturday_night_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', FALSE),\n one_hot_encode(CAST(r.random_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', FALSE)\n ),\n x -> CAST(x AS FLOAT) -- tree_predict expects LIST\n ),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt',\n FALSE\n )\nFROM Expedia_S_listings_extension2 AS s\nJOIN Expedia_R1_hotels2 AS h ON s.prop_id = h.prop_id\nJOIN Expedia_R2_searches AS r ON s.srch_id = r.srch_id\nWHERE s.prop_location_score1 > 1\n AND s.prop_location_score2 > 0.1\n AND s.prop_log_historical_price > 4\n AND h.count_bookings > 5\n AND r.srch_booking_window > 10\n AND r.srch_length_of_stay > 1;", + "query_name": "SELECT\n s.prop_id,\n s.srch_id,\n tree_predict(\n list_transform(\n list_concat(\n /* 1) NUMERICALS — per-column scaling (exact order) */\n min_max_scaler(list_value(CAST(s.prop_location_score1 AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'),\n min_max_scaler(list_value(CAST(s.prop_location_score2 AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'),\n min_max_scaler(list_value(CAST(s.prop_log_historical_price AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'),\n min_max_scaler(list_value(CAST(s.price_usd AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'),\n min_max_scaler(list_value(CAST(s.orig_destination_distance AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'),\n min_max_scaler(list_value(CAST(h.prop_review_score AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'),\n min_max_scaler(list_value(CAST(h.avg_bookings_usd AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'),\n min_max_scaler(list_value(CAST(h.stdev_bookings_usd AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'),\n\n /* 2) CATEGORICALS — one-hot, exact order of your Python */\n -- stringEncoderSpecs (string→int): TRUE\n one_hot_encode(CAST(s.position AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', TRUE),\n one_hot_encode(CAST(h.prop_country_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', TRUE),\n\n -- encoderSpecs (int→int): FALSE\n one_hot_encode(CAST(h.prop_starrating AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', FALSE),\n one_hot_encode(CAST(h.prop_brand_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', FALSE),\n one_hot_encode(CAST(h.count_clicks AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', FALSE),\n one_hot_encode(CAST(h.count_bookings AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', FALSE),\n\n -- stringEncoderSpecs (string→int): TRUE\n one_hot_encode(CAST(r.year AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', TRUE),\n one_hot_encode(CAST(r.month AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', TRUE),\n one_hot_encode(CAST(r.weekofyear AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', TRUE),\n one_hot_encode(CAST(r.time AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', TRUE),\n one_hot_encode(CAST(r.site_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', TRUE),\n one_hot_encode(CAST(r.visitor_location_country_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', TRUE),\n one_hot_encode(CAST(r.srch_destination_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', TRUE),\n\n -- encoderSpecs (int→int): FALSE\n one_hot_encode(CAST(r.srch_length_of_stay AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', FALSE),\n one_hot_encode(CAST(r.srch_booking_window AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', FALSE),\n one_hot_encode(CAST(r.srch_adults_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_children_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_room_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_saturday_night_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', FALSE),\n one_hot_encode(CAST(r.random_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', FALSE)\n ),\n x -> CAST(x AS FLOAT) -- tree_predict expects LIST\n ),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt',\n FALSE\n )\nFROM Expedia_S_listings_extension2 AS s\nJOIN Expedia_R1_hotels2 AS h ON s.prop_id = h.prop_id\nJOIN Expedia_R2_searches AS r ON s.srch_id = r.srch_id\nWHERE s.prop_location_score1 > 1\n AND s.prop_location_score2 > 0.1\n AND s.prop_log_historical_price > 4\n AND h.count_bookings > 5\n AND r.srch_booking_window > 10\n AND r.srch_length_of_stay > 1;", "blocked_thread_time": 0.0, - "system_peak_buffer_memory": 41269248, + "system_peak_buffer_memory": 41287680, "system_peak_temp_dir_size": 0, - "cpu_time": 4.15678951, + "cpu_time": 4.310444025000001, "extra_info": {}, "cumulative_cardinality": 637172, "cumulative_rows_scanned": 7586096, @@ -18,12 +18,12 @@ "total_bytes_read": 0, "result_set_size": 2871216, "operator_name": "PROJECTION", - "cpu_time": 4.15678951, + "cpu_time": 4.310444025000001, "extra_info": { "Projections": [ "prop_id", "srch_id", - "tree_predict(list_transform(list_concat(min_max_scaler(list_value(prop_location_score1), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'), min_max_scaler(list_value(prop_location_score2), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'), min_max_scaler(list_value(prop_log_historical_price), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'), min_max_scaler(list_value(price_usd), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'), min_max_scaler(list_value(orig_destination_distance), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'), min_max_scaler(list_value(CAST(prop_review_score AS DOUBLE)), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'), min_max_scaler(list_value(avg_bookings_usd), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'), min_max_scaler(list_value(stdev_bookings_usd), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'), one_hot_encode(position, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', true), one_hot_encode(prop_country_id, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', true), one_hot_encode(CAST(prop_starrating AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', false), one_hot_encode(CAST(prop_brand_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', false), one_hot_encode(CAST(count_clicks AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', false), one_hot_encode(CAST(count_bookings AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', false), one_hot_encode(year, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', true), one_hot_encode(month, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', true), one_hot_encode(weekofyear, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', true), one_hot_encode(time, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', true), one_hot_encode(site_id, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', true), one_hot_encode(visitor_location_country_id, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', true), one_hot_encode(srch_destination_id, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', true), one_hot_encode(CAST(srch_length_of_stay AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', false), one_hot_encode(CAST(srch_booking_window AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', false), one_hot_encode(CAST(srch_adults_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', false), one_hot_encode(CAST(srch_children_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', false), one_hot_encode(CAST(srch_room_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', false), one_hot_encode(CAST(srch_saturday_night_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', false), one_hot_encode(CAST(random_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', false))), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt', false)" + "tree_predict(list_transform(list_concat(min_max_scaler(list_value(prop_location_score1), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'), min_max_scaler(list_value(prop_location_score2), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'), min_max_scaler(list_value(prop_log_historical_price), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'), min_max_scaler(list_value(price_usd), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'), min_max_scaler(list_value(orig_destination_distance), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'), min_max_scaler(list_value(CAST(prop_review_score AS DOUBLE)), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'), min_max_scaler(list_value(avg_bookings_usd), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'), min_max_scaler(list_value(stdev_bookings_usd), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'), one_hot_encode(position, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', true), one_hot_encode(prop_country_id, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', true), one_hot_encode(CAST(prop_starrating AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', false), one_hot_encode(CAST(prop_brand_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', false), one_hot_encode(CAST(count_clicks AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', false), one_hot_encode(CAST(count_bookings AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', false), one_hot_encode(year, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', true), one_hot_encode(month, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', true), one_hot_encode(weekofyear, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', true), one_hot_encode(time, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', true), one_hot_encode(site_id, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', true), one_hot_encode(visitor_location_country_id, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', true), one_hot_encode(srch_destination_id, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', true), one_hot_encode(CAST(srch_length_of_stay AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', false), one_hot_encode(CAST(srch_booking_window AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', false), one_hot_encode(CAST(srch_adults_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', false), one_hot_encode(CAST(srch_children_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', false), one_hot_encode(CAST(srch_room_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', false), one_hot_encode(CAST(srch_saturday_night_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', false), one_hot_encode(CAST(random_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', false))), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt', false)" ], "Estimated Cardinality": "7220" }, @@ -32,14 +32,14 @@ "operator_cardinality": 79756, "cumulative_rows_scanned": 7586096, "operator_rows_scanned": 0, - "operator_timing": 3.9706238020000004, + "operator_timing": 4.160415685, "children": [ { "total_bytes_written": 0, "total_bytes_read": 0, "result_set_size": 26798016, "operator_name": "HASH_JOIN", - "cpu_time": 0.18616570799999999, + "cpu_time": 0.15002833999999998, "extra_info": { "Join Type": "INNER", "Conditions": "srch_id = srch_id", @@ -50,14 +50,14 @@ "operator_cardinality": 79756, "cumulative_rows_scanned": 7586096, "operator_rows_scanned": 0, - "operator_timing": 0.027968030999999997, + "operator_timing": 0.023002200999999996, "children": [ { "total_bytes_written": 0, "total_bytes_read": 0, "result_set_size": 35619696, "operator_name": "HASH_JOIN", - "cpu_time": 0.153371983, + "cpu_time": 0.12518312399999998, "extra_info": { "Join Type": "INNER", "Conditions": "prop_id = prop_id", @@ -68,14 +68,14 @@ "operator_cardinality": 212022, "cumulative_rows_scanned": 7549075, "operator_rows_scanned": 0, - "operator_timing": 0.040581439999999996, + "operator_timing": 0.045933729000000006, "children": [ { "total_bytes_written": 0, "total_bytes_read": 0, "result_set_size": 21772608, "operator_name": "SEQ_SCAN ", - "cpu_time": 0.11156367, + "cpu_time": 0.07825701199999997, "extra_info": { "Table": "Expedia_S_listings_extension2", "Type": "Sequential Scan", @@ -101,7 +101,7 @@ "operator_cardinality": 247416, "cumulative_rows_scanned": 7537136, "operator_rows_scanned": 7537136, - "operator_timing": 0.11156367, + "operator_timing": 0.07825701199999997, "children": [] }, { @@ -109,7 +109,7 @@ "total_bytes_read": 0, "result_set_size": 688320, "operator_name": "SEQ_SCAN ", - "cpu_time": 0.0012268730000000003, + "cpu_time": 0.000992383, "extra_info": { "Table": "Expedia_R1_hotels2", "Type": "Sequential Scan", @@ -132,7 +132,7 @@ "operator_cardinality": 7170, "cumulative_rows_scanned": 11939, "operator_rows_scanned": 11939, - "operator_timing": 0.0012268730000000003, + "operator_timing": 0.000992383, "children": [] } ] @@ -142,7 +142,7 @@ "total_bytes_read": 0, "result_set_size": 2033568, "operator_name": "SEQ_SCAN ", - "cpu_time": 0.004825693999999999, + "cpu_time": 0.001843015, "extra_info": { "Table": "Expedia_R2_searches", "Type": "Sequential Scan", @@ -174,7 +174,7 @@ "operator_cardinality": 11052, "cumulative_rows_scanned": 37021, "operator_rows_scanned": 37021, - "operator_timing": 0.004825693999999999, + "operator_timing": 0.001843015, "children": [] } ] diff --git a/plugins/opt_passes/.gitkeep b/plugins/opt_passes/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/plugins/opt_passes/_manifest.cpp b/plugins/opt_passes/_manifest.cpp new file mode 100644 index 0000000..8bcdb47 --- /dev/null +++ b/plugins/opt_passes/_manifest.cpp @@ -0,0 +1,13 @@ +// AUTO-GENERATED by the frontend upload flow — do not edit by hand. +#include + +extern "C" void optpass_anchor_greedy_factorization(); + +namespace duckdb { + +void LinkOptPassPlugins() { + static std::vector keep; + keep.push_back(reinterpret_cast(&optpass_anchor_greedy_factorization)); +} + +} // namespace duckdb diff --git a/plugins/opt_passes/include/.gitkeep b/plugins/opt_passes/include/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/queries/expedia_query.sql b/queries/expedia_query.sql index 0918b3c..7bdfd25 100644 --- a/queries/expedia_query.sql +++ b/queries/expedia_query.sql @@ -1,12 +1,12 @@ -- Attach the database file and use its schema -ATTACH '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/data/expedia_imbridge2.db' +ATTACH '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/data/expedia_imbridge2.db' AS expedia_db (READ_ONLY); SET schema 'expedia_db.main'; -- Optimizer Config PRAGMA enable_profiling; SET enable_profiling = 'json'; -SET profiling_output = '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/expedia.json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/expedia.json'; -- PRAGMA profiling_output='/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/plans/profile.json'; -- PRAGMA disable_optimizer; -- PRAGMA explain_output='json'; @@ -29,54 +29,54 @@ SELECT list_concat( /* 1) NUMERICALS — per-column scaling (exact order) */ min_max_scaler(list_value(CAST(s.prop_location_score1 AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'), min_max_scaler(list_value(CAST(s.prop_location_score2 AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'), min_max_scaler(list_value(CAST(s.prop_log_historical_price AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'), min_max_scaler(list_value(CAST(s.price_usd AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'), min_max_scaler(list_value(CAST(s.orig_destination_distance AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'), min_max_scaler(list_value(CAST(h.prop_review_score AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'), min_max_scaler(list_value(CAST(h.avg_bookings_usd AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'), min_max_scaler(list_value(CAST(h.stdev_bookings_usd AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'), /* 2) CATEGORICALS — one-hot, exact order of your Python */ -- stringEncoderSpecs (string→int): TRUE - one_hot_encode(CAST(s.position AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', TRUE), - one_hot_encode(CAST(h.prop_country_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', TRUE), + one_hot_encode(CAST(s.position AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', TRUE), + one_hot_encode(CAST(h.prop_country_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', TRUE), -- encoderSpecs (int→int): FALSE - one_hot_encode(CAST(h.prop_starrating AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', FALSE), - one_hot_encode(CAST(h.prop_brand_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', FALSE), - one_hot_encode(CAST(h.count_clicks AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', FALSE), - one_hot_encode(CAST(h.count_bookings AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', FALSE), + one_hot_encode(CAST(h.prop_starrating AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', FALSE), + one_hot_encode(CAST(h.prop_brand_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', FALSE), + one_hot_encode(CAST(h.count_clicks AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', FALSE), + one_hot_encode(CAST(h.count_bookings AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', FALSE), -- stringEncoderSpecs (string→int): TRUE - one_hot_encode(CAST(r.year AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', TRUE), - one_hot_encode(CAST(r.month AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', TRUE), - one_hot_encode(CAST(r.weekofyear AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', TRUE), - one_hot_encode(CAST(r.time AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', TRUE), - one_hot_encode(CAST(r.site_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', TRUE), - one_hot_encode(CAST(r.visitor_location_country_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', TRUE), - one_hot_encode(CAST(r.srch_destination_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', TRUE), + one_hot_encode(CAST(r.year AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', TRUE), + one_hot_encode(CAST(r.month AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', TRUE), + one_hot_encode(CAST(r.weekofyear AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', TRUE), + one_hot_encode(CAST(r.time AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', TRUE), + one_hot_encode(CAST(r.site_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', TRUE), + one_hot_encode(CAST(r.visitor_location_country_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', TRUE), + one_hot_encode(CAST(r.srch_destination_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', TRUE), -- encoderSpecs (int→int): FALSE - one_hot_encode(CAST(r.srch_length_of_stay AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', FALSE), - one_hot_encode(CAST(r.srch_booking_window AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', FALSE), - one_hot_encode(CAST(r.srch_adults_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', FALSE), - one_hot_encode(CAST(r.srch_children_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', FALSE), - one_hot_encode(CAST(r.srch_room_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', FALSE), - one_hot_encode(CAST(r.srch_saturday_night_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', FALSE), - one_hot_encode(CAST(r.random_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', FALSE) + one_hot_encode(CAST(r.srch_length_of_stay AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', FALSE), + one_hot_encode(CAST(r.srch_booking_window AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', FALSE), + one_hot_encode(CAST(r.srch_adults_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', FALSE), + one_hot_encode(CAST(r.srch_children_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', FALSE), + one_hot_encode(CAST(r.srch_room_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', FALSE), + one_hot_encode(CAST(r.srch_saturday_night_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', FALSE), + one_hot_encode(CAST(r.random_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', FALSE) ), x -> CAST(x AS FLOAT) -- tree_predict expects LIST ), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt', + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt', FALSE ) FROM Expedia_S_listings_extension2 AS s diff --git a/queries/flights_query.sql b/queries/flights_query.sql new file mode 100644 index 0000000..c85225e --- /dev/null +++ b/queries/flights_query.sql @@ -0,0 +1,90 @@ +-- Q_Flights: route attribute prediction (codeshare) via decision forest +-- Joins routes + airlines + source/destination airports, then runs forest inference. +-- Run setup_flights_db_local.py once to create flights_imbridge2.db first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/data/flights_imbridge2.db' + AS flights_db (READ_ONLY); +SET schema 'flights_db.main'; + +PRAGMA enable_profiling; +SET enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/flights.json'; + +LOAD 'cactusdb'; +SET threads = 8; + +SELECT + r.airlineid, + r.sairportid, + r.dairportid, + decision_forest_predict( + list_transform( + list_concat( + /* 1) NUMERICALS — per-column min-max scaling (exact training order) */ + min_max_scaler(list_value(CAST(s.slatitude AS DOUBLE)), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/scaler_files/slatitude.txt'), + min_max_scaler(list_value(CAST(s.slongitude AS DOUBLE)), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/scaler_files/slongitude.txt'), + min_max_scaler(list_value(CAST(d.dlatitude AS DOUBLE)), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/scaler_files/dlatitude.txt'), + min_max_scaler(list_value(CAST(d.dlongitude AS DOUBLE)), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/scaler_files/dlongitude.txt'), + + /* 2) CATEGORICALS — one-hot encoding (exact training order) */ + -- name1: integer-valued airline attribute (is_string=FALSE) + one_hot_encode(CAST(a.name1 AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/name1.csv', + FALSE), + -- name2, name4, active: 't'/'f' booleans (is_string=TRUE) + one_hot_encode(CAST(a.name2 AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/name2.csv', + TRUE), + one_hot_encode(CAST(a.name4 AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/name4.csv', + TRUE), + one_hot_encode(CAST(a.acountry AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/acountry.csv', + TRUE), + one_hot_encode(CAST(a.active AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/active.csv', + TRUE), + one_hot_encode(CAST(s.scity AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/scity.csv', + TRUE), + one_hot_encode(CAST(s.scountry AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/scountry.csv', + TRUE), + -- stimezone: integer timezone offset (is_string=FALSE) + one_hot_encode(CAST(s.stimezone AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/stimezone.csv', + FALSE), + one_hot_encode(CAST(s.sdst AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/sdst.csv', + TRUE), + one_hot_encode(CAST(d.dcity AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/dcity.csv', + TRUE), + one_hot_encode(CAST(d.dcountry AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/dcountry.csv', + TRUE), + -- dtimezone: integer timezone offset (is_string=FALSE) + one_hot_encode(CAST(d.dtimezone AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/dtimezone.csv', + FALSE), + one_hot_encode(CAST(d.ddst AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/ddst.csv', + TRUE) + ), + x -> CAST(x AS FLOAT) -- decision_forest_predict expects LIST + ), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_dot_trees_custom', + 6756, -- total features: 4 numerical + 6752 one-hot categorical + TRUE -- classification: predicting codeshare (binary) + ) AS codeshare +FROM Flights_S_routes_extension2 AS r +JOIN Flights_R1_airlines2 AS a ON r.airlineid = a.airlineid +JOIN Flights_R2_sairports AS s ON r.sairportid = s.sairportid +JOIN Flights_R3_dairports AS d ON r.dairportid = d.dairportid +WHERE a.name2 = 't' + AND a.name4 = 't' + AND a.name1 > 2.8; diff --git a/queries/forest_narrow_baseline.sql b/queries/forest_narrow_baseline.sql new file mode 100644 index 0000000..d0345a0 --- /dev/null +++ b/queries/forest_narrow_baseline.sql @@ -0,0 +1,26 @@ +LOAD cactusdb; +SET threads = 8; +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; +PRAGMA enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/plans/forest_narrow_baseline.json'; + +WITH feature_rows AS ( + SELECT + id, + list(CAST(i AS FLOAT)) AS features + FROM ( + SELECT * FROM range(0, 500) AS ids(id), + range(0, 20) AS r(i) + ) + GROUP BY id +) +SELECT + id, + decision_forest_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_narrow_1600', + 20, + TRUE + ) AS prediction +FROM feature_rows +ORDER BY id; diff --git a/queries/forest_narrow_optimized.sql b/queries/forest_narrow_optimized.sql new file mode 100644 index 0000000..03f358e --- /dev/null +++ b/queries/forest_narrow_optimized.sql @@ -0,0 +1,27 @@ +LOAD cactusdb; +SET threads = 8; +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; +SET enable_extended_optimizer = 'RULE##DecisionForestUDF2RelationRewriteAction'; +PRAGMA enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/plans/forest_narrow_optimized.json'; + +WITH feature_rows AS ( + SELECT + id, + list(CAST(i AS FLOAT)) AS features + FROM ( + SELECT * FROM range(0, 500) AS ids(id), + range(0, 20) AS r(i) + ) + GROUP BY id +) +SELECT + id, + decision_forest_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_narrow_1600', + 20, + TRUE + ) AS prediction +FROM feature_rows +ORDER BY id; diff --git a/queries/forest_query.sql b/queries/forest_query.sql index a54da06..2334973 100644 --- a/queries/forest_query.sql +++ b/queries/forest_query.sql @@ -2,8 +2,10 @@ -Load cactusdb; +LOAD cactusdb; +SET enable_extended_optimizer = 'RULE##DecisionForestUDF2RelationRewriteAction'; +EXPLAIN -- Build 3 rows of 6756-dimensional feature vectors WITH feature_rows AS ( SELECT @@ -22,7 +24,7 @@ SELECT features, decision_forest_predict( features, - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_dot_trees_custom', + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_dot_trees_custom', 6756, TRUE ) diff --git a/queries/forest_stress_baseline.sql b/queries/forest_stress_baseline.sql new file mode 100644 index 0000000..94968a5 --- /dev/null +++ b/queries/forest_stress_baseline.sql @@ -0,0 +1,26 @@ +LOAD cactusdb; +SET threads = 8; +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; +PRAGMA enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/plans/forest_stress_baseline.json'; + +WITH feature_rows AS ( + SELECT + id, + list(CAST(i AS FLOAT)) AS features + FROM ( + SELECT * FROM range(0, 500) AS ids(id), + range(0, 6756) AS r(i) + ) + GROUP BY id +) +SELECT + id, + decision_forest_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_stress_1600', + 6756, + TRUE + ) AS prediction +FROM feature_rows +ORDER BY id; diff --git a/queries/forest_stress_optimized.sql b/queries/forest_stress_optimized.sql new file mode 100644 index 0000000..8638eef --- /dev/null +++ b/queries/forest_stress_optimized.sql @@ -0,0 +1,27 @@ +LOAD cactusdb; +SET threads = 8; +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; +SET enable_extended_optimizer = 'RULE##DecisionForestUDF2RelationRewriteAction'; +PRAGMA enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/plans/forest_stress_optimized.json'; + +WITH feature_rows AS ( + SELECT + id, + list(CAST(i AS FLOAT)) AS features + FROM ( + SELECT * FROM range(0, 500) AS ids(id), + range(0, 6756) AS r(i) + ) + GROUP BY id +) +SELECT + id, + decision_forest_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_stress_1600', + 6756, + TRUE + ) AS prediction +FROM feature_rows +ORDER BY id; diff --git a/queries/uc01_query.sql b/queries/uc01_query.sql new file mode 100644 index 0000000..71bbe1b --- /dev/null +++ b/queries/uc01_query.sql @@ -0,0 +1,74 @@ +-- Q_UC01: KMeans++ customer-segmentation inference with min-max scaling. +-- Features: avg orders/year (frequency) + avg return ratio per customer. +-- Run resources/uc01/setup_uc01.py once to create all artifacts first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc01/uc01.db' + AS uc01_db (READ_ONLY); +SET schema 'uc01_db.main'; + +LOAD 'cactusdb'; + +WITH table_groups AS ( + -- Join orders + line items + returns, group per (customer, order) + SELECT + MIN(EXTRACT(YEAR FROM CAST(date AS DATE))) AS invoice_year, + SUM(quantity * price) AS row_price, + SUM(COALESCE(or_return_quantity, 0) * price) AS return_row_price, + o_customer_sk AS customer_id, + o_order_id AS order_id + FROM lineitem li + LEFT JOIN order_returns oret + ON li.li_order_id = oret.or_order_id + AND li.li_product_id = oret.or_product_id + JOIN "order" ord ON li.li_order_id = ord.o_order_id + GROUP BY o_customer_sk, o_order_id +), +table_ratios AS ( + SELECT + customer_id, + AVG(return_row_price / NULLIF(row_price, 0)) AS return_ratio + FROM table_groups + GROUP BY customer_id +), +table_frequency_groups AS ( + SELECT + customer_id, + invoice_year, + COUNT(DISTINCT order_id) AS orders_per_year + FROM table_groups + GROUP BY customer_id, invoice_year +), +table_frequency AS ( + SELECT + customer_id, + AVG(orders_per_year) AS frequency + FROM table_frequency_groups + GROUP BY customer_id +), +features_raw AS ( + SELECT + r.customer_id, + CAST(f.frequency AS DOUBLE) AS frequency, + CAST(r.return_ratio AS DOUBLE) AS return_ratio + FROM table_ratios r + JOIN table_frequency f ON r.customer_id = f.customer_id +), +features AS ( + SELECT + customer_id, + -- min-max scale [frequency, return_ratio] using the fitted scaler + min_max_scaler( + list_value(frequency, return_ratio), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc01/uc01_scaler.csv' + ) AS features + FROM features_raw +) +SELECT + customer_id, + -- assign to nearest KMeans centroid (0-based cluster index) + kmeans_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc01/uc01_centroids.csv' + ) AS cluster_id +FROM features +ORDER BY customer_id; diff --git a/queries/uc03_query.sql b/queries/uc03_query.sql new file mode 100644 index 0000000..50c2e41 --- /dev/null +++ b/queries/uc03_query.sql @@ -0,0 +1,75 @@ +-- Q_UC03: DNN inference over encoded store-department features +-- Architecture: Input(80) --ReLU--> 64 --ReLU--> 32 --Sigmoid--> 1 +-- Run resources/uc03/setup_uc03.py once to create all artifacts first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/store_dept.db' + AS uc03_db (READ_ONLY); +SET schema 'uc03_db.main'; + +LOAD 'cactusdb'; + +WITH base AS ( + SELECT + store, + department, + num_of_week, + one_hot_encode( + CAST(store AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/encoders/store_encoder.csv', + FALSE + ) AS store_encoded, + one_hot_encode( + department, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/encoders/dept_encoder.csv', + TRUE + ) AS dept_encoded, + list_value(CAST(num_of_week / 156.0 AS DOUBLE)) AS week_norm + FROM store_dept +), +feat AS ( + SELECT + store, + department, + num_of_week, + list_concat(store_encoded, dept_encoded, week_norm) AS features + FROM base +) +SELECT + store, + department, + num_of_week, + -- Layer 1: features -> mat_mul(W1) -> mat_add(b1) -> relu [80 -> 64] + -- Layer 2: h1 -> mat_mul(W2) -> mat_add(b2) -> relu [64 -> 32] + -- Layer 3: h2 -> mat_mul(W3) -> mat_add(b3) -> sigmoid -> scalar [32 -> 1] + list_extract( + sigmoid( + mat_add( + mat_mul( + relu( + mat_add( + mat_mul( + relu( + mat_add( + mat_mul( + features, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_W1.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_b1.csv' + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_W2.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_b2.csv' + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_W3.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_b3.csv' + ) + ), + 1 + ) AS prediction +FROM feat; diff --git a/queries/uc04_query.sql b/queries/uc04_query.sql new file mode 100644 index 0000000..3e4e855 --- /dev/null +++ b/queries/uc04_query.sql @@ -0,0 +1,46 @@ +-- Q_UC04: Naive Bayes spam inference over tokenized reviews. +-- Pipeline: clean_text -> tokenize_text -> stem_token -> log-sum scoring +-- Run resources/uc04/setup_uc04.py once to create all artifacts first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc04/uc04.db' + AS uc04_db (READ_ONLY); +SET schema 'uc04_db.main'; + +LOAD 'cactusdb'; + +WITH tokens AS ( + -- UDFs: clean (strip non-alpha) -> tokenize (split) -> stem (suffix strip) + SELECT + r.id, + stem_token(tok) AS token + FROM review r, + UNNEST(tokenize_text(clean_text(r.review_text))) AS t(tok) + WHERE tok <> '' +), +token_scores AS ( + -- Accumulate log-likelihood for each class per document + SELECT + t.id, + SUM(LN(COALESCE(m.spam_prob, 1e-9))) AS log_spam_score, + SUM(LN(COALESCE(m.non_spam_prob, 1e-9))) AS log_non_spam_score + FROM tokens t + LEFT JOIN uc04_model m ON m.token = t.token + GROUP BY t.id +), +base_priors AS ( + -- Class priors computed from labeled training set + SELECT + LN(SUM(CASE WHEN spam = 1 THEN 1 ELSE 0 END) * 1.0 / COUNT(*)) AS log_spam_prior, + LN(SUM(CASE WHEN spam = 0 THEN 1 ELSE 0 END) * 1.0 / COUNT(*)) AS log_non_spam_prior + FROM uc04_train_preprocessed +) +SELECT + s.id, + CASE + WHEN s.log_spam_score + p.log_spam_prior + > s.log_non_spam_score + p.log_non_spam_prior + THEN 1 ELSE 0 + END AS predicted_spam +FROM token_scores s +CROSS JOIN base_priors p +ORDER BY s.id; diff --git a/queries/uc08_query.sql b/queries/uc08_query.sql new file mode 100644 index 0000000..163f325 --- /dev/null +++ b/queries/uc08_query.sql @@ -0,0 +1,97 @@ +-- Q_UC08: DNN inference over order-lineitem-product features; softmax output. +-- Architecture: Input(4) --ReLU--> 64 --ReLU--> 128 --Softmax--> 384 +-- features = [quantity, scan_count, weekday, dept_label] +-- Run resources/uc08/setup_uc08.py once to create all artifacts first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/uc08.db' + AS uc08_db (READ_ONLY); +SET schema 'uc08_db.main'; + +LOAD 'cactusdb'; + +WITH ord AS ( + SELECT o_order_id, CAST(date AS TIMESTAMP) AS ts + FROM "order" +), +ord2 AS ( + SELECT o_order_id, ts, dayofweek(ts) AS weekday + FROM ord +), +j1 AS ( + SELECT + o.o_order_id, o.ts, o.weekday, + li.li_product_id, li.quantity + FROM ord2 o + JOIN lineitem li ON o.o_order_id = li.li_order_id +), +j2 AS ( + SELECT + j1.o_order_id, j1.ts, j1.weekday, + j1.quantity, p.department + FROM j1 + JOIN product p ON j1.li_product_id = p.p_product_id +), +agg AS ( + SELECT + o_order_id, + ts AS date, + department, + quantity, + SUM(quantity) AS scan_count, + MIN(weekday) AS weekday + FROM j2 + GROUP BY o_order_id, ts, department, quantity +), +feat AS ( + SELECT + o_order_id, + date, + -- 4-dim feature: [quantity, scan_count, weekday, dept_label] + -- dept_label via argmax of one-hot encoding = integer index 0..46 + list_value( + CAST(quantity AS DOUBLE), + CAST(scan_count AS DOUBLE), + CAST(weekday AS DOUBLE), + CAST(ml_argmax(one_hot_encode( + department, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/encoders/dept_label_encoder.csv', + TRUE + )) AS DOUBLE) + ) AS features + FROM agg +) +SELECT + o_order_id, + date, + -- Layer 1: features -> mat_mul(W1) -> mat_add(b1) -> relu [4 -> 64] + -- Layer 2: h1 -> mat_mul(W2) -> mat_add(b2) -> relu [64 -> 128] + -- Layer 3: h2 -> mat_mul(W3) -> mat_add(b3) -> softmax [128 -> 384] + cdb_softmax( + mat_add( + mat_mul( + relu( + mat_add( + mat_mul( + relu( + mat_add( + mat_mul( + features, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_W1.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_b1.csv' + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_W2.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_b2.csv' + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_W3.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_b3.csv' + ) + ) AS prediction +FROM feat; diff --git a/queries/uc08_query_factorizable.sql b/queries/uc08_query_factorizable.sql new file mode 100644 index 0000000..ffc3304 --- /dev/null +++ b/queries/uc08_query_factorizable.sql @@ -0,0 +1,63 @@ +-- Q_UC08 (factorizable form) +-- Single-layer: mat_mul(list_concat(LEFT_3, RIGHT_47), W1_wide[50x64]) +-- Right join side uses the full 47-dim one-hot encoding of department. +-- +-- With enable_extended_optimizer='4', the factorization action will: +-- 1. Detect mat_mul(list_concat(LEFT_3, RIGHT_47), W1_wide) over the COMPARISON_JOIN. +-- 2. Split W1_wide into W1_left (3x64) and W1_right (47x64). +-- 3. Push mat_mul(left_features, W1_left) into a projection on the left side (2.1M rows). +-- 4. Push mat_mul(right_features, W1_right) into a projection on the right side (707 rows). +-- 5. Replace the original mat_mul with list_add_vv(#left_col, #right_col). +-- +-- Speedup: mat_mul(47-dim, W1_right[47x64]) runs 707 times instead of 2.1M times. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/uc08.db' + AS uc08_db (READ_ONLY); +SET schema 'uc08_db.main'; + +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, + regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, + statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, + limit_pushdown, top_n, build_side_probe_side, compressed_materialization, + duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, + materialized_cte, sum_rewriter, late_materialization, cte_inlining'; + +SET enable_extended_optimizer = '4'; + +LOAD 'cactusdb'; + +SELECT + a.o_order_id, + a.date, + mat_mul( + list_concat( + -- LEFT_3: [quantity, scan_count, weekday] from order+lineitem agg + list_value( + CAST(a.quantity AS DOUBLE), + CAST(a.scan_count AS DOUBLE), + CAST(a.weekday AS DOUBLE) + ), + -- RIGHT_47: full one-hot dept vector from product (47-dim) + -- Factorization pushes this mat_mul to the product scan (707 rows). + one_hot_encode( + p.department, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/encoders/dept_label_encoder.csv', + TRUE + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_W1_wide.csv', + 'dense' + ) AS h1 +FROM ( + SELECT + o.o_order_id, + CAST(o.date AS DATE) AS date, + li.li_product_id, + li.quantity, + SUM(li.quantity) AS scan_count, + MIN(dayofweek(CAST(o.date AS TIMESTAMP))) AS weekday + FROM order_tbl o + JOIN lineitem li ON o.o_order_id = li.li_order_id + GROUP BY o.o_order_id, o.date, li.li_product_id, li.quantity +) a +JOIN product p ON a.li_product_id = p.p_product_id; diff --git a/src/cactusdb_extension.cpp b/src/cactusdb_extension.cpp index 186be78..0b562c4 100644 --- a/src/cactusdb_extension.cpp +++ b/src/cactusdb_extension.cpp @@ -2,6 +2,7 @@ #include "duckdb.hpp" #include "functions.hpp" #include "optimization/register_optimizers.hpp" +#include "optimization/optimizer_pass.hpp" #include "duckdb/main/database.hpp" // OpenSSL linked through vcpkg @@ -23,6 +24,8 @@ static void LoadInternal(ExtensionLoader &loader) { RegisterMLScalarFunctions(loader); RegisterCactusOptimizers(db); + RegisterOptimizerPassListFunction(loader); + LinkOptPassPlugins(); // pulls uploaded plugin objects into the static link } diff --git a/src/include/ml_functions/kmeans.hpp b/src/include/ml_functions/kmeans.hpp new file mode 100644 index 0000000..d59998a --- /dev/null +++ b/src/include/ml_functions/kmeans.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +#include "duckdb.hpp" +#include "duckdb/function/function.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +// kmeans_predict(feature_list LIST, centroids_file VARCHAR) -> INTEGER +// Loads a CSV of K centroids (K rows, D columns) at bind time and returns +// the 0-based index of the nearest centroid for each feature vector. +struct ML_KMeansPredict { + struct BindData final : public FunctionData { + std::vector> centroids; // [K][D] + idx_t K = 0; + idx_t D = 0; + std::string file_path; + + BindData() = default; + BindData(std::vector> c, idx_t k, idx_t d, std::string p) + : centroids(std::move(c)), K(k), D(d), file_path(std::move(p)) {} + + unique_ptr Copy() const override { + return make_uniq(centroids, K, D, file_path); + } + bool Equals(const FunctionData &o) const override { + return file_path == o.Cast().file_path; + } + }; + + static unique_ptr Bind(ClientContext &context, + ScalarFunction &bound_function, + vector> &args); + + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + + static void Register(ExtensionLoader &loader); +}; + +} // namespace duckdb diff --git a/src/include/ml_functions/one_hot_encoder.hpp b/src/include/ml_functions/one_hot_encoder.hpp index 17b8c8e..5ca2070 100644 --- a/src/include/ml_functions/one_hot_encoder.hpp +++ b/src/include/ml_functions/one_hot_encoder.hpp @@ -28,10 +28,14 @@ static void one_hot_encoder_bind_constructor( if (is_string) { while (std::getline(in, line)) { - std::istringstream ss(line); - std::string valStr, posStr; - if (!std::getline(ss, valStr, ',')) continue; - if (!std::getline(ss, posStr, ',')) continue; + // Split on the LAST comma so category names containing commas work. + auto sep = line.rfind(','); + if (sep == std::string::npos) continue; + std::string valStr = line.substr(0, sep); + std::string posStr = line.substr(sep + 1); + auto l = posStr.find_first_not_of(" \t\r\n"); + if (l == std::string::npos) continue; + posStr = posStr.substr(l); int position = std::stoi(posStr); str_map[valStr] = position; } diff --git a/src/include/ml_functions/relu.hpp b/src/include/ml_functions/relu.hpp new file mode 100644 index 0000000..84a431c --- /dev/null +++ b/src/include/ml_functions/relu.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include + +#include "duckdb.hpp" +#include "duckdb/function/function.hpp" + +namespace duckdb { + +struct ML_Relu { + static unique_ptr Bind(ClientContext &context, + ScalarFunction &bound_function, + vector> &args); + + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + + static void Register(ExtensionLoader &loader); +}; + +} // namespace duckdb diff --git a/src/include/ml_functions/text_functions.hpp b/src/include/ml_functions/text_functions.hpp new file mode 100644 index 0000000..969b765 --- /dev/null +++ b/src/include/ml_functions/text_functions.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/function/function.hpp" + +namespace duckdb { + +// clean_text(VARCHAR) -> VARCHAR +// Strips non-alphabetic characters and collapses whitespace. +struct ML_CleanText { + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + static void Register(ExtensionLoader &loader); +}; + +// tokenize_text(VARCHAR) -> LIST +// Splits a cleaned string on whitespace; returns only non-empty tokens. +struct ML_TokenizeText { + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + static void Register(ExtensionLoader &loader); +}; + +// stem_token(VARCHAR) -> VARCHAR +// Simple suffix-stripping stemmer (handles -ing, -ed, -er, -ion, -es, -s). +struct ML_StemToken { + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + static void Register(ExtensionLoader &loader); +}; + +} // namespace duckdb diff --git a/src/include/optimization/metric_catalog.hpp b/src/include/optimization/metric_catalog.hpp index c3e4443..3e1e2a5 100644 --- a/src/include/optimization/metric_catalog.hpp +++ b/src/include/optimization/metric_catalog.hpp @@ -90,31 +90,67 @@ class MetricCatalog { s.values["feature_width"] = 150; s.values["card_search_impressions"] = 500000; s.values["card_listing_metadata"] = 200000; + s.values["ml_flops"] = 280e6; // conservative default — below 5B threshold } void Initialize() { - // ---- Define metrics for each synthetic schema here ---- - // Example schema 1: synthetic dense features + // ---- Synthetic schemas (in-memory; GetQuerySchema returns just the schema name) ---- { SchemaMetricSet s; - s.values["join_ratio"] = 5e-5; // big joins - s.values["sparsity"] = 0.20; // 95% non-zero -> dense + s.values["join_ratio"] = 5e-5; + s.values["sparsity"] = 0.20; + s.values["ml_flops"] = 280e6; AddFrontendMetrics(s); schema_metrics["synthetic"] = std::move(s); } - - // Example schema 2: synthetic sparse features { SchemaMetricSet s; s.values["join_ratio"] = 5e-5; - s.values["sparsity"] = 0.10; // 10% non-zero -> sparse + s.values["sparsity"] = 0.10; + s.values["ml_flops"] = 280e6; AddFrontendMetrics(s); schema_metrics["syn_sparse"] = std::move(s); } - // Fallback schema: used by the dynamic rule evaluator when the query's - // resolved schema name is not found above (e.g. "main"). Carries the - // demo metrics under the frontend metric names. + // ---- UC08: order-feature DNN, 47-dim one-hot right side, fan-out=3001x ---- + // ml_flops = 6.79B → above 5B threshold → factorization candidate + { + SchemaMetricSet s; + s.values["join_ratio"] = 0.0003; + s.values["sparsity"] = 0.021; + s.values["ml_flops"] = 6.79e9; + s.values["join_cardinality_ratio"] = 0.0003; + schema_metrics["uc08_db.main"] = std::move(s); + } + + // ---- UC03: store-department DNN, low ml_flops ---- + { + SchemaMetricSet s; + s.values["join_ratio"] = 0.0; + s.values["sparsity"] = 0.025; + s.values["ml_flops"] = 280e6; + schema_metrics["uc03_db.main"] = std::move(s); + } + + // ---- Expedia: decision-tree inference, multi-join ---- + { + SchemaMetricSet s; + s.values["join_ratio"] = 1e-4; + s.values["sparsity"] = 0.15; + s.values["ml_flops"] = 1.2e9; + schema_metrics["expedia_db.main"] = std::move(s); + } + + // ---- Flights: decision-forest inference, multi-join ---- + { + SchemaMetricSet s; + s.values["join_ratio"] = 1e-4; + s.values["sparsity"] = 0.12; + s.values["ml_flops"] = 2.1e9; + schema_metrics["flights_db.main"] = std::move(s); + } + + // ---- Fallback: any schema not listed above ---- { SchemaMetricSet s; s.values["join_ratio"] = 5e-5; @@ -122,15 +158,6 @@ class MetricCatalog { AddFrontendMetrics(s); schema_metrics["_default"] = std::move(s); } - - // Add more schemas as needed: - // { - // SchemaMetricSet s; - // s.values["join_ratio"] = ...; - // s.values["sparsity"] = ...; - // s.values["some_other_metric"] = ...; - // schema_metrics["my_schema_name"] = std::move(s); - // } } private: diff --git a/src/include/optimization/optimizer_pass.hpp b/src/include/optimization/optimizer_pass.hpp new file mode 100644 index 0000000..4db3bff --- /dev/null +++ b/src/include/optimization/optimizer_pass.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/optimizer/optimizer_extension.hpp" + +#include +#include +#include + +namespace duckdb { + +class ExtensionLoader; + +// A self-contained optimizer pass uploaded as a plugin. The function-pointer +// signature deliberately matches the static check/apply methods every existing +// rewrite-action class already exposes (e.g. MLGreedyFactorizationOptimizer), so +// an existing pass becomes a plugin by adding a single REGISTER_OPT_PASS line. +struct OptimizerPass { + std::string name; + bool (*check)(OptimizerExtensionInput &, unique_ptr &); + bool (*apply)(OptimizerExtensionInput &, unique_ptr &); +}; + +// Process-wide registry of optimizer passes, populated by static initializers in +// compiled-in plugin sources (see REGISTER_OPT_PASS). Selected at query time via +// the "OPT#" enable_extended_optimizer setting. +class OptimizerPassRegistry { +public: + static OptimizerPassRegistry &Get(); // Meyers singleton + + void Register(OptimizerPass pass); // last-wins on duplicate name + const OptimizerPass *Find(const std::string &name) const; // nullptr if absent + std::vector Names() const; // registered names, sorted + +private: + std::unordered_map passes_; +}; + +// Registers the cactusdb_list_opt_passes() table function, which returns the +// names currently in OptimizerPassRegistry so the frontend can list the passes +// that are actually compiled in (single source of truth, no Python drift). +void RegisterOptimizerPassListFunction(ExtensionLoader &loader); + +// Called once at extension load. Its body lives in the AUTO-GENERATED +// plugins/opt_passes/_manifest.cpp, which references each plugin's anchor symbol +// (see REGISTER_OPT_PASS). That reference is what forces the static linker to +// retain each plugin object file — and therefore run its REGISTER_OPT_PASS +// initializer — when cactusdb is linked into the duckdb CLI as a static archive +// (object files with only static initializers are otherwise dropped). Always +// defined (the baseline manifest has an empty body). +void LinkOptPassPlugins(); + +} // namespace duckdb + +// Self-registration. Given a bare identifier IDENT (the pass name), this: +// 1. defines an extern "C" anchor symbol the generated manifest references +// (forces archive retention — see LinkOptPassPlugins), and +// 2. defines a file-scope object whose ctor registers the pass at load time, +// using the stringified IDENT as the registry name. +// Usage: +// REGISTER_OPT_PASS(greedy_factorization, &MyPass::check, &MyPass::apply); +#define REGISTER_OPT_PASS(IDENT, CHECK_FN, APPLY_FN) \ + extern "C" void optpass_anchor_##IDENT() {} \ + namespace { \ + struct OptPassReg_##IDENT { \ + OptPassReg_##IDENT() { \ + ::duckdb::OptimizerPassRegistry::Get().Register({#IDENT, (CHECK_FN), (APPLY_FN)}); \ + } \ + } g_optpassreg_##IDENT; \ + } diff --git a/src/ml_functions/CMakeLists.txt b/src/ml_functions/CMakeLists.txt index 2924345..d7fb609 100644 --- a/src/ml_functions/CMakeLists.txt +++ b/src/ml_functions/CMakeLists.txt @@ -9,9 +9,12 @@ set(ML_FUNCTIONS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/argmax.cpp ${CMAKE_CURRENT_SOURCE_DIR}/softmax.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sigmoid.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/relu.cpp ${CMAKE_CURRENT_SOURCE_DIR}/torch_dnn.cpp ${CMAKE_CURRENT_SOURCE_DIR}/decision_forest.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ml_decision_forest_table.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/text_functions.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/kmeans.cpp ) set(EXTENSION_SOURCES diff --git a/src/ml_functions/kmeans.cpp b/src/ml_functions/kmeans.cpp new file mode 100644 index 0000000..a990e24 --- /dev/null +++ b/src/ml_functions/kmeans.cpp @@ -0,0 +1,135 @@ +#include "ml_functions/kmeans.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/types/vector.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +// ─── Bind ───────────────────────────────────────────────────────────────────── + +unique_ptr ML_KMeansPredict::Bind(ClientContext &context, + ScalarFunction &, + vector> &args) { + if (args.size() != 2) { + throw BinderException("kmeans_predict: expects 2 arguments: features LIST, centroids_file VARCHAR"); + } + + auto v_path = ExpressionExecutor::EvaluateScalar(context, *args[1]); + if (v_path.IsNull()) throw BinderException("kmeans_predict: centroids_file is NULL"); + const std::string path = v_path.ToString(); + + if (!std::filesystem::exists(path)) { + throw InvalidInputException("kmeans_predict: file not found: %s", path); + } + + // Parse CSV: one centroid per line, values comma-separated + std::vector> centroids; + std::ifstream f(path); + std::string line; + while (std::getline(f, line)) { + if (line.empty()) continue; + std::vector row; + std::istringstream iss(line); + std::string cell; + while (std::getline(iss, cell, ',')) { + row.push_back(std::stod(cell)); + } + if (!row.empty()) centroids.push_back(std::move(row)); + } + + if (centroids.empty()) { + throw InvalidInputException("kmeans_predict: no centroids found in %s", path); + } + const idx_t K = centroids.size(); + const idx_t D = centroids[0].size(); + for (idx_t k = 1; k < K; ++k) { + if (centroids[k].size() != D) { + throw InvalidInputException("kmeans_predict: centroid dimension mismatch at row %llu", (unsigned long long)k); + } + } + + return make_uniq(std::move(centroids), K, D, path); +} + +// ─── Execute ────────────────────────────────────────────────────────────────── + +void ML_KMeansPredict::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + const auto &info = state.expr.Cast().bind_info->Cast(); + const idx_t K = info.K; + const idx_t D = info.D; + + // Input feature list + Vector &feat = args.data[0]; + UnifiedVectorFormat feat_uvf; + feat.ToUnifiedFormat(count, feat_uvf); + const auto *entries = UnifiedVectorFormat::GetData(feat_uvf); + auto &child = ListVector::GetEntry(feat); + UnifiedVectorFormat child_uvf; + child.ToUnifiedFormat(ListVector::GetListSize(feat), child_uvf); + const double *feat_data = UnifiedVectorFormat::GetData(child_uvf); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto *out = FlatVector::GetData(result); + auto &out_valid = FlatVector::Validity(result); + + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = feat_uvf.sel->get_index(i); + if (!feat_uvf.validity.RowIsValid(sel)) { + out_valid.Set(i, false); + continue; + } + + const auto le = entries[sel]; + if (le.length != D) { + throw InvalidInputException( + "kmeans_predict: feature length %llu does not match centroid dimension %llu", + (unsigned long long)le.length, (unsigned long long)D); + } + + const double *x = feat_data + le.offset; + + double best_dist = std::numeric_limits::max(); + int32_t best_k = 0; + for (idx_t k = 0; k < K; ++k) { + double dist = 0.0; + for (idx_t d = 0; d < D; ++d) { + double diff = x[d] - info.centroids[k][d]; + dist += diff * diff; + } + if (dist < best_dist) { + best_dist = dist; + best_k = static_cast(k); + } + } + out[i] = best_k; + } +} + +// ─── Register ───────────────────────────────────────────────────────────────── + +void ML_KMeansPredict::Register(ExtensionLoader &loader) { + ScalarFunction fn( + "kmeans_predict", + {LogicalType::LIST(LogicalType::DOUBLE), LogicalType::VARCHAR}, + LogicalType::INTEGER, + Execute + ); + fn.bind = Bind; + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + +} // namespace duckdb diff --git a/src/ml_functions/ml_functions_scalar.cpp b/src/ml_functions/ml_functions_scalar.cpp index 0126340..e287128 100644 --- a/src/ml_functions/ml_functions_scalar.cpp +++ b/src/ml_functions/ml_functions_scalar.cpp @@ -8,9 +8,12 @@ #include "ml_functions/argmax.hpp" #include "ml_functions/softmax.hpp" #include "ml_functions/sigmoid.hpp" +#include "ml_functions/relu.hpp" #include "ml_functions/torch_dnn.hpp" #include "ml_functions/decision_forest.hpp" #include "ml_functions/ml_decision_forest_table.hpp" +#include "ml_functions/text_functions.hpp" +#include "ml_functions/kmeans.hpp" namespace duckdb { @@ -25,9 +28,14 @@ void RegisterMLScalarFunctions(ExtensionLoader &loader) { ML_Argmax::Register(loader); ML_Softmax::Register(loader); ML_Sigmoid::Register(loader); + ML_Relu::Register(loader); ML_TorchDNN::Register(loader); ML_DecisionForest::Register(loader); ML_ForestTable::Register(loader); + ML_CleanText::Register(loader); + ML_TokenizeText::Register(loader); + ML_StemToken::Register(loader); + ML_KMeansPredict::Register(loader); // RegisterSoftmax(loader); // RegisterSigmoid(loader); } diff --git a/src/ml_functions/relu.cpp b/src/ml_functions/relu.cpp new file mode 100644 index 0000000..704a322 --- /dev/null +++ b/src/ml_functions/relu.cpp @@ -0,0 +1,90 @@ +#include "ml_functions/relu.hpp" + +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/common/vector.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +unique_ptr ML_Relu::Bind(ClientContext &context, + ScalarFunction &bound_function, + vector> &args) { + if (args.size() != 1) { + throw BinderException("relu: expects exactly 1 argument: LIST"); + } + return nullptr; +} + +void ML_Relu::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + Vector &in = args.data[0]; + UnifiedVectorFormat in_uvf; + in.ToUnifiedFormat(count, in_uvf); + const auto *entries = UnifiedVectorFormat::GetData(in_uvf); + + auto &child = ListVector::GetEntry(in); + const idx_t total_elems = ListVector::GetListSize(in); + UnifiedVectorFormat child_uvf; + child.ToUnifiedFormat(total_elems, child_uvf); + const double *child_data = UnifiedVectorFormat::GetData(child_uvf); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto *res_entries = FlatVector::GetData(result); + auto &res_child = ListVector::GetEntry(result); + auto &res_valid = FlatVector::Validity(result); + + ListVector::Reserve(result, total_elems); + + idx_t out_cursor = 0; + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + + if (!in_uvf.validity.RowIsValid(sel)) { + res_entries[i] = {0, 0}; + res_valid.Set(i, false); + continue; + } + + const auto le = entries[sel]; + const idx_t len = le.length; + const idx_t off = le.offset; + + if (len == 0) { + throw InvalidInputException("relu: empty list at row %llu", (unsigned long long)i); + } + + using RowVec = Eigen::Matrix; + Eigen::Map row(child_data + off, (Eigen::Index)len); + // ReLU: max(0, x) element-wise + RowVec out_row = row.array().max(0.0).matrix(); + + const idx_t prev_total = ListVector::GetListSize(result); + ListVector::SetListSize(result, prev_total + len); + double *out = FlatVector::GetData(res_child); + + res_entries[i].offset = out_cursor; + res_entries[i].length = len; + std::memcpy(out + out_cursor, out_row.data(), sizeof(double) * (size_t)len); + out_cursor += len; + } +} + +void ML_Relu::Register(ExtensionLoader &loader) { + ScalarFunction fn( + "relu", + { LogicalType::LIST(LogicalType::DOUBLE) }, + LogicalType::LIST(LogicalType::DOUBLE), + Execute + ); + fn.bind = Bind; + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + +} // namespace duckdb diff --git a/src/ml_functions/text_functions.cpp b/src/ml_functions/text_functions.cpp new file mode 100644 index 0000000..245c6c9 --- /dev/null +++ b/src/ml_functions/text_functions.cpp @@ -0,0 +1,205 @@ +#include "ml_functions/text_functions.hpp" + +#include +#include +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/types/vector.hpp" +#include "duckdb/common/vector_operations/vector_operations.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +// ─── clean_text ─────────────────────────────────────────────────────────────── +// Replaces any character that is not a-z/A-Z/space with a space, then +// collapses runs of whitespace into a single space and trims ends. + +void ML_CleanText::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + Vector &in = args.data[0]; + UnifiedVectorFormat in_uvf; + in.ToUnifiedFormat(count, in_uvf); + const string_t *in_data = UnifiedVectorFormat::GetData(in_uvf); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto &res_valid = FlatVector::Validity(result); + + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + if (!in_uvf.validity.RowIsValid(sel)) { + res_valid.Set(i, false); + FlatVector::GetData(result)[i] = string_t(); + continue; + } + + std::string s = in_data[sel].GetString(); + // Replace non-alpha with space + for (char &c : s) { + if (!std::isalpha((unsigned char)c)) c = ' '; + } + // Collapse whitespace + std::string out; + out.reserve(s.size()); + bool prev_space = true; // leading trim + for (char c : s) { + if (c == ' ') { + if (!prev_space) out += ' '; + prev_space = true; + } else { + out += c; + prev_space = false; + } + } + // Trailing trim + if (!out.empty() && out.back() == ' ') out.pop_back(); + + FlatVector::GetData(result)[i] = StringVector::AddString(result, out); + } +} + +void ML_CleanText::Register(ExtensionLoader &loader) { + ScalarFunction fn("clean_text", {LogicalType::VARCHAR}, LogicalType::VARCHAR, Execute); + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + + +// ─── tokenize_text ──────────────────────────────────────────────────────────── +// Splits a string on whitespace and returns LIST of tokens. + +void ML_TokenizeText::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + Vector &in = args.data[0]; + UnifiedVectorFormat in_uvf; + in.ToUnifiedFormat(count, in_uvf); + const string_t *in_data = UnifiedVectorFormat::GetData(in_uvf); + + // Pre-compute tokens for every row + std::vector> all_tokens(count); + idx_t total = 0; + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + if (!in_uvf.validity.RowIsValid(sel)) continue; + + std::istringstream iss(in_data[sel].GetString()); + std::string tok; + while (iss >> tok) { + if (!tok.empty()) { + all_tokens[i].push_back(std::move(tok)); + ++total; + } + } + } + + result.SetVectorType(VectorType::FLAT_VECTOR); + ListVector::Reserve(result, total); + + auto *entries = FlatVector::GetData(result); + auto &child = ListVector::GetEntry(result); + auto &res_valid = FlatVector::Validity(result); + auto *child_data = FlatVector::GetData(child); + + idx_t offset = 0; + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + if (!in_uvf.validity.RowIsValid(sel)) { + res_valid.Set(i, false); + entries[i] = {0, 0}; + continue; + } + const auto &toks = all_tokens[i]; + entries[i] = {offset, toks.size()}; + for (const auto &t : toks) { + child_data[offset] = StringVector::AddString(child, t); + ++offset; + } + } + ListVector::SetListSize(result, offset); +} + +void ML_TokenizeText::Register(ExtensionLoader &loader) { + ScalarFunction fn( + "tokenize_text", + {LogicalType::VARCHAR}, + LogicalType::LIST(LogicalType::VARCHAR), + Execute + ); + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + + +// ─── stem_token ─────────────────────────────────────────────────────────────── +// Simple suffix-stripping stemmer. Handles the most common English endings. +// Minimum stem length of 3 is preserved to avoid over-stemming. + +static std::string simple_stem(std::string w) { + const idx_t min_stem = 3; + auto try_strip = [&](const char *suffix, idx_t suffix_len) -> bool { + if (w.size() <= min_stem + suffix_len) return false; + if (w.compare(w.size() - suffix_len, suffix_len, suffix) == 0) { + w.resize(w.size() - suffix_len); + return true; + } + return false; + }; + // Order matters: longer suffixes first + if (try_strip("ational", 7)) { w += "ate"; return w; } + if (try_strip("tional", 6)) { w += "tion"; return w; } + if (try_strip("ization", 7)) { w += "ize"; return w; } + if (try_strip("ation", 5)) { w += "ate"; return w; } + if (try_strip("alism", 5)) { w += "al"; return w; } + if (try_strip("ness", 4)) return w; + if (try_strip("ment", 4)) return w; + if (try_strip("ful", 3)) return w; + if (try_strip("ing", 3)) return w; + if (try_strip("ion", 3)) return w; + if (try_strip("ies", 3)) { w += 'y'; return w; } + if (try_strip("ied", 3)) { w += 'y'; return w; } + if (try_strip("ess", 3)) return w; + if (try_strip("ers", 3)) return w; + if (try_strip("er", 2)) return w; + if (try_strip("ed", 2)) return w; + if (try_strip("es", 2)) return w; + // Final -s (not -ss) + if (w.size() > min_stem + 1 && w.back() == 's' && *(w.end()-2) != 's') { + w.pop_back(); + } + return w; +} + +void ML_StemToken::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + Vector &in = args.data[0]; + UnifiedVectorFormat in_uvf; + in.ToUnifiedFormat(count, in_uvf); + const string_t *in_data = UnifiedVectorFormat::GetData(in_uvf); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto &res_valid = FlatVector::Validity(result); + + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + if (!in_uvf.validity.RowIsValid(sel)) { + res_valid.Set(i, false); + FlatVector::GetData(result)[i] = string_t(); + continue; + } + std::string stemmed = simple_stem(in_data[sel].GetString()); + FlatVector::GetData(result)[i] = StringVector::AddString(result, stemmed); + } +} + +void ML_StemToken::Register(ExtensionLoader &loader) { + ScalarFunction fn("stem_token", {LogicalType::VARCHAR}, LogicalType::VARCHAR, Execute); + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + +} // namespace duckdb diff --git a/src/optimization/CMakeLists.txt b/src/optimization/CMakeLists.txt index 5ae8696..f2fb32a 100644 --- a/src/optimization/CMakeLists.txt +++ b/src/optimization/CMakeLists.txt @@ -6,14 +6,17 @@ list(APPEND EXTENSION_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_optimizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_rule.cpp ${CMAKE_CURRENT_SOURCE_DIR}/catalog.cpp - + ${CMAKE_CURRENT_SOURCE_DIR}/optimizer_pass.cpp + #All Actions ${CMAKE_CURRENT_SOURCE_DIR}/actions/MatMulDense2SparseRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/MultiLayerUDF2TorchNNRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/DecisionForestUDF2RelationRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLDecompositionPushdownRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLFactorizationRewriteAction.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLGreedyFactorizationOptimizer.cpp + # MLGreedyFactorizationOptimizer.cpp is intentionally NOT compiled in: + # the baseline ships without the greedy cost-based pass. It is added at + # runtime via the frontend upload -> plugins/opt_passes/ -> rebuild flow. #Feature Extraction Helpers ${CMAKE_CURRENT_SOURCE_DIR}/feature_extraction/query2vec/query_annotation.cpp diff --git a/src/optimization/actions/DecisionForestUDF2RelationRewriteAction.cpp b/src/optimization/actions/DecisionForestUDF2RelationRewriteAction.cpp index 410ded4..a4d8ec3 100644 --- a/src/optimization/actions/DecisionForestUDF2RelationRewriteAction.cpp +++ b/src/optimization/actions/DecisionForestUDF2RelationRewriteAction.cpp @@ -11,6 +11,8 @@ #include "duckdb/function/aggregate_function.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_operator_expression.hpp" #include "duckdb/planner/expression/bound_case_expression.hpp" @@ -111,6 +113,11 @@ static unique_ptr BuildForestGet(ClientContext &context, const auto get = make_uniq(table_index, std::move(tf), std::move(bind_data), std::move(return_types), std::move(names), std::move(virtual_columns)); + // Explicitly declare both columns so GetColumnBindings() returns {(0,0),(0,1)}. + // Without this, column_ids is empty and only (0,0) is visible to ColumnBindingResolver, + // causing it to fail when forest_path_proj references (0,1) = tree_path. + get->AddColumnId(0); // tree_id + get->AddColumnId(1); // tree_path std::cout << "Built forest_trees GET over path: " << forest_path << "\n"; return std::move(get); } @@ -121,15 +128,13 @@ static AggregateFunction GetAvgFloatAggregate(ClientContext &context) { auto &entry = catalog.GetEntry(context, DEFAULT_SCHEMA, "avg"); - // Pick the overload with FLOAT (or DOUBLE) argument + // DuckDB's avg only has a DOUBLE overload; the FLOAT input is cast before calling it. for (auto &fn : entry.functions.functions) { - if (fn.arguments.size() == 1 && - (fn.arguments[0].id() == LogicalTypeId::FLOAT || - fn.arguments[0].id() == LogicalTypeId::DOUBLE)) { + if (fn.arguments.size() == 1 && fn.arguments[0].id() == LogicalTypeId::DOUBLE) { return fn; } } - throw InternalException("DecisionForest rewrite: AVG(FLOAT/DOUBLE) aggregate not found"); + throw InternalException("DecisionForest rewrite: AVG(DOUBLE) aggregate not found"); } // Core rewrite: @@ -220,8 +225,21 @@ static bool RewriteForestProjectionToCrossProduct(OptimizerExtensionInput &input // Copy the original features expression as a generic bound expression unique_ptr feat_expr_original = forest_func->children[0]->Copy(); - // 5) Build right-hand side GET forest_trees(forest_path) + // 5) Build right-hand side GET forest_trees(forest_path), then project + // out tree_id so only tree_path flows into the CROSS_PRODUCT. + // Without this, DuckDB's column pruner drops tree_id at the physical + // level (since nothing references it), shifting tree_path from index + // left_cols+1 to left_cols+0 and causing an out-of-bounds access. auto right_get = BuildForestGet(context, forest_path); + // forest_trees outputs (tree_id INT=0, tree_path VARCHAR=1); keep only tree_path. + // Use BoundColumnRefExpression so DuckDB's ColumnBindingResolver correctly remaps + // the index after RemoveUnusedColumns prunes tree_id from the LogicalGet. + // BuildForestGet uses table_index=0 for the FOREST_TREES LogicalGet. + vector> forest_proj_exprs; + forest_proj_exprs.push_back( + make_uniq(LogicalType::VARCHAR, ColumnBinding(0, 1))); + auto forest_path_proj = make_uniq(99, std::move(forest_proj_exprs)); + forest_path_proj->children.push_back(std::move(right_get)); // 6) Original child of projection becomes left side of CROSS_PRODUCT if (proj.children.empty()) { @@ -230,19 +248,15 @@ static bool RewriteForestProjectionToCrossProduct(OptimizerExtensionInput &input return false; } auto left_child = std::move(proj.children[0]); + idx_t left_cols = left_child->types.size(); - // CROSS_PRODUCT(left_child, forest_trees) - auto cross = make_uniq(std::move(left_child), std::move(right_get)); + // CROSS_PRODUCT(left_child, forest_path_proj) + // Schema: [ all columns from left_child | tree_path at left_cols ] + auto cross = make_uniq(std::move(left_child), std::move(forest_path_proj)); - // After CROSS_PRODUCT schema: - // [ all columns from left_child | tree_id, tree_path ] - idx_t left_cols = cross->children[0]->types.size(); - idx_t tree_id_col_idx = left_cols + 0; - (void)tree_id_col_idx; // not used right now - idx_t tree_path_col_idx = left_cols + 1; + idx_t tree_path_col_idx = left_cols; // tree_path is the only right-side column std::cout << "CROSS_PRODUCT schema: left_cols = " << left_cols - << ", tree_id_col_idx = " << tree_id_col_idx << ", tree_path_col_idx = " << tree_path_col_idx << "\n"; // 7) Build inner PROJECTION on top of CROSS_PRODUCT @@ -262,8 +276,9 @@ static bool RewriteForestProjectionToCrossProduct(OptimizerExtensionInput &input continue; } auto &orig = *proj.expressions[i]; - inner_proj_exprs.push_back( - make_uniq(orig.return_type, next_col_idx)); + // Copy the original expression directly — it already carries the correct + // BoundColumnRefExpression binding from the binder, which survives column pruning. + inner_proj_exprs.push_back(proj.expressions[i]->Copy()); group_types.push_back(orig.return_type); next_col_idx++; } @@ -276,13 +291,7 @@ static bool RewriteForestProjectionToCrossProduct(OptimizerExtensionInput &input // idx_t left_cols = cross->children[0]->types.size(); // idx_t tree_path_col_idx = left_cols + 1; -// 1) Figure out which column is "features" on the left side. -// In your current setup, the projection child is [id, features], -// so features is column index 1. -// For now we hard-code that, since this rule is specialized to your pattern. -idx_t features_col_idx = 1; - -// 2) Lookup tree_predict(...) function from the catalog +// 1) Lookup tree_predict(...) function from the catalog auto &catalog = Catalog::GetSystemCatalog(context); auto &tree_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, "tree_predict"); @@ -308,19 +317,17 @@ ScalarFunction tree_func = *tree_func_ptr; // 3) Build children for tree_predict(features, tree_path, false) -// features: directly reference the LIST(FLOAT) column from CROSS_PRODUCT -auto feat_ref_for_child = - make_uniq(LogicalType::LIST(LogicalType::FLOAT), - features_col_idx); -auto feat_ref_for_bind = - make_uniq(LogicalType::LIST(LogicalType::FLOAT), - features_col_idx); +// features: reuse the original BoundColumnRefExpression from the forest UDF call — +// it already carries the correct binding for features in the left child subplan. +auto feat_ref_for_child = feat_expr_original->Copy(); +auto feat_ref_for_bind = feat_expr_original->Copy(); -// tree_path: VARCHAR column from CROSS_PRODUCT +// tree_path: forest_path_proj has table_index=99, outputs tree_path at column 0. +// Use BoundColumnRefExpression so ColumnBindingResolver remaps it correctly. auto tree_path_ref_for_child = - make_uniq(LogicalType::VARCHAR, tree_path_col_idx); + make_uniq(LogicalType::VARCHAR, ColumnBinding(99, 0)); auto tree_path_ref_for_bind = - make_uniq(LogicalType::VARCHAR, tree_path_col_idx); + make_uniq(LogicalType::VARCHAR, ColumnBinding(99, 0)); // has_missing: FALSE literal auto has_missing_child = @@ -353,7 +360,7 @@ auto tree_pred_expr = make_uniq( inner_proj_exprs.push_back(std::move(tree_pred_expr)); // Build LogicalProjection with the new API (table_index + select_list) - idx_t inner_table_index = 0; + idx_t inner_table_index = 98; // unique index; must not collide with FOREST_TREES(0) or outer proj auto inner_proj = make_uniq(inner_table_index, std::move(inner_proj_exprs)); inner_proj->children.push_back(std::move(cross)); @@ -361,24 +368,30 @@ auto tree_pred_expr = make_uniq( // 8) Build AGGREGATE on top of inner_proj: // SELECT group_keys..., AVG(pred) // - // LogicalAggregate takes a select_list of expressions, where: - // - [0..group_count-1] are group keys - // - [group_count..] are BoundAggregateExpression(s) - vector> aggregate_select; + // LogicalAggregate: group keys go into aggregate->groups, + // only BoundAggregateExpression(s) go in the select_list/expressions. + // Mixing BoundReferenceExpression into expressions causes a type-resolution crash. - // group keys: references [0..group_types.size()-1] of inner_proj output + // group keys for aggregate->groups — reference inner_proj output by ColumnBinding + vector> group_exprs; for (idx_t i = 0; i < group_types.size(); i++) { - aggregate_select.push_back( - make_uniq(group_types[i], i)); + group_exprs.push_back( + make_uniq(group_types[i], ColumnBinding(inner_table_index, i))); } // prediction column index in inner_proj idx_t pred_idx = group_types.size(); - // children for AVG: just reference the prediction column + // children for AVG: reference prediction column via ColumnBinding. + // DuckDB's AVG only has a DOUBLE overload; cast FLOAT → DOUBLE explicitly + // so the executor doesn't see a type mismatch at runtime. vector> avg_children; - avg_children.push_back( - make_uniq(tree_func.return_type, pred_idx)); + { + auto pred_ref = make_uniq( + tree_func.return_type, ColumnBinding(inner_table_index, pred_idx)); + avg_children.push_back( + BoundCastExpression::AddCastToType(context, std::move(pred_ref), LogicalType::DOUBLE)); + } // lookup AVG(FLOAT/DOUBLE) auto avg_fun = GetAvgFloatAggregate(context); @@ -393,15 +406,18 @@ auto tree_pred_expr = make_uniq( avg_fun, std::move(avg_children), std::move(avg_filter), std::move(avg_bind_info), agg_type); - // append AVG(pred) to select list - aggregate_select.push_back(std::move(avg_agg)); + // select_list contains only aggregate expressions + vector> agg_select; + agg_select.push_back(std::move(avg_agg)); - // group_index = 0, aggregate_index = group_count - idx_t group_index = 0; - idx_t aggregate_index = group_types.size(); + // group_index and aggregate_index are table-level binding indices; + // use distinct values that won't collide with inner_proj (0) or original projection. + idx_t group_index = 100; + idx_t aggregate_index = 101; auto aggregate = make_uniq( - group_index, aggregate_index, std::move(aggregate_select)); + group_index, aggregate_index, std::move(agg_select)); + aggregate->groups = std::move(group_exprs); aggregate->children.push_back(std::move(inner_proj)); // 9) Final projection on top: @@ -409,23 +425,23 @@ auto tree_pred_expr = make_uniq( // - forest_result = (is_classification ? (avg > 0 ? 1 : 0) : avg) vector> final_proj_exprs; - // group keys: references [0..group_types.size()-1] of aggregate output + // group keys from aggregate output — reference by ColumnBinding(group_index, i) for (idx_t i = 0; i < group_types.size(); i++) { final_proj_exprs.push_back( - make_uniq(group_types[i], i)); + make_uniq(group_types[i], ColumnBinding(group_index, i))); } - // avg_pred is at col index = group_types.size() - idx_t avg_idx = group_types.size(); + // avg result is the first (and only) aggregate expression — ColumnBinding(aggregate_index, 0) auto avg_ref = - make_uniq(avg_type, avg_idx); + make_uniq(avg_type, ColumnBinding(aggregate_index, 0)); unique_ptr forest_result_expr; if (is_classification) { // CASE WHEN avg > 0 THEN 1 ELSE 0 - auto zero_val = Value::FLOAT(0.0f); - auto zero_expr = make_uniq(zero_val); + // avg_type is DOUBLE (AVG(FLOAT) returns DOUBLE in DuckDB), so both sides + // of the comparison must be DOUBLE to avoid a bound-expression type mismatch. + auto zero_expr = make_uniq(Value(0).DefaultCastAs(avg_type)); auto cond = make_uniq( ExpressionType::COMPARE_GREATERTHAN, avg_ref->Copy(), diff --git a/src/optimization/actions/MLFactorizationRewriteAction.cpp b/src/optimization/actions/MLFactorizationRewriteAction.cpp index fe3dd48..b3f2e3a 100644 --- a/src/optimization/actions/MLFactorizationRewriteAction.cpp +++ b/src/optimization/actions/MLFactorizationRewriteAction.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -280,63 +279,59 @@ static bool InsertProjectionOnSide(LogicalComparisonJoin &join, int side_idx, // Pattern detection (used by both check and apply) // --------------------------------------------------------------------------- -// Return true iff expr is: +// Recursively search expr for a sub-expression matching: // mat_mul( list_concat(L, R), W_file, mode ) -// where L references only left_tables, R references only right_tables, -// and k_left = GetStaticListLength(L) is positive. -static bool MatchesFactorizationPattern(const unique_ptr &expr, - const vector &left_tables, - const vector &right_tables) { +// where L references only left_tables, R only right_tables, and k_left > 0. +// Returns a pointer to the unique_ptr holding the matching mat_mul +// so the caller can replace it in-place, or nullptr if not found. +static unique_ptr *FindFactorizableMatMul(unique_ptr &expr, + const vector &left_tables, + const vector &right_tables) { if (!expr || expr->expression_class != ExpressionClass::BOUND_FUNCTION) { - std::cout << "[MLFact] skip: expr not BOUND_FUNCTION\n"; - return false; + return nullptr; } auto &f = expr->Cast(); - std::cout << "[MLFact] top expr function: '" << f.function.name << "'\n"; - if (!IsMatMul(f) || f.children.empty()) { - return false; - } - auto *bd = dynamic_cast(f.bind_info.get()); - if (!bd || !bd->has_bound_weights) { - std::cout << "[MLFact] skip: no bound weights\n"; - return false; - } - - auto &first_child = f.children[0]; - if (!first_child || first_child->expression_class != ExpressionClass::BOUND_FUNCTION) { - std::cout << "[MLFact] skip: first child of mat_mul not BOUND_FUNCTION (class=" - << (first_child ? std::to_string((int)first_child->expression_class) : "null") << ")\n"; - return false; - } - auto &lc = first_child->Cast(); - std::cout << "[MLFact] first child function: '" << lc.function.name - << "' children=" << lc.children.size() << "\n"; - if (!IsListConcat(lc) || lc.children.size() != 2) { - std::cout << "[MLFact] skip: not list_concat or wrong child count\n"; - return false; + // Check whether THIS node is the mat_mul(list_concat(L, R), W) pattern. + if (IsMatMul(f) && !f.children.empty()) { + auto *bd = dynamic_cast(f.bind_info.get()); + if (bd && bd->has_bound_weights) { + auto &first_child = f.children[0]; + if (first_child && + first_child->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &lc = first_child->Cast(); + if (IsListConcat(lc) && lc.children.size() == 2) { + std::unordered_set left_refs, right_refs; + CollectTableIndexes(lc.children[0], left_refs); + CollectTableIndexes(lc.children[1], right_refs); + int64_t k_left = GetStaticListLength(lc.children[0]); + if (k_left > 0 && + TablesOnlyInSide(left_refs, left_tables) && + TablesOnlyInSide(right_refs, right_tables)) { + return &expr; // caller owns this unique_ptr and may replace it + } + } + } + } } - std::unordered_set left_refs, right_refs; - CollectTableIndexes(lc.children[0], left_refs); - CollectTableIndexes(lc.children[1], right_refs); - - std::cout << "[MLFact] left_refs count=" << left_refs.size() - << " right_refs count=" << right_refs.size() << "\n"; - - if (!TablesOnlyInSide(left_refs, left_tables)) { - std::cout << "[MLFact] skip: left_refs not contained in join left side\n"; - return false; - } - if (!TablesOnlyInSide(right_refs, right_tables)) { - std::cout << "[MLFact] skip: right_refs not contained in join right side\n"; - return false; + // Recurse into all children of this bound function. + for (auto &child : f.children) { + auto *result = FindFactorizableMatMul(child, left_tables, right_tables); + if (result) { + return result; + } } + return nullptr; +} - int64_t k_left = GetStaticListLength(lc.children[0]); - std::cout << "[MLFact] k_left=" << k_left - << " (type=" << lc.children[0]->return_type.ToString() << ")\n"; - return k_left > 0; +// Thin wrapper used by check() — const-safe via const_cast (read-only traversal). +static bool MatchesFactorizationPattern(const unique_ptr &expr, + const vector &left_tables, + const vector &right_tables) { + return FindFactorizableMatMul( + const_cast &>(expr), left_tables, right_tables) != + nullptr; } } // anonymous namespace @@ -353,14 +348,11 @@ bool MLFactorizationRewriteAction::check(OptimizerExtensionInput &input, bool found = false; - std::cout << "[MLFact check] node type=" << EnumUtil::ToString(plan->type) << "\n"; - if (plan->type == LogicalOperatorType::LOGICAL_PROJECTION && plan->children.size() == 1 && plan->children[0] && plan->children[0]->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { - std::cout << "[MLFact check] found PROJECTION over COMPARISON_JOIN\n"; auto &join = plan->children[0]->Cast(); auto left_tables = join.children[0]->GetTableIndex(); auto right_tables = join.children[1]->GetTableIndex(); @@ -419,12 +411,16 @@ bool MLFactorizationRewriteAction::apply(OptimizerExtensionInput &input, } for (auto &expr_ptr : proj.expressions) { - if (!MatchesFactorizationPattern(expr_ptr, left_tables, right_tables)) { + // Recursively locate the mat_mul(list_concat(L,R), W) sub-expression. + // target_ptr points to the unique_ptr that holds it, so we + // can replace it in-place without disturbing the rest of the DNN chain. + auto *target_ptr = FindFactorizableMatMul(expr_ptr, left_tables, right_tables); + if (!target_ptr) { continue; } - // ---- Unpack the matched expression tree ---- - auto &f = expr_ptr->Cast(); + // ---- Unpack the matched mat_mul sub-expression ---- + auto &f = (*target_ptr)->Cast(); auto &lc = f.children[0]->Cast(); // list_concat auto *bd = dynamic_cast(f.bind_info.get()); @@ -495,15 +491,18 @@ bool MLFactorizationRewriteAction::apply(OptimizerExtensionInput &input, const idx_t left_global_idx = L_width; const idx_t right_global_idx = L_width + 1 + R_width; - // ---- Replace original mat_mul with list_add_vv(ref_left, ref_right) ---- + // ---- Replace the mat_mul sub-expression with list_add_vv(ref_left, ref_right) ---- + // *target_ptr is the unique_ptr holding the mat_mul node; replacing it + // patches it in-place so any enclosing relu/mat_add/softmax chain is + // preserved untouched. auto left_ref = make_uniq( LogicalType::LIST(LogicalType::DOUBLE), left_global_idx); auto right_ref = make_uniq( LogicalType::LIST(LogicalType::DOUBLE), right_global_idx); - expr_ptr = BuildListAddVVExpr(input.context, - std::move(left_ref), - std::move(right_ref)); + *target_ptr = BuildListAddVVExpr(input.context, + std::move(left_ref), + std::move(right_ref)); changed = true; break; // one factorization per projection per pass diff --git a/src/optimization/dynamic_rule.cpp b/src/optimization/dynamic_rule.cpp index 0425daf..e2dafeb 100644 --- a/src/optimization/dynamic_rule.cpp +++ b/src/optimization/dynamic_rule.cpp @@ -5,7 +5,6 @@ #include "optimization/actions/MultiLayerUDF2TorchNNRewriteAction.hpp" #include "optimization/actions/MLDecompositionPushdownRewriteAction.hpp" #include "optimization/actions/MLFactorizationRewriteAction.hpp" -#include "optimization/actions/MLGreedyFactorizationOptimizer.hpp" #include "optimization/actions/DecisionForestUDF2RelationRewriteAction.hpp" #include @@ -68,7 +67,6 @@ const std::unordered_map &ActionTable() { {"MatMulDense2SparseRewriteAction", &MatMulDense2SparseRewriteAction::apply}, {"MultiLayerUDF2TorchNNRewriteAction", &MultiLayerUDF2TorchNNRewriteAction::apply}, {"MLFactorizationRewriteAction", &MLFactorizationRewriteAction::apply}, - {"MLGreedyFactorizationOptimizer", &MLGreedyFactorizationOptimizer::apply}, {"DecisionForestUDF2RelationRewriteAction", &DecisionForestUDF2RelationRewriteAction::apply}, }; return table; diff --git a/src/optimization/optimizer_pass.cpp b/src/optimization/optimizer_pass.cpp new file mode 100644 index 0000000..579fa75 --- /dev/null +++ b/src/optimization/optimizer_pass.cpp @@ -0,0 +1,84 @@ +#include "optimization/optimizer_pass.hpp" + +#include "duckdb/function/table_function.hpp" +#include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/common/types/data_chunk.hpp" + +#include + +namespace duckdb { + +OptimizerPassRegistry &OptimizerPassRegistry::Get() { + static OptimizerPassRegistry instance; + return instance; +} + +void OptimizerPassRegistry::Register(OptimizerPass pass) { + passes_[pass.name] = std::move(pass); // last-wins so a re-upload supersedes +} + +const OptimizerPass *OptimizerPassRegistry::Find(const std::string &name) const { + auto it = passes_.find(name); + return it == passes_.end() ? nullptr : &it->second; +} + +std::vector OptimizerPassRegistry::Names() const { + std::vector names; + names.reserve(passes_.size()); + for (const auto &kv : passes_) { + names.push_back(kv.first); + } + std::sort(names.begin(), names.end()); + return names; +} + +// --------------------------------------------------------------------------- +// cactusdb_list_opt_passes() table function +// --------------------------------------------------------------------------- + +namespace { + +struct ListPassesBindData : public TableFunctionData { + std::vector names; +}; + +struct ListPassesState : public GlobalTableFunctionState { + idx_t cursor = 0; +}; + +unique_ptr ListPassesBind(ClientContext &, TableFunctionBindInput &, + vector &return_types, vector &names) { + return_types.push_back(LogicalType::VARCHAR); + names.push_back("name"); + auto bind = make_uniq(); + bind->names = OptimizerPassRegistry::Get().Names(); + return std::move(bind); +} + +unique_ptr ListPassesInit(ClientContext &, TableFunctionInitInput &) { + return make_uniq(); +} + +void ListPassesFunction(ClientContext &, TableFunctionInput &data, DataChunk &output) { + auto &bind = data.bind_data->Cast(); + auto &state = data.global_state->Cast(); + + idx_t remaining = bind.names.size() - state.cursor; + idx_t count = std::min(STANDARD_VECTOR_SIZE, remaining); + output.SetCardinality(count); + for (idx_t i = 0; i < count; i++) { + output.data[0].SetValue(i, Value(bind.names[state.cursor + i])); + } + state.cursor += count; +} + +} // namespace + +void RegisterOptimizerPassListFunction(ExtensionLoader &loader) { + TableFunction tf("cactusdb_list_opt_passes", {}, ListPassesFunction); + tf.bind = ListPassesBind; + tf.init_global = ListPassesInit; + loader.RegisterFunction(tf); +} + +} // namespace duckdb diff --git a/src/optimization/register_optimizers.cpp b/src/optimization/register_optimizers.cpp index 46c7a98..8ec4201 100644 --- a/src/optimization/register_optimizers.cpp +++ b/src/optimization/register_optimizers.cpp @@ -9,6 +9,7 @@ #include "optimization/dynamic_optimizer.hpp" #include "optimization/metric_catalog.hpp" #include "optimization/dynamic_rule.hpp" +#include "optimization/optimizer_pass.hpp" #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" @@ -25,21 +26,29 @@ #include "optimization/actions/MultiLayerUDF2TorchNNRewriteAction.hpp" #include "optimization/actions/MLDecompositionPushdownRewriteAction.hpp" #include "optimization/actions/MLFactorizationRewriteAction.hpp" -#include "optimization/actions/MLGreedyFactorizationOptimizer.hpp" +// MLGreedyFactorizationOptimizer is no longer compiled into the baseline; it is +// added at runtime via an uploaded plugin that self-registers into +// OptimizerPassRegistry and is selected with the "OPT#" setting. namespace duckdb { -//helper function to get schema name from LogicalOperator +// Return a catalog-qualified schema name ("catalog.schema") from the first +// LOGICAL_GET found in the plan tree. This lets the MetricCatalog distinguish +// uc08_db.main from uc03_db.main even though both have schema name "main". +// For the default in-memory catalog ("memory") we return just the schema name +// so that the "synthetic" entry in MetricCatalog continues to match. static string GetQuerySchema(LogicalOperator &op) { if (op.type == LogicalOperatorType::LOGICAL_GET) { auto &get = op.Cast(); auto table_entry = get.GetTable(); if (table_entry) { - // This line may or may not compile depending on visibility: - return table_entry->schema.name; - - // If that complains about access, just fall back: - // return table_entry->GetName(); // as described above + const string &schema_name = table_entry->schema.name; + const string catalog_name = table_entry->catalog.GetName(); + if (!catalog_name.empty() && + catalog_name != "memory" && catalog_name != "system") { + return catalog_name + "." + schema_name; + } + return schema_name; } return string(); } @@ -144,6 +153,22 @@ struct CactusDynamicOpt : public OptimizerExtension { return; } + // "OPT#" — run a single registered optimizer pass by name. + // Passes self-register into OptimizerPassRegistry from compiled-in plugin + // sources (plugins/opt_passes/*.cpp). An unknown name is a no-op so the + // baseline behaves correctly before a pass has been uploaded + rebuilt. + if (opt_setting.rfind("OPT#", 0) == 0) { + const std::string pass_name = opt_setting.substr(4); + if (const auto *pass = OptimizerPassRegistry::Get().Find(pass_name)) { + if (pass->check(in, plan)) { + pass->apply(in, plan); + } + } else { + std::cerr << "[OptPass] unknown pass skipped: " << pass_name << "\n"; + } + return; + } + if (opt_setting.find("1") != std::string::npos) { if (join_ratio > HIGH_JOIN_RATIO) { MLDecompositionPushdownRewriteAction::apply(in, plan); @@ -163,25 +188,14 @@ struct CactusDynamicOpt : public OptimizerExtension { if (opt_setting.find("4") != std::string::npos) { if (MLFactorizationRewriteAction::check(in, plan)) { - std::cout << "\n[MLFactorization] BEFORE:\n" << plan->ToString() << "\n"; MLFactorizationRewriteAction::apply(in, plan); - std::cout << "\n[MLFactorization] AFTER:\n" << plan->ToString() << "\n"; } } - if (opt_setting.find("5") != std::string::npos) { - if (MLGreedyFactorizationOptimizer::check(in, plan)) { - std::cout << "\n[MLGreedyFactorization] BEFORE:\n" << plan->ToString() << "\n"; - auto rewritten = MLGreedyFactorizationOptimizer::apply(in, plan); - if (rewritten) { - std::cout << "\n[MLGreedyFactorization] AFTER:\n" << plan->ToString() << "\n"; - } else { - std::cout << "\n[MLGreedyFactorization] SKIPPED: no safe rewrite for this plan shape\n"; - } - } - } + // Flag "5" (greedy factorization) is intentionally gone from the baseline. + // The greedy pass is now an uploadable plugin selected via "OPT#greedy_factorization". - // pretty = plan->ToString(); + // pretty = plan->ToString(); // std::cout << "\n[CactusDynamicOpt] Final Plan:\n" << pretty << "\n"; } @@ -192,12 +206,8 @@ struct CactusDynamicOpt1 : public OptimizerExtension { static void Run(OptimizerExtensionInput &in, unique_ptr &plan) { // DumpPlanBase64(in.context, *plan, "./plans/initial_plan.duckdbplan"); - std::string pretty = plan->ToString(); - std::cout << "\nInitial Plan:\n" << pretty << "\n"; MatMulDense2SparseRewriteAction::apply(in, plan); // DumpPlanBase64(in.context, *plan, "./plans/final_plan.duckdbplan"); - pretty = plan->ToString(); - std::cout << "\nFinal Plan:\n" << pretty << "\n"; } };