diff --git a/.gitignore b/.gitignore index 9d67701..b0da62b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,11 @@ archived_tests/* queries/synthetic_query/*.csv queries/synthetic_query/*.db *.zip + +# Uploaded optimizer-pass plugins are runtime artifacts; only the baseline +# (empty) manifest and the dir placeholder are tracked. +plugins/opt_passes/*.cpp +!plugins/opt_passes/_manifest.cpp +plugins/opt_passes/*.rejected +plugins/opt_passes/include/* +!plugins/opt_passes/include/.gitkeep diff --git a/CMakeLists.txt b/CMakeLists.txt index b00bbb8..02e96d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,16 @@ add_subdirectory(src/ml_functions) add_subdirectory(src/util) add_subdirectory(src/optimization) add_subdirectory(src/cost_model) + +# Uploadable optimizer passes. Files dropped in plugins/opt_passes/ by the +# frontend upload flow are compiled directly into the extension target (NOT a +# static archive) so their REGISTER_OPT_PASS static initializers survive linking. +# CONFIGURE_DEPENDS re-globs when a new upload appears, so no manual reconfigure. +file(GLOB OPT_PASS_PLUGINS CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/plugins/opt_passes/*.cpp") +list(APPEND EXTENSION_SOURCES ${OPT_PASS_PLUGINS}) +include_directories(plugins/opt_passes/include) # uploaded .hpp headers land here + message(STATUS "EXTENSION_SOURCES include ${EXTENSION_SOURCES}") build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) diff --git a/Frontend/__pycache__/app.cpython-311.pyc b/Frontend/__pycache__/app.cpython-311.pyc new file mode 100644 index 0000000..5eaa430 Binary files /dev/null and b/Frontend/__pycache__/app.cpython-311.pyc differ diff --git a/Frontend/__pycache__/app.cpython-312.pyc b/Frontend/__pycache__/app.cpython-312.pyc new file mode 100644 index 0000000..8b45f4e Binary files /dev/null and b/Frontend/__pycache__/app.cpython-312.pyc differ diff --git a/Frontend/app.py b/Frontend/app.py new file mode 100644 index 0000000..72911e2 --- /dev/null +++ b/Frontend/app.py @@ -0,0 +1,1490 @@ +from flask import Flask, render_template, request, redirect, url_for, flash, jsonify +from werkzeug.utils import secure_filename +import os +import re +import shutil +import subprocess +import tempfile +import threading +import time + +app = Flask(__name__) +app.secret_key = "demo-secret-key" + +UPLOAD_FOLDER = "uploads" +os.makedirs(UPLOAD_FOLDER, exist_ok=True) +app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER + +# --- Paths ------------------------------------------------------------------ + +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DUCKDB_BIN = os.path.join(PROJECT_ROOT, "build/release/duckdb") +EXTENSION_PATH = os.path.join( + PROJECT_ROOT, "build/release/extension/cactusdb/cactusdb.duckdb_extension" +) +SYNTHETIC_SQL_PATH = os.path.join( + PROJECT_ROOT, "queries/synthetic_query/synthetic_query.sql" +) + +# --- Uploadable optimizer-pass plugin flow ---------------------------------- + +PLUGIN_DIR = os.path.join(PROJECT_ROOT, "plugins/opt_passes") +PLUGIN_INCLUDE_DIR = os.path.join(PLUGIN_DIR, "include") +BUILD_LOG = "/tmp/optbench_build.log" + +# In-process build status, polled by the UI after an upload. +# status: "idle" | "building" | "ok" | "error" +BUILD_STATE = {"status": "idle", "pass_name": None, "slot_id": None, "message": ""} +BUILD_LOCK = threading.Lock() + +# Extract the registered name from REGISTER_OPT_PASS(, ...). +REGISTER_RE = re.compile(r"REGISTER_OPT_PASS\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)") +INCLUDE_RE = re.compile(r'^\s*#\s*include\s*[<"]([^">]+)[">]') + +# Uploaded C++ is compiled into a privileged extension; restrict #includes to a +# project/std allowlist as a lint guard (trusted-author-only — see plan §Challenge 4). +ALLOWED_INCLUDE_PREFIXES = ("duckdb", "optimization/", "ml_functions/", "cost_model/") +ALLOWED_STD_HEADERS = { + "algorithm", "cmath", "cstdint", "cstddef", "fstream", "functional", "iostream", + "memory", "queue", "sstream", "string", "unordered_map", "unordered_set", + "utility", "vector", "map", "set", "limits", "array", "tuple", "optional", +} + + +def _regenerate_manifest(): + """Rewrite plugins/opt_passes/_manifest.cpp so LinkOptPassPlugins() references + every currently-present plugin's anchor symbol. This reference is what keeps + each plugin object in the static link (see optimizer_pass.hpp).""" + idents = [] + if os.path.isdir(PLUGIN_DIR): + for fn in sorted(os.listdir(PLUGIN_DIR)): + if not fn.endswith(".cpp") or fn == "_manifest.cpp": + continue + with open(os.path.join(PLUGIN_DIR, fn)) as f: + m = REGISTER_RE.search(f.read()) + if m: + idents.append(m.group(1)) + + decls = "".join(f"extern \"C\" void optpass_anchor_{i}();\n" for i in idents) + refs = "".join( + f"\tkeep.push_back(reinterpret_cast(&optpass_anchor_{i}));\n" for i in idents + ) + body = ( + "// AUTO-GENERATED by the frontend upload flow — do not edit by hand.\n" + "#include \n\n" + f"{decls}\n" + "namespace duckdb {\n\n" + "void LinkOptPassPlugins() {\n" + + ("\tstatic std::vector keep;\n" + refs if idents else "\t// no plugins registered\n") + + "}\n\n" + "} // namespace duckdb\n" + ) + with open(os.path.join(PLUGIN_DIR, "_manifest.cpp"), "w") as f: + f.write(body) + + +def _lint_plugin_source(text): + """Return (pass_name, error). pass_name is None when error is set.""" + m = REGISTER_RE.search(text) + if not m: + return None, "Source has no REGISTER_OPT_PASS(, check, apply) line." + pass_name = m.group(1) + if not re.fullmatch(r"[A-Za-z0-9_]+", pass_name): + return None, f"Pass name '{pass_name}' must be alphanumeric/underscore." + for line in text.splitlines(): + im = INCLUDE_RE.match(line) + if not im: + continue + header = im.group(1) + if header in ALLOWED_STD_HEADERS: + continue + if any(header.startswith(p) for p in ALLOWED_INCLUDE_PREFIXES): + continue + return None, f"Disallowed #include \"{header}\" (outside the project/std allowlist)." + return pass_name, None + + +def _run_build_async(slot_id, pass_name, name): + """Rebuild the extension so the freshly placed plugin compiles in, then + activate the optimizer slot on success. Runs in a background thread.""" + try: + # `make release` reconfigures from scratch and needs Torch discoverable. + # Honor an existing LIBTORCH_DIR, else fall back to the conventional path. + build_env = dict(os.environ) + if "LIBTORCH_DIR" not in build_env: + default_torch = os.path.expanduser("~/libtorch") + if os.path.isdir(os.path.join(default_torch, "share/cmake/Torch")): + build_env["LIBTORCH_DIR"] = default_torch + with open(BUILD_LOG, "w") as log: + proc = subprocess.run( + ["make", "release"], + cwd=PROJECT_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + env=build_env, + ) + ok = proc.returncode == 0 and os.path.exists(EXTENSION_PATH) + except Exception as e: # noqa: BLE001 - surface any build launch failure + with BUILD_LOCK: + BUILD_STATE.update(status="error", message=f"build launch failed: {e}") + return + + if ok: + _activate_pass_slot(slot_id, pass_name, name) + with BUILD_LOCK: + BUILD_STATE.update(status="ok", slot_id=slot_id, + message=f"'{name}' compiled and activated as {slot_id}.") + else: + # Move the offending file aside so the next build is green again. + bad = os.path.join(PLUGIN_DIR, f"{pass_name}_pass.cpp") + if os.path.exists(bad): + shutil.move(bad, bad + ".rejected") + _regenerate_manifest() # drop the rejected plugin from the manifest + with BUILD_LOCK: + BUILD_STATE.update(status="error", + message=f"build failed; see {BUILD_LOG}. Plugin moved aside.") + + +def _activate_pass_slot(slot_id, pass_name, name): + """Create/replace a cost-based optimizer slot wired to OPT#.""" + entry = { + "id": slot_id, + "name": name, + "description": f"cost-based pass: {pass_name}", + "plan_key": "baseline", + "opt_setting": f"OPT#{pass_name}", + "custom": True, + "kind": "pass", # distinguishes uploaded passes from RULE# optimizers + } + existing = next((o for o in OPTIMIZERS if o["id"] == slot_id), None) + if existing: + existing.update(entry) + else: + OPTIMIZERS.append(entry) + OPTIMIZER_DEFINITIONS[slot_id] = ( + "// Cost-based optimizer pass (uploaded, compiled-in)\n" + f"WHEN {pass_name}.check(plan) // structural guard\n" + "CHOOSE plan minimizing the pass's internal cost model\n" + f"APPLY {pass_name} // setting: OPT#{pass_name}\n" + ) + ACTIVE_OPTIMIZER_IDS.add(slot_id) + if slot_id not in BENCHMARK_OPT_ORDER: + BENCHMARK_OPT_ORDER.append(slot_id) + BENCHMARK_RESULTS["latencies"].append([None] * len(BENCHMARK_RESULTS["queries"])) + BENCHMARK_LABELS[slot_id] = name + + +def _parse_synthetic_sql(): + """Return (preamble_stmts, select_stmt) parsed from synthetic_query.sql.""" + try: + with open(SYNTHETIC_SQL_PATH) as f: + content = f.read() + # Make the ATTACH path absolute so DuckDB finds it regardless of cwd + db_path = os.path.join( + PROJECT_ROOT, "queries/synthetic_query/synthetic_database.db" + ) + content = content.replace( + "'queries/synthetic_query/synthetic_database.db'", + f"'{db_path}'", + ) + stmts = [s.strip() for s in content.split(";") if s.strip()] + preamble, select = [], None + for stmt in stmts: + # Find first non-comment token to classify the statement + first_tok = "" + for line in stmt.split("\n"): + clean = line.split("--")[0].strip() + if clean: + parts = clean.split() + first_tok = parts[0].upper() if parts else "" + break + if first_tok == "SELECT": + select = stmt + elif first_tok == "SET" and "enable_extended_optimizer" in stmt.lower(): + pass # skip — we override this per-optimizer at runtime + else: + preamble.append(stmt) + return preamble, select + except Exception as e: + app.logger.warning("Could not parse synthetic SQL: %s", e) + return [], None + + +SQL_PREAMBLE, SQL_SELECT = _parse_synthetic_sql() + + +_PROFILING_KEYWORDS = ("enable_profiling", "profiling_output", "profiling_mode") + + +def _parse_uc_sql(path): + """Return (preamble_stmts, select_stmt) parsed from any UC query SQL file. + Strips LOAD, PRAGMA, profiling SETs, and SET enable_extended_optimizer — + all are either injected at runtime or irrelevant to plan visualization.""" + try: + with open(path) as f: + content = f.read() + stmts = [s.strip() for s in content.split(";") if s.strip()] + preamble, select = [], None + for stmt in stmts: + first_tok = "" + for line in stmt.split("\n"): + clean = line.split("--")[0].strip() + if clean: + parts = clean.split() + first_tok = parts[0].upper() if parts else "" + break + if not first_tok: + pass # comment-only chunk — skip + elif first_tok in ("SELECT", "WITH"): + select = stmt + elif first_tok in ("LOAD", "PRAGMA"): + pass # LOAD injected by _build_session_sql; PRAGMA not needed for EXPLAIN + elif first_tok == "SET" and "enable_extended_optimizer" in stmt.lower(): + pass # overridden per-optimizer at runtime + elif first_tok == "SET" and any(kw in stmt.lower() for kw in _PROFILING_KEYWORDS): + pass # profiling directives not needed for plan visualization + else: + preamble.append(stmt) + return preamble, select + except Exception as e: + app.logger.warning("Could not parse UC SQL %s: %s", path, e) + return [], None + + +UC01_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/uc01_query.sql") +UC03_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/uc03_query.sql") +UC04_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/uc04_query.sql") +UC08_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/uc08_query_factorizable.sql") +EXPEDIA_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/expedia_query.sql") +FLIGHTS_SQL_PATH = os.path.join(PROJECT_ROOT, "queries/flights_query.sql") + +UC01_PREAMBLE, UC01_SELECT = _parse_uc_sql(UC01_SQL_PATH) +UC03_PREAMBLE, UC03_SELECT = _parse_uc_sql(UC03_SQL_PATH) +UC04_PREAMBLE, UC04_SELECT = _parse_uc_sql(UC04_SQL_PATH) +UC08_PREAMBLE, UC08_SELECT = _parse_uc_sql(UC08_SQL_PATH) +EXPEDIA_PREAMBLE, EXPEDIA_SELECT = _parse_uc_sql(EXPEDIA_SQL_PATH) +FLIGHTS_PREAMBLE, FLIGHTS_SELECT = _parse_uc_sql(FLIGHTS_SQL_PATH) + + +# --- Demo in-memory data ---------------------------------------------------- + +QUERIES = [ + { + "id": "ranker_q1", + "name": "Q_Ranker: SQL+ML Ranking (NN over features)", + "sql": SQL_SELECT or """SELECT + s.impression_id AS impression_id, + m.listing_row_id AS listing_row_id, + m.property_type AS property_type, + ml_argmax( + cdb_softmax( + mat_add( + mat_mul( + mat_add( + mat_mul( + list_concat( + s.user_profile_vec, + s.search_context_vec, + s.listing_feature_vec + ), + 'queries/synthetic_query/nn_params_150_512_2/nn_W1.csv', + 'dense' + ), + 'queries/synthetic_query/nn_params_150_512_2/nn_b1.csv' + ), + 'queries/synthetic_query/nn_params_150_512_2/nn_W2.csv', + 'dense' + ), + 'queries/synthetic_query/nn_params_150_512_2/nn_b2.csv' + ) + ) + ) +FROM search_impressions AS s +JOIN listing_metadata AS m + USING (listing_id) +ORDER BY s.impression_id;""", + }, + { + "id": "q_expedia", + "name": "Q_Expedia: decision tree inference", + "sql": """SELECT + Expedia_S_listings_extension2.prop_id, + Expedia_S_listings_extension2.srch_id, + udf(prop_location_score1, prop_location_score2, prop_log_historical_price, price_usd, + orig_destination_distance, prop_review_score, avg_bookings_usd, stdev_bookings_usd, + position, prop_country_id, prop_starrating, prop_brand_bool, count_clicks, count_bookings, + year, month, weekofyear, time, site_id, visitor_location_country_id, srch_destination_id, + srch_length_of_stay, srch_booking_window, srch_adults_count, srch_children_count, + srch_room_count, srch_saturday_night_bool, random_bool) +FROM Expedia_S_listings_extension2 +JOIN Expedia_R1_hotels2 + ON Expedia_S_listings_extension2.prop_id = Expedia_R1_hotels2.prop_id +JOIN Expedia_R2_searches + ON Expedia_S_listings_extension2.srch_id = Expedia_R2_searches.srch_id +WHERE prop_location_score1 > 1 + AND prop_location_score2 > 0.1 + AND prop_log_historical_price > 4 + AND count_bookings > 5 + AND srch_booking_window > 10 + AND srch_length_of_stay > 1;""", + }, + { + "id": "q_flights", + "name": "Q_Flights: decision forest inference", + "sql": """SELECT + Flights_S_routes_extension2.airlineid, + Flights_S_routes_extension2.sairportid, + Flights_S_routes_extension2.dairportid, + udf(slatitude, slongitude, dlatitude, dlongitude, + name1, name2, name4, acountry, active, + scity, scountry, stimezone, sdst, + dcity, dcountry, dtimezone, ddst) AS codeshare +FROM Flights_S_routes_extension2 +JOIN Flights_R1_airlines2 + ON Flights_S_routes_extension2.airlineid = Flights_R1_airlines2.airlineid +JOIN Flights_R2_sairports + ON Flights_S_routes_extension2.sairportid = Flights_R2_sairports.sairportid +JOIN Flights_R3_dairports + ON Flights_S_routes_extension2.dairportid = Flights_R3_dairports.dairportid +WHERE name2 = 't' + AND name4 = 't' + AND name1 > 2.8;""", + }, + { + "id": "q_credit", + "name": "Q_Credit: XGBoost inference", + "sql": """SELECT + Time, + Amount, + udf(V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, + V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, Amount) AS Class +FROM Credit_Card_extension +WHERE V1 > 1 + AND V2 < 0.27 + AND V3 > 0.3;""", + }, + { + "id": "q_uc01", + "name": "Q_UC01: KMeans++ + min–max scaling", + "sql": """WITH table_groups AS ( + SELECT + MIN(EXTRACT(YEAR FROM CAST(date AS DATE))) AS invoice_year, + SUM(quantity * price) AS row_price, + SUM(COALESCE(or_return_quantity, 0) * price) AS return_row_price, + o_customer_sk AS customer_id, + o_order_id AS order_id + FROM lineitem li + LEFT JOIN order_returns oret + ON li.li_order_id = oret.or_order_id + AND li.li_product_id = oret.or_product_id + JOIN "order" ord + ON li.li_order_id = ord.o_order_id + GROUP BY o_customer_sk, o_order_id +), +table_ratios AS ( + SELECT + customer_id, + AVG(return_row_price / NULLIF(row_price, 0)) AS return_ratio + FROM table_groups + GROUP BY customer_id +), +table_frequency_groups AS ( + SELECT + customer_id, + invoice_year, + COUNT(DISTINCT order_id) AS orders_per_year + FROM table_groups + GROUP BY customer_id, invoice_year +), +table_frequency AS ( + SELECT + customer_id, + AVG(orders_per_year) AS frequency + FROM table_frequency_groups + GROUP BY customer_id +), +features_raw AS ( + SELECT + r.customer_id, + f.frequency, + r.return_ratio + FROM table_ratios r + JOIN table_frequency f + ON r.customer_id = f.customer_id +), +features AS ( + SELECT + customer_id, + minmax_apply('uc01_scaler', ARRAY[frequency, return_ratio]) AS features + FROM features_raw +) +SELECT + customer_id, + kmeans_predict('uc01_kmeans_model', features) AS cluster_id +FROM features;""", + }, + { + "id": "q_uc03", + "name": "Q_UC03: DNN over encoded store–department features", + "sql": """WITH base AS ( + SELECT + store, + department, + num_of_week, + store_id_encoder(CAST(store AS INTEGER)) AS store_id_encoded, + department_encoder(department) AS department_encoded, + CAST(num_of_week / 156.0 AS REAL) AS num_of_week_norm + FROM store_dept +), +feat AS ( + SELECT + store, + department, + num_of_week, + CAST(concat(store_id_encoded, department_encoded, num_of_week_norm) AS REAL[]) AS features + FROM base +) +SELECT + store, + department, + num_of_week, + dnn_predict('uc03_model', features) AS prediction +FROM feat;""", + }, + { + "id": "q_uc08", + "name": "Q_UC08: DNN over order features (softmax)", + "sql": """WITH ord AS ( + SELECT o_order_id, CAST(date AS TIMESTAMP) AS ts + FROM "order" +), +ord2 AS ( + SELECT o_order_id, ts, day_of_week(ts) AS weekday + FROM ord +), +j1 AS ( + SELECT + o.o_order_id, o.ts, o.weekday, + li.li_product_id, li.quantity + FROM ord2 o + JOIN lineitem li ON o.o_order_id = li.li_order_id +), +j2 AS ( + SELECT + j1.o_order_id, j1.ts, j1.weekday, + j1.quantity, p.department + FROM j1 + JOIN product p ON j1.li_product_id = p.p_product_id +), +agg AS ( + SELECT + o_order_id, + ts AS date, + department, + quantity, + SUM(quantity) AS scan_count, + MIN(weekday) AS weekday + FROM j2 + GROUP BY o_order_id, ts, department, quantity +), +feat AS ( + SELECT + o_order_id, + date, + CAST(concat(quantity, scan_count, weekday, department_encoder(department)) AS REAL[]) AS features + FROM agg +) +SELECT + o_order_id, + date, + softmax(dnn_predict('uc08_dnn_model', features)) AS prediction +FROM feat;""", + }, + { + "id": "q_uc10", + "name": "Q_UC10: DNN fraud detection (sigmoid)", + "sql": """WITH tx AS ( + SELECT + transaction_id, + sender_id, + CAST(hour(CAST(time AS TIMESTAMP)) AS DOUBLE) AS business_hour_norm, + amount + FROM financial_transactions +), +j AS ( + SELECT + tx.transaction_id, + tx.business_hour_norm, + tx.amount, + fa.transaction_limit + FROM financial_account fa + JOIN tx ON fa.fa_customer_sk = tx.sender_id +), +feat AS ( + SELECT + transaction_id, + CAST(concat(amount / transaction_limit, business_hour_norm) AS REAL[]) AS features + FROM j +) +SELECT + transaction_id, + sigmoid(dnn_predict('uc10_dnn_model', features)) AS prediction +FROM feat;""", + }, + { + "id": "q_uc04", + "name": "Q_UC04: Naive Bayes spam detection", + "sql": """WITH docs AS ( + SELECT + r.id, + ARRAY( + SELECT stem_token(t) + FROM UNNEST(tokenize(clean(lower(r.text)))) AS t + WHERE t <> '' AND t !~ '^[0-9]+$' + ) AS tokens + FROM review r +), +base_priors AS ( + SELECT + LN(SUM(CASE WHEN spam = 1 THEN 1 ELSE 0 END) * 1.0 / COUNT(*)) AS log_spam_prior, + LN(SUM(CASE WHEN spam = 0 THEN 1 ELSE 0 END) * 1.0 / COUNT(*)) AS log_non_spam_prior + FROM uc04_train_preprocessed +), +token_scores AS ( + SELECT + d.id, + SUM(LN(COALESCE(m.spam_prob, 1e-9))) AS log_spam_score, + SUM(LN(COALESCE(m.non_spam_prob, 1e-9))) AS log_non_spam_score + FROM docs d + CROSS JOIN UNNEST(d.tokens) AS tok(token) + LEFT JOIN uc04_model m + ON m.token = tok.token + GROUP BY d.id +) +SELECT + s.id, + CASE + WHEN s.log_spam_score + p.log_spam_prior + > s.log_non_spam_score + p.log_non_spam_prior + THEN 1 ELSE 0 + END AS predicted_spam +FROM token_scores s +CROSS JOIN base_priors p +ORDER BY s.id;""", + }, +] + +# --- Per-query runtime metadata ------------------------------------------- +# Generalize the query model so the live-plan path is query-driven rather than +# hardcoded to ranker_q1. A query is "runnable" if it has a loadable DB + +# preamble and a MetricCatalog schema; only ranker_q1 qualifies today. New +# queries light up by setting these fields (and adding a MetricCatalog schema). +for _q in QUERIES: + _q.setdefault("runnable", False) + _q.setdefault("schema", None) + _q.setdefault("preamble", []) + +for _q in QUERIES: + if _q["id"] == "ranker_q1": + _q["runnable"] = bool(SQL_SELECT) + _q["schema"] = "synthetic" + _q["preamble"] = SQL_PREAMBLE + if _q["id"] == "q_uc01": + _q["runnable"] = bool(UC01_SELECT) + _q["sql"] = UC01_SELECT or _q["sql"] + _q["preamble"] = UC01_PREAMBLE + if _q["id"] == "q_uc03": + _q["runnable"] = bool(UC03_SELECT) + _q["sql"] = UC03_SELECT or _q["sql"] + _q["preamble"] = UC03_PREAMBLE + if _q["id"] == "q_uc04": + _q["runnable"] = bool(UC04_SELECT) + _q["sql"] = UC04_SELECT or _q["sql"] + _q["preamble"] = UC04_PREAMBLE + if _q["id"] == "q_uc08": + _q["runnable"] = bool(UC08_SELECT) + _q["sql"] = UC08_SELECT or _q["sql"] + _q["preamble"] = UC08_PREAMBLE + if _q["id"] == "q_expedia": + _q["runnable"] = bool(EXPEDIA_SELECT) + _q["sql"] = EXPEDIA_SELECT or _q["sql"] + _q["preamble"] = EXPEDIA_PREAMBLE + if _q["id"] == "q_flights": + _q["runnable"] = bool(FLIGHTS_SELECT) + _q["sql"] = FLIGHTS_SELECT or _q["sql"] + _q["preamble"] = FLIGHTS_PREAMBLE + + +def _get_query(query_id): + return next((q for q in QUERIES if q["id"] == query_id), None) + + +# All available actions +ACTIONS = [ + "TorchNN2TwoLayerUDFRewriteAction", + "Mul2JoinAggHorizontalRewriteAction", + "DecisionForestUDF2RelationRewriteAction", + "Mul2JoinAggRewriteAction", + "MultiLayerUDF2TorchNNRewriteAction", + "MLDecompositionPushdownRewriteAction", + "MLFactorizationRewriteAction", + "MatMulDense2SparseRewriteAction", + "TreeModelTeePruningRewriteAction", +] + +# Optimizers catalog. +# opt_setting: the value passed to SET enable_extended_optimizer. +# "0" means no extended pass fires (baseline). Each digit maps to the +# corresponding if-block in register_optimizers.cpp CactusDynamicOpt::Run. +OPTIMIZERS = [ + { + "id": "noopt", + "name": "No optimizer (baseline)", + "description": "DuckDB default plan for the SQL+ML query", + "plan_key": "baseline", + "opt_setting": "0", + }, + { + "id": "opt1", + "name": "Optimizer1: ML decomposition pushdown", + "description": "Push NN below the join (dense matmul)", + "plan_key": "optimizer1", + "opt_setting": "1", + }, + { + "id": "opt2", + "name": "Optimizer2: ML decomposition + sparse matmul", + "description": "Push NN below the join and switch to sparse matmul", + "plan_key": "optimizer2", + "opt_setting": "2", + }, + { + "id": "opt_factorize", + "name": "Optimizer3: ML factorization (high FLOP queries)", + "description": "Split mat_mul across join boundary when ml_flops > 5B", + "plan_key": "baseline", + "opt_setting": "4", + }, +] + +# --- Optimizer definitions shown in the Optimizer Definition pane ---------- + +OPTIMIZER_DEFINITIONS = { + "noopt": """// Baseline: uses DuckDB's built-in optimizer only. +IF + [builtin] duckdb_default_optimizer_enabled +THEN + [action] (no extra SQL+ML rules) +""", + + "opt1": r"""IF + [metric] join_cardinality_ratio > 1e-5 +THEN + [action] MLDecompositionPushdownRewriteAction +""", + + "opt2": r"""IF + [metric] join_cardinality_ratio > 1e-5 +AND + [metric] feature_nonzero_ratio < 0.30 +THEN + [action] MLDecompositionPushdownRewriteAction + [action] MatMulDense2SparseRewriteAction +""", + + "opt_factorize": r"""IF + [metric] ml_flops > 5e9 +THEN + [action] MLFactorizationRewriteAction +""", +} + + +# Custom optimizer slots created from the frontend (max 5). +CUSTOM_SLOT_IDS = ["custom1", "custom2", "custom3", "custom4", "custom5"] + + +def parse_optimizer_rule(text): + """Parse the IF-THEN grammar into {'conditions': [...], 'actions': [...]}. + + Grammar: + IF + [metric] + AND|OR + [metric] + THEN + [action] + [action] + """ + conditions, actions = [], [] + state, pending_logic = "start", "AND" + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("//"): + continue + up = line.upper() + if up == "IF": + state = "condition" + elif up in ("AND", "OR"): + pending_logic = up + elif up == "THEN": + state = "action" + elif state == "condition" and line.lower().startswith("[metric]"): + parts = line[len("[metric]"):].split() + if len(parts) >= 3: + conditions.append({ + "metric": parts[0], + "op": parts[1], + "value": parts[2], + "logic": pending_logic, + }) + pending_logic = "AND" + elif state == "action" and line.lower().startswith("[action]"): + actions.append(line[len("[action]"):].strip()) + return {"conditions": conditions, "actions": actions} + + +def serialize_rule(parsed): + """Serialize a parsed rule to the DuckDB setting string consumed by the + C++ DynamicRule interpreter: RULE##.""" + cond = "" + for i, c in enumerate(parsed["conditions"]): + if i > 0: + cond += f" {c['logic']} " + cond += f"{c['metric']} {c['op']} {c['value']}" + return f"RULE#{cond}#{','.join(parsed['actions'])}" + + +ACTIVE_OPTIMIZER_IDS = {"noopt", "opt1", "opt2", "opt_factorize"} + +BENCHMARK_OPT_ORDER = ["noopt", "opt1", "opt2", "opt_factorize"] +BENCHMARK_LABELS = { + "noopt": "NoOptimizer", + "opt1": "Optimizer1", + "opt2": "Optimizer2", + "opt_factorize": "Optimizer3", +} + +METRICS = { + "ranker_q1": { + "card_search_impressions": 500_000, + "card_listing_metadata": 200_000, + "join_cardinality_ratio": 0.0002, + "feature_nonzero_ratio": 0.002, + "feature_width": "150", + }, + "q_uc01": { + "customers": "7,071", + "orders": "367,695", + "lineitems": "2,306,263", + "order_returns": "134,204", + "features": "2 (frequency, return_ratio)", + "kmeans_k": "5", + }, + "q_uc04": { + "inference_reviews": "2,687", + "training_reviews": "10,746", + "vocab_size": "18,925", + "spam_ratio": "48.6%", + "model": "Naive Bayes (Laplace smoothed)", + }, + "q_uc03": { + "cardinality": "38,896", + "nnz_ratio": "0.025", + "zero_rows": "0.0", + "zero_cols": "0.0", + "ml_flops": "280 M", + "num_parameters": "7,297", + }, + "q_uc08": { + "cardinality": "2,122,088", + "join_cardinality_ratio": "0.0003", + "nnz_ratio": "0.021", + "zero_rows": "0.0", + "zero_cols": "0.0", + "ml_flops": "6.79 B", + "num_parameters": "3,200", + }, +} + + +# Box-shaped DuckDB plans (fallback when live DuckDB execution is unavailable) +PLANS = { + "baseline": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ ml_argmax(cdb_softmax │ +│ (mat_add(mat_mul(mat_add │ +│ (mat_mul(list_concat │ +│ (user_profile_vec, │ +│ search_context_vec, │ +│ listing_feature_vec), ' │ +│ nn_params_150_512_2/nn_W1 │ +│ .csv', 'dense'), 'nn_b1 │ +│ .csv'), 'nn_W2.csv', │ +│ 'dense'), 'nn_b2.csv'))) │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ Table: ││ Table: │ +│ search_impressions ││ listing_metadata │ +│ ││ │ +│ Type: Sequential Scan ││ Type: Sequential Scan │ +└───────────────────────────┘└───────────────────────────┘""", + + "optimizer1": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ #5 │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ PROJECTION ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ ml_argmax(cdb_softmax ││ Table: │ +│ (mat_add(mat_mul(mat_add ││ listing_metadata │ +│ (mat_mul(list_concat ││ │ +│ ...), 'dense'), ...))) ││ Type: Sequential Scan │ +└─────────────┬─────────────┘└───────────────────────────┘ +┌─────────────┴─────────────┐ +│ SEQ_SCAN │ +│ ──────────────────── │ +│ Table: │ +│ search_impressions │ +│ │ +│ Type: Sequential Scan │ +└───────────────────────────┘""", + + "optimizer2": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ #5 │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ PROJECTION ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ ml_argmax(cdb_softmax ││ Table: │ +│ (mat_add(mat_mul(mat_add ││ listing_metadata │ +│ (mat_mul(list_concat ││ │ +│ ...), 'sparse'), ...))) ││ Type: Sequential Scan │ +└─────────────┬─────────────┘└───────────────────────────┘ +┌─────────────┴─────────────┐ +│ SEQ_SCAN │ +│ ──────────────────── │ +│ Table: │ +│ search_impressions │ +│ │ +│ Type: Sequential Scan │ +└───────────────────────────┘""", +} + +# Per-query static plans keyed by (query_id, optimizer_id). +# Used by api_query_plan before falling back to the generic PLANS dict. +QUERY_PLANS = { + ("q_uc01", "noopt"): """\ +┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ customer_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ customer_id, │ +│ kmeans_predict( │ +│ min_max_scaler( │ +│ [frequency, │ +│ return_ratio], │ +│ uc01_scaler.csv), │ +│ uc01_centroids.csv) │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ r.customer_id = ├──────────────┐ +│ f.customer_id │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ AGGREGATE ││ AGGREGATE │ +│ AVG(orders_per_year) ││ AVG(ret_price/ │ +│ GROUP BY customer_id ││ row_price) │ +└─────────────┬─────────────┘└─────────────┬─────────────┘ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ COMPARISON_JOIN ││ HASH_JOIN │ +│ li_order_id=o_order_id ││ li_order_id=or_order_id │ +│ (+ LEFT JOIN ││ li_product_id= │ +│ order_returns) ││ or_product_id │ +└─────────────┬─────────────┘└─────────────┬─────────────┘ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ Table: lineitem ││ Table: order_returns │ +│ (2,306,263 rows) ││ (134,204 rows) │ +└───────────────────────────┘└───────────────────────────┘""", + + ("q_uc04", "noopt"): """\ +┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ id, │ +│ CASE WHEN │ +│ log_spam_score │ +│ + log_spam_prior │ +│ > log_ham_score │ +│ + log_ham_prior │ +│ THEN 1 ELSE 0 END │ +│ AS predicted_spam │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ CROSS_JOIN │ +│ token_scores × priors │ +└──────────┬────────────────┘ +┌──────────┴────────────────┐┌──────────────────────────────┐ +│ AGGREGATE ││ SEQ_SCAN │ +│ SUM(LN(spam_prob)), ││ Table: uc04_train_ │ +│ SUM(LN(non_spam_prob)) ││ preprocessed │ +│ GROUP BY id ││ (10,746 rows) │ +└──────────┬────────────────┘└──────────────────────────────┘ +┌──────────┴────────────────┐ +│ HASH_JOIN │ +│ ──────────────────── │ +│ Join Type: LEFT │ +│ token = token │ +└──────────┬────────────────┘ +┌──────────┴────────────────┐┌──────────────────────────────┐ +│ UNNEST(tokenize_text( ││ SEQ_SCAN │ +│ clean_text( ││ Table: uc04_model │ +│ stem_token(text)))) ││ (18,925 tokens) │ +└──────────┬────────────────┘└──────────────────────────────┘ +┌──────────┴────────────────┐ +│ SEQ_SCAN │ +│ Table: review │ +│ (2,687 rows) │ +└───────────────────────────┘""", + + ("q_uc08", "noopt"): """\ +┌───────────────────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ o_order_id, date, h1 │ +│ mat_mul( │ +│ list_concat( │ +│ [qty, scan_count, wd], │ +│ one_hot_encode(dept) │ +│ ), W1_wide, 'dense') │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ li_product_id = ├────────────────────────────┐ +│ p_product_id │ │ +└─────────────┬─────────────┘ ┌─────────────────┴──────────────┐ +┌─────────────┴─────────────┐ │ SEQ_SCAN │ +│ PROJECTION (agg cols) │ │ Table: product │ +└─────────────┬─────────────┘ │ (707 rows) │ +┌─────────────┴─────────────┐ └────────────────────────────────┘ +│ AGGREGATE │ +│ SUM(qty), MIN(weekday) │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ o_order_id=li_order_id ├──────────────┐ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ Table: order_tbl ││ Table: lineitem │ +└───────────────────────────┘└───────────────────────────┘""", + + ("q_uc08", "opt_factorize"): """\ +┌───────────────────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ o_order_id, date, │ +│ list_add_vv(#left, #right│ +│ ) ← replaces mat_mul │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ li_product_id = ├────────────────────────────┐ +│ p_product_id │ │ +└─────────────┬─────────────┘ ┌─────────────────┴──────────────┐ +┌─────────────┴─────────────┐ │ PROJECTION │ +│ PROJECTION │ │ mat_mul( │ +│ mat_mul( │ │ one_hot_encode(dept), │ +│ [qty, scan_count, wd], │ │ W1_right[47×64]) │ +│ W1_left[3×64]) │ │ (707 rows — amortized!) │ +│ (2,122,088 rows) │ └─────────────────┬──────────────┘ +└─────────────┬─────────────┘ ┌─────────────────┴──────────────┐ +┌─────────────┴─────────────┐ │ SEQ_SCAN │ +│ AGGREGATE │ │ Table: product │ +│ SUM(qty), MIN(weekday) │ └────────────────────────────────┘ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ o_order_id=li_order_id ├──────────────┐ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ Table: order_tbl ││ Table: lineitem │ +└───────────────────────────┘└───────────────────────────┘""", +} + +# Benchmark results (latency in ms) — hardcoded for the full suite display +BENCHMARK_RESULTS = { + "queries": [ + "Q1", "Q2", "Q3", "Q4", "Q_UC01", "Q_UC03", "Q_UC08", "Q8", "Q_UC04", "Q10", + "Q_UC03", "Q_UC08", + ], + "query_ids": [ + "ranker_q1", "q_expedia", "q_flights", "q_credit", "q_uc01", + "q_uc03", "q_uc08", "q_uc10", "q_uc04", "ranker_q1", + "q_uc03", "q_uc08", + ], + "query_types": [ + "ml", "sql", "ml", "ml", "ml", "ml", "ml", "sql", "ml", "sql", + "ml", "ml", + ], + "optimizers": ["NoOptimizer", "Optimizer1", "Optimizer2", "Optimizer3"], + "latencies": [ + [85000.0, 12000.0, 60000.0, 15000.0, 4800.0, 18000.0, 40000.0, 22000.0, 620.0, 5000.0, 864.0, 1030.0], + [3674.0, 7000.0, 9000.0, 8000.0, 4800.0, 9000.0, 12000.0, 15000.0, 620.0, 4500.0, 864.0, 1030.0], + [1976.0, 6800.0, 6000.0, 7800.0, 4800.0, 8500.0, 8000.0, 14000.0, 620.0, 4400.0, 864.0, 1030.0], + [1976.0, 6800.0, 6000.0, 7800.0, 4800.0, 8500.0, 8000.0, 14000.0, 620.0, 4400.0, 864.0, 924.0], + ], +} + + +# --- DuckDB helpers --------------------------------------------------------- + + +def _build_session_sql(opt_setting: str, body: str, preamble=None) -> str: + """Assemble a complete DuckDB session script: extension load, preamble, + optimizer setting, then the body (EXPLAIN or SELECT). The preamble is the + selected query's own setup (defaults to the synthetic query's preamble).""" + if preamble is None: + preamble = SQL_PREAMBLE + # Use the bare extension name so DuckDB resolves it relative to its own + # binary directory (build/release/extension/cactusdb/) — same as the SQL + # files themselves use when run directly from the command line. + lines = ["LOAD 'cactusdb';"] + for stmt in preamble: + lines.append(stmt + ";") + lines.append(f"SET enable_extended_optimizer = '{opt_setting}';") + lines.append(body + ";") + return "\n".join(lines) + + +def _run_duckdb(sql: str) -> subprocess.CompletedProcess: + """Run a SQL script through the bundled DuckDB CLI (project build). + The CLI is invoked from PROJECT_ROOT so relative paths in SQL resolve.""" + return subprocess.run( + [DUCKDB_BIN, "-unsigned"], + input=sql, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + ) + + +# --- Routes ----------------------------------------------------------------- + + +@app.route("/", methods=["GET"]) +def index(): + selected_query_id = request.args.get("query_id", QUERIES[0]["id"]) + selected_query = next(q for q in QUERIES if q["id"] == selected_query_id) + selected_metrics = METRICS.get(selected_query["id"], {}) + + optimizer_left_id = request.args.get("optimizer_left_id", "noopt") + optimizer_right_id = request.args.get("optimizer_right_id", "opt1") + + definition_opt_id = request.args.get("definition_opt_id", "opt1") + if definition_opt_id not in OPTIMIZER_DEFINITIONS: + definition_opt_id = "opt1" + rules_snippet = OPTIMIZER_DEFINITIONS.get(definition_opt_id, "") + + # Cost-based passes (OPT#...) aren't editable through the IF-THEN form; the + # definition window shows them read-only and points the user at re-uploading. + definition_opt = next((o for o in OPTIMIZERS if o["id"] == definition_opt_id), None) + definition_is_pass = bool(definition_opt and definition_opt.get("kind") == "pass") + + def get_opt(opt_id, default_id): + if opt_id not in ACTIVE_OPTIMIZER_IDS: + opt_id = default_id + opt = next((o for o in OPTIMIZERS if o["id"] == opt_id), None) + if opt is None: + opt = next(o for o in OPTIMIZERS if o["id"] == default_id) + return opt + + active_optimizers = [o for o in OPTIMIZERS if o["id"] in ACTIVE_OPTIMIZER_IDS] + selected_optimizer_left = get_opt(optimizer_left_id, "noopt") + selected_optimizer_right = get_opt(optimizer_right_id, "opt1") + + # Custom-slot state for the optimizer builder UI. + used_slots = {o["id"] for o in OPTIMIZERS if o["id"] in CUSTOM_SLOT_IDS} + custom_slots = [ + {"id": s, "used": s in used_slots, + "name": next((o["name"] for o in OPTIMIZERS if o["id"] == s), "")} + for s in CUSTOM_SLOT_IDS + ] + + return render_template( + "index.html", + queries=QUERIES, + actions=ACTIONS, + optimizers=active_optimizers, + selected_query=selected_query, + metrics=selected_metrics, + selected_optimizer_left=selected_optimizer_left, + selected_optimizer_right=selected_optimizer_right, + rules_snippet=rules_snippet, + definition_optimizer_id=definition_opt_id, + definition_is_pass=definition_is_pass, + custom_slots=custom_slots, + ) + + +def _extract_explain_output(raw: str) -> str: + """Return the DuckDB EXPLAIN plan tree, stripping the 'Physical Plan' title box.""" + lines = raw.splitlines() + # Strip the "Physical Plan" title box (always the first 5-line box DuckDB emits). + phys_idx = next((i for i, l in enumerate(lines) if "Physical Plan" in l), None) + if phys_idx is not None: + # Advance past the closing └ of the title box, then drop leading blank lines. + end = phys_idx + while end < len(lines) and not lines[end].startswith("└"): + end += 1 + lines = lines[end + 1:] + return "\n".join(lines).strip() + + +@app.route("/api/query_plan") +def api_query_plan(): + """Return the DuckDB physical plan for a query+optimizer pair. + Falls back to the hardcoded PLANS dict if live execution fails.""" + query_id = request.args.get("query_id", QUERIES[0]["id"]) + optimizer_id = request.args.get("optimizer_id", "noopt") + + opt = next((o for o in OPTIMIZERS if o["id"] == optimizer_id), None) + if opt is None: + return jsonify({"error": "Unknown optimizer"}), 400 + + query = _get_query(query_id) + if query and query.get("runnable") and query.get("sql"): + try: + sql = _build_session_sql( + opt["opt_setting"], + f"EXPLAIN {query['sql']}", + preamble=query.get("preamble"), + ) + proc = _run_duckdb(sql) + if proc.returncode == 0 and proc.stdout.strip(): + return jsonify({"plan": _extract_explain_output(proc.stdout)}) + app.logger.warning("Live plan failed (rc=%d): %s", + proc.returncode, proc.stderr[:300]) + except Exception as e: + app.logger.warning("Live plan failed, using fallback: %s", e) + + plan = PLANS.get(opt.get("plan_key", "baseline"), "No plan available.") + return jsonify({"plan": plan, "fallback": True}) + + +@app.route("/api/run_benchmark") +def api_run_benchmark(): + """Run the selected query under every active optimizer and return runtimes.""" + query_id = request.args.get("query_id", QUERIES[0]["id"]) + + query = _get_query(query_id) + if not query or not query.get("runnable") or not query.get("sql"): + return jsonify({"error": "Live benchmarking is not supported for this query yet"}), 400 + + results = [] + for opt in [o for o in OPTIMIZERS if o["id"] in ACTIVE_OPTIMIZER_IDS]: + try: + sql = _build_session_sql(opt["opt_setting"], query["sql"], + preamble=query.get("preamble")) + t0 = time.perf_counter() + proc = _run_duckdb(sql) + runtime_ms = round((time.perf_counter() - t0) * 1000, 1) + if proc.returncode != 0: + raise RuntimeError(proc.stderr[:300]) + results.append({ + "optimizer_id": opt["id"], + "name": opt["name"], + "runtime_ms": runtime_ms, + }) + except Exception as e: + app.logger.error("Benchmark failed for %s: %s", opt["id"], e) + results.append({ + "optimizer_id": opt["id"], + "name": opt["name"], + "runtime_ms": None, + "error": str(e), + }) + + return jsonify({"query_id": query_id, "results": results}) + + +@app.route("/create_optimizer", methods=["POST"]) +def create_optimizer(): + """Create or edit a custom optimizer from an IF-THEN rule. The rule is + serialized to a RULE# setting string that the C++ DynamicRule interpreter + evaluates against the query's metrics and applies as rewrite actions.""" + name = request.form.get("optimizer_name", "").strip() + rules_text = request.form.get("optimizer_definition", "").strip() + target = request.form.get("target_slot", "").strip() + + if not name or not rules_text: + flash("Optimizer name and rule definition are both required.", "danger") + return redirect(url_for("index")) + + parsed = parse_optimizer_rule(rules_text) + + # --- validation --- + valid_metrics = set().union(*[set(m) for m in METRICS.values()]) if METRICS else set() + for c in parsed["conditions"]: + if valid_metrics and c["metric"] not in valid_metrics: + flash(f"Unknown metric '{c['metric']}'. Known: {', '.join(sorted(valid_metrics))}.", "danger") + return redirect(url_for("index")) + try: + float(c["value"]) + except ValueError: + flash(f"Threshold '{c['value']}' for '{c['metric']}' is not numeric.", "danger") + return redirect(url_for("index")) + if not parsed["actions"]: + flash("Rule has no [action] lines.", "danger") + return redirect(url_for("index")) + bad = [a for a in parsed["actions"] if a not in ACTIONS] + if bad: + flash(f"Unknown action(s): {', '.join(bad)}.", "danger") + return redirect(url_for("index")) + + # --- choose slot: edit target, else next free, else cap reached --- + used = {o["id"] for o in OPTIMIZERS if o["id"] in CUSTOM_SLOT_IDS} + if target in CUSTOM_SLOT_IDS: + slot_id = target + else: + free = [s for s in CUSTOM_SLOT_IDS if s not in used] + if not free: + flash("All 5 custom optimizer slots are in use. Pick one to overwrite.", "danger") + return redirect(url_for("index")) + slot_id = free[0] + + opt_setting = serialize_rule(parsed) + description = ", ".join(parsed["actions"][:2]) + ("…" if len(parsed["actions"]) > 2 else "") + entry = { + "id": slot_id, + "name": name, + "description": description, + "plan_key": "baseline", + "opt_setting": opt_setting, + "custom": True, + } + + existing = next((o for o in OPTIMIZERS if o["id"] == slot_id), None) + if existing: + existing.update(entry) + else: + OPTIMIZERS.append(entry) + + OPTIMIZER_DEFINITIONS[slot_id] = rules_text + ACTIVE_OPTIMIZER_IDS.add(slot_id) + if slot_id not in BENCHMARK_OPT_ORDER: + BENCHMARK_OPT_ORDER.append(slot_id) + BENCHMARK_RESULTS["latencies"].append([None] * len(BENCHMARK_RESULTS["queries"])) + BENCHMARK_LABELS[slot_id] = name + + flash(f"Optimizer '{name}' saved to {slot_id}.", "success") + return redirect(url_for("index", definition_opt_id=slot_id, + optimizer_left_id="noopt", optimizer_right_id=slot_id)) + + +@app.route("/upload_optimizer", methods=["POST"]) +def upload_optimizer(): + """Upload a self-contained cost-based optimizer pass (.cpp + optional .hpp), + validate it, drop it into plugins/opt_passes/, rebuild the extension, and on a + green build activate it as a selectable optimizer (OPT#).""" + file = request.files.get("optimizer_file") + if not file or file.filename == "": + flash("No optimizer .cpp selected.", "danger") + return redirect(url_for("index")) + + with BUILD_LOCK: + if BUILD_STATE["status"] == "building": + flash("A build is already in progress. Wait for it to finish.", "warning") + return redirect(url_for("index")) + + name = request.form.get("optimizer_name", "").strip() + expected = request.form.get("pass_name", "").strip() + + text = file.read().decode("utf-8", errors="replace") + pass_name, err = _lint_plugin_source(text) + if err: + flash(err, "danger") + return redirect(url_for("index")) + if expected and expected != pass_name: + flash(f"Declared pass name '{expected}' != REGISTER_OPT_PASS name '{pass_name}'.", "danger") + return redirect(url_for("index")) + if not name: + name = f"Greedy/{pass_name}" + + # --- choose a custom slot (shared with the rule-based builder) --- + used = {o["id"] for o in OPTIMIZERS if o["id"] in CUSTOM_SLOT_IDS} + existing_for_pass = next( + (o["id"] for o in OPTIMIZERS if o.get("opt_setting") == f"OPT#{pass_name}"), None + ) + if existing_for_pass: + slot_id = existing_for_pass # re-upload of the same pass overwrites + else: + free = [s for s in CUSTOM_SLOT_IDS if s not in used] + if not free: + flash("All 5 custom optimizer slots are in use. Free one first.", "danger") + return redirect(url_for("index")) + slot_id = free[0] + + # --- place the sources where CMake globs them --- + os.makedirs(PLUGIN_INCLUDE_DIR, exist_ok=True) + cpp_path = os.path.join(PLUGIN_DIR, f"{pass_name}_pass.cpp") + with open(cpp_path, "w") as f: + f.write(text) + header = request.files.get("header_file") + if header and header.filename: + header.save(os.path.join(PLUGIN_INCLUDE_DIR, secure_filename(header.filename))) + _regenerate_manifest() # make the static link retain this plugin + + # --- kick off the rebuild asynchronously --- + with BUILD_LOCK: + BUILD_STATE.update(status="building", pass_name=pass_name, slot_id=slot_id, + message=f"Building extension with '{pass_name}'…") + threading.Thread( + target=_run_build_async, args=(slot_id, pass_name, name), daemon=True + ).start() + + flash(f"Uploaded '{pass_name}'. Building extension — watch the build status.", "info") + return redirect(url_for("index")) + + +@app.route("/api/build_status") +def api_build_status(): + """Build status + a tail of the compiler log for the upload UI to poll. + A terminal 'ok' is one-shot: reported once, then reset to 'idle' so the + client reload doesn't re-trigger. 'error' persists so the user can read it.""" + with BUILD_LOCK: + state = dict(BUILD_STATE) + if BUILD_STATE["status"] == "ok": + BUILD_STATE.update(status="idle", message="") + tail = "" + try: + with open(BUILD_LOG) as f: + tail = "".join(f.readlines()[-40:]) + except OSError: + pass + state["log_tail"] = tail + return jsonify(state) + + +@app.route("/upload_action", methods=["POST"]) +def upload_action(): + file = request.files.get("action_file") + if not file or file.filename == "": + flash("No action file selected.", "danger") + return redirect(url_for("index")) + + filename = secure_filename(file.filename) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + flash(f"Action definition '{filename}' uploaded successfully.", "success") + return redirect(url_for("index")) + + +@app.route("/api/query_stats") +def api_query_stats(): + """Return statistics for a given query_id (used by the live stats panel).""" + query_id = request.args.get("query_id", "") + return jsonify({"stats": METRICS.get(query_id, {})}) + + +@app.route("/api/benchmark_data") +def api_benchmark_data(): + queries = BENCHMARK_RESULTS["queries"] + query_types = BENCHMARK_RESULTS["query_types"] + base_latencies = BENCHMARK_RESULTS["latencies"] + + active_labels = [] + active_latencies = [] + + for row_idx, opt_id in enumerate(BENCHMARK_OPT_ORDER): + if opt_id in ACTIVE_OPTIMIZER_IDS: + active_labels.append(BENCHMARK_LABELS[opt_id]) + active_latencies.append(base_latencies[row_idx]) + + return jsonify({ + "queries": queries, + "query_ids": BENCHMARK_RESULTS.get("query_ids", []), + "query_types": query_types, + "optimizers": active_labels, + "latencies": active_latencies, + }) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/Frontend/archive/app.py b/Frontend/archive/app.py new file mode 100644 index 0000000..d81f5f8 --- /dev/null +++ b/Frontend/archive/app.py @@ -0,0 +1,454 @@ +from flask import Flask, render_template, request, redirect, url_for, flash, jsonify +from werkzeug.utils import secure_filename +import os + +app = Flask(__name__) +app.secret_key = "demo-secret-key" + +# Where uploaded files will go (for demo purposes) +UPLOAD_FOLDER = "uploads" +os.makedirs(UPLOAD_FOLDER, exist_ok=True) +app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER + +# --- Demo in-memory data ---------------------------------------------------- +# Single SQL+ML query: 150-dim NN over feature vectors with join + +QUERIES = [ + { + "id": "ranker_q1", + "name": "Q1: SQL+ML Ranking (NN over features)", + "sql": """SELECT + s.impression_id AS impression_id, + m.listing_row_id AS listing_row_id, + m.property_type AS property_type, + ml_argmax( + cdb_softmax( + -- layer 2: (h * W2) + b2 -> 1 x 2 logits + mat_add( + mat_mul( + -- layer 1: (x * W1) + b1 -> 1 x 16 hidden + mat_add( + mat_mul( + list_concat( + s.user_profile_vec, + s.search_context_vec, + s.listing_feature_vec + ), -- 150-dim feature vector + '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/queries/synthetic_query/nn_params_150_512_2/nn_W1.csv', + 'dense' + ), + '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/queries/synthetic_query/nn_params_150_512_2/nn_b1.csv' + ), + '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/queries/synthetic_query/nn_params_150_512_2/nn_W2.csv', + 'dense' + ), + '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/queries/synthetic_query/nn_params_150_512_2/nn_b2.csv' + ) + ) + ) +FROM search_impressions AS s +JOIN listing_metadata AS m + USING (listing_id) +ORDER BY s.impression_id;""" + } +] + +# All available actions +ACTIONS = [ + "TorchNN2TwoLayerUDFRewriteAction", + "Mul2JoinAggHorizontalRewriteAction", + "DecisionForestUDF2RelationRewriteAction", + "Mul2JoinAggRewriteAction", + "MultiLayerUDF2TorchNNRewriteAction", + "MLDecompositionPushdownRewriteAction", + "MatMulDense2SparseRewriteAction", + "TreeModelTeePruningRewriteAction", +] + +# Optimizers catalog (baseline + two configs) +OPTIMIZERS = [ + { + "id": "noopt", + "name": "No optimizer (baseline)", + "description": "DuckDB default plan for the SQL+ML query", + "plan_key": "baseline", + }, + { + "id": "opt1", + "name": "Optimizer1: ML decomposition pushdown", + "description": "Push NN below the join (dense matmul)", + "plan_key": "optimizer1", + }, + { + "id": "opt2", + "name": "Optimizer2: ML decomposition + sparse matmul", + "description": "Push NN below the join and switch to sparse matmul", + "plan_key": "optimizer2", + }, +] + +# Metrics for this query +METRICS = { + "ranker_q1": { + "card_search_impressions": 500_000, + "card_listing_metadata": 200_000, + "join_cardinality": 20_006_670, + "feature_nonzero_ratio": 0.002, + "feature_width": "150", + } +} + +# Box-shaped DuckDB plans +PLANS = { + "baseline": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ ml_argmax(cdb_softmax │ +│ (mat_add(mat_mul(mat_add │ +│ (mat_mul(list_concat │ +│ (user_profile_vec, │ +│ search_context_vec, │ +│ listing_feature_vec), ' │ +│ /home/local/ASUAD/jrtandel│ +│ /cactusdb-duckdb-extension│ +│ /queries/synthetic_query │ +│ /nn_params_150_512_2/nn_W1│ +│ .csv', 'dense'), '/home │ +│ /local/ASUAD/jrtandel │ +│ /cactusdb-duckdb-extension│ +│ /queries/synthetic_query │ +│ /nn_params_150_512_2/nn_b1│ +│ .csv'), '/home/local/ASUAD│ +│ /jrtandel/cactusdb-duckdb │ +│ -extension/queries │ +│ /synthetic_query │ +│ /nn_params_150_512_2/nn_W2│ +│ .csv', 'dense'), '/home │ +│ /local/ASUAD/jrtandel │ +│ /cactusdb-duckdb-extension│ +│ /queries/synthetic_query │ +│ /nn_params_150_512_2/nn_b2│ +│ .csv'))) │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ SEQ_SCAN ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ Table: ││ Table: │ +│ search_impressions ││ listing_metadata │ +│ ││ │ +│ Type: Sequential Scan ││ Type: Sequential Scan │ +└───────────────────────────┘└───────────────────────────┘""", + + "optimizer1": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ #5 │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ PROJECTION ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ Expressions: ││ Table: │ +│ #0 ││ listing_metadata │ +│ #1 ││ │ +│ #2 ││ Type: Sequential Scan │ +│ #3 ││ │ +│ #4 ││ │ +│ ml_argmax(cdb_softmax ││ │ +│ (mat_add(mat_mul(mat_add ││ │ +│ (mat_mul(list_concat ││ │ +│ (user_profile_vec, ││ │ +│ search_context_vec, ││ │ +│ listing_feature_vec), ' ││ │ +│ /home/local/ASUAD/jrtandel││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_W1││ │ +│ .csv', 'dense'), '/home ││ │ +│ /local/ASUAD/jrtandel ││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_b1││ │ +│ .csv'), '/home/local/ASUAD││ │ +│ /jrtandel/cactusdb-duckdb ││ │ +│ -extension/queries ││ │ +│ /synthetic_query ││ │ +│ /nn_params_150_512_2/nn_W2││ │ +│ .csv', 'dense'), '/home ││ │ +│ /local/ASUAD/jrtandel ││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_b2││ │ +│ .csv'))) ││ │ +└─────────────┬─────────────┘└───────────────────────────┘ +┌─────────────┴─────────────┐ +│ SEQ_SCAN │ +│ ──────────────────── │ +│ Table: │ +│ search_impressions │ +│ │ +│ Type: Sequential Scan │ +└───────────────────────────┘""", + + "optimizer2": """┌───────────────────────────┐ +│ ORDER_BY │ +│ ──────────────────── │ +│ s.impression_id │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ PROJECTION │ +│ ──────────────────── │ +│ Expressions: │ +│ impression_id │ +│ listing_row_id │ +│ property_type │ +│ #5 │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ +│ COMPARISON_JOIN │ +│ ──────────────────── │ +│ Join Type: INNER │ +│ ├──────────────┐ +│ Conditions: │ │ +│ (listing_id = listing_id) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ PROJECTION ││ SEQ_SCAN │ +│ ──────────────────── ││ ──────────────────── │ +│ Expressions: ││ Table: │ +│ #0 ││ listing_metadata │ +│ #1 ││ │ +│ #2 ││ Type: Sequential Scan │ +│ #3 ││ │ +│ #4 ││ │ +│ ml_argmax(cdb_softmax ││ │ +│ (mat_add(mat_mul(mat_add ││ │ +│ (mat_mul(list_concat ││ │ +│ (user_profile_vec, ││ │ +│ search_context_vec, ││ │ +│ listing_feature_vec), ' ││ │ +│ /home/local/ASUAD/jrtandel││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_W1││ │ +│ .csv', 'sparse'), '/home ││ │ +│ /local/ASUAD/jrtandel ││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_b1││ │ +│ .csv'), '/home/local/ASUAD││ │ +│ /jrtandel/cactusdb-duckdb ││ │ +│ -extension/queries ││ │ +│ /synthetic_query ││ │ +│ /nn_params_150_512_2/nn_W2││ │ +│ .csv', 'sparse'), '/home ││ │ +│ /local/ASUAD/jrtandel ││ │ +│ /cactusdb-duckdb-extension││ │ +│ /queries/synthetic_query ││ │ +│ /nn_params_150_512_2/nn_b2││ │ +│ .csv'))) ││ │ +└─────────────┬─────────────┘└───────────────────────────┘ +┌─────────────┴─────────────┐ +│ SEQ_SCAN │ +│ ──────────────────── │ +│ Table: │ +│ search_impressions │ +│ │ +│ Type: Sequential Scan │ +└───────────────────────────┘""", +} + +# Rule snippet shown as if built via drag-and-drop +RULE_SNIPPET = r""" +IF + [metric] join_ratio > 1e-5 +AND + [metric] feature_nonzero_ratio < 0.30 +THEN + [action] MLDecompositionPushdownRewriteAction + [action] MatMulDense2SparseRewriteAction +""" + +# Benchmark results (latency in ms) for this query +BENCHMARK_RESULTS = { + "queries": [ + "Q1", + "Q2", + "Q3", + "Q4", + "Q5", + "Q6", + "Q7", + "Q8", + "Q9", + "Q10", + ], + # Crude labels for filtering in the UI + "query_types": [ + "ml", # Q1 + "sql", # Q2 + "ml", # Q3 + "ml", # Q4 + "sql", # Q5 + "ml", # Q6 + "ml", # Q7 + "sql", # Q8 + "ml", # Q9 + "sql", # Q10 + ], + "optimizers": ["NoOptimizer", "Optimizer1", "Optimizer2"], + # Latencies in milliseconds: [optimizer][query_idx] + "latencies": [ + # NoOptimizer (baseline / no extended optimizations) + [ + 85000.0, # Q1: SQL+ML Ranker (real: 1 min 25 s) + 12000.0, # Q2 + 60000.0, # Q3 + 15000.0, # Q4 + 20000.0, # Q5 + 18000.0, # Q6 + 40000.0, # Q7 + 22000.0, # Q8 + 30000.0, # Q9 + 5000.0, # Q10 + ], + # Optimizer1: ML decomposition pushdown (dense matmul) + [ + 3674.0, # Q1 (real: 3.674 s) + 7000.0, # Q2 + 9000.0, # Q3 + 8000.0, # Q4 + 12000.0, # Q5 + 9000.0, # Q6 + 12000.0, # Q7 + 15000.0, # Q8 + 11000.0, # Q9 + 4500.0, # Q10 + ], + # Optimizer2: ML decomposition + sparse matmul + [ + 1976.0, # Q1 (real: 1.976 s) + 6800.0, # Q2 + 6000.0, # Q3 + 7800.0, # Q4 + 11500.0, # Q5 + 8500.0, # Q6 + 8000.0, # Q7 + 14000.0, # Q8 + 9000.0, # Q9 + 4400.0, # Q10 + ], + ], +} + + + +# --- Routes ----------------------------------------------------------------- + + +@app.route("/", methods=["GET"]) +def index(): + # Single query for now, but keep structure + selected_query_id = request.args.get("query_id", QUERIES[0]["id"]) + selected_query = next(q for q in QUERIES if q["id"] == selected_query_id) + selected_metrics = METRICS.get(selected_query["id"], {}) + + # Which optimizers to compare? + optimizer_left_id = request.args.get("optimizer_left_id", "noopt") + optimizer_right_id = request.args.get("optimizer_right_id", "opt2") + + # Helper to look up optimizer by id with fallback + def get_opt(opt_id, default_id): + opt = next((o for o in OPTIMIZERS if o["id"] == opt_id), None) + if opt is None: + opt = next(o for o in OPTIMIZERS if o["id"] == default_id) + return opt + + selected_optimizer_left = get_opt(optimizer_left_id, "noopt") + selected_optimizer_right = get_opt(optimizer_right_id, "opt2") + + plan_left = PLANS.get(selected_optimizer_left["plan_key"], "No plan available.") + plan_right = PLANS.get(selected_optimizer_right["plan_key"], "No plan available.") + + return render_template( + "index.html", + queries=QUERIES, + actions=ACTIONS, + optimizers=OPTIMIZERS, + selected_query=selected_query, + metrics=selected_metrics, + selected_optimizer_left=selected_optimizer_left, + selected_optimizer_right=selected_optimizer_right, + query_plan_left=plan_left, + query_plan_right=plan_right, + rules_snippet=RULE_SNIPPET, + ) + + +@app.route("/upload_optimizer", methods=["POST"]) +def upload_optimizer(): + file = request.files.get("optimizer_file") + if not file or file.filename == "": + flash("No optimizer file selected.", "danger") + return redirect(url_for("index")) + + filename = secure_filename(file.filename) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + flash(f"Optimizer '{filename}' uploaded successfully.", "success") + return redirect(url_for("index")) + + +@app.route("/upload_action", methods=["POST"]) +def upload_action(): + file = request.files.get("action_file") + if not file or file.filename == "": + flash("No action file selected.", "danger") + return redirect(url_for("index")) + + filename = secure_filename(file.filename) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + flash(f"Action definition '{filename}' uploaded successfully.", "success") + return redirect(url_for("index")) + + +@app.route("/api/benchmark_data") +def api_benchmark_data(): + # Simple JSON endpoint used by Chart.js on the frontend + return jsonify(BENCHMARK_RESULTS) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/Frontend/archive/index.html b/Frontend/archive/index.html new file mode 100644 index 0000000..c052573 --- /dev/null +++ b/Frontend/archive/index.html @@ -0,0 +1,611 @@ + + + + + OPTBench + + + + + + + + + + + + + +
+ {% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} + {% for category, message in messages %} + + {% endfor %} + {% endif %} + {% endwith %} +
+ + + + +
+ +
+ +
+
+
+ Query Selector +
+
+
+ + +
+ +
+ SQL + ML + +
+ + 
{{ selected_query.sql }}
+
+
+
+ + +
+
+
+ Query Plan Visualization +
+
+

+ Compare the logical plans produced by any two optimizers for the same SQL+ML query. +

+ +
+ +
+
+ Left plan +
+ + + + + +
+
+ 
{{ query_plan_left }}
+
+ + +
+
+ Right plan +
+ + + + + +
+
+ 
{{ query_plan_right }}
+
+
+
+
+
+
+ + +
+ + +
+
+
+ Statistics Window +
+
+

+ Data / plan statistics for this SQL+ML query (used as input to optimizer decisions). +

+ {% if metrics %} +
+ + + {% for k, v in metrics.items() %} + + + + + {% endfor %} + +
{{ k }}{{ v }}
+
+ {% else %} + No statistics available for this query yet. + {% endif %} +
+
+
+ + +
+
+
+ Rewrite Actions +
+
+

+ Available rewrite actions that can be used by any optimizer. +

+
    + {% for action in actions %} +
  • + {{ action }} + rewrite +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+ Optimizers +
+
+

+ Existing optimizer profiles available in this demo. +

+
    + {% for opt in optimizers %} +
  • +
    + {{ opt.name }} + {% if opt.id == 'noopt' %} + baseline + {% else %} + optimizer + {% endif %} +
    + {% if opt.description %} + {{ opt.description }} + {% endif %} +
    • + {% endfor %} +
+
+
+
+ + + +
+
+
+ Optimizer Definition +
+
+

+ Definition authored using a drag-and-drop statistics & action builder. +

+ + 
{{ rules_snippet }}
+ +
+ +
+ +
+
+
+ + +
+ + +
+ +
+
+
+ Upload Optimizer / Action +
+
+

+ Upload optimizer configs or action definitions to extend the workbench during the demo. +

+ +
+ +
+ + +
+
+ +
+ +
+ + +
+
+ +

+ Uploaded files are stored under uploads/. +

+
+
+
+ + +
+
+
+ Benchmarking Results +
+
+ + +
+
+ + +
+ + +
+ + +
+
+
+
+
+ + + + + + + diff --git a/Frontend/templates/index.html b/Frontend/templates/index.html new file mode 100644 index 0000000..62b0f93 --- /dev/null +++ b/Frontend/templates/index.html @@ -0,0 +1,816 @@ + + + + + OPTBench + + + + + + + + + + + + + +
+ {% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} + {% for category, message in messages %} + + {% endfor %} + {% endif %} + {% endwith %} +
+ + + + +
+ +
+ +
+
+
+ Query Selector +
+
+
+ + +
+ +
+ SQL + ML +
+ + 
{{ selected_query.sql }}
+
+
+
+ + +
+
+
+ Query Plan Visualization +
+
+

+ Compare the logical plans produced by any two optimizers for the same SQL+ML query. +

+ +
+ +
+
+ Left plan +
+ + + + + + +
+
+ 
Loading plan…
+
+ + +
+
+ Right plan +
+ + + + + + +
+
+ 
Loading plan…
+
+
+
+
+
+
+ + +
+ + +
+
+
+ Statistics Window +
+
+

+ Data / plan statistics for this SQL+ML query (used as input to optimizer decisions). +

+
+ {% if metrics %} +
+ + + {% for k, v in metrics.items() %} + + + + + {% endfor %} + +
{{ k }}{{ v }}
+
+ {% else %} + No statistics available for this query yet. + {% endif %} +
+
+
+
+ + +
+
+
+ Rewrite Actions +
+
+

+ Available rewrite actions that can be used by any optimizer. +

+
    + {% for action in actions %} +
  • + {{ action }} + +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+ Optimizers +
+
+

+ Existing optimizer profiles available in this demo. +

+
    + {% for opt in optimizers %} +
  • +
    + + {{ opt.name }} + + {% if opt.id == 'noopt' %} + baseline + {% elif opt.custom is defined and opt.custom %} + custom + {% else %} + optimizer + {% endif %} +
    + {% if opt.description %} + {{ opt.description }} + {% endif %} +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+ Optimizer Definition +
+
+

+ Definition for {{ definition_optimizer_id }}{% if not definition_is_pass %} (use it as a starting point){% endif %}: +

+ 
{{ rules_snippet }}
+ +
+ + {% if definition_is_pass %} +

Cost-Based Optimizer Pass

+

+ This is a compiled-in cost-based pass — it is not expressible as an + IF-THEN metric rule. To change it, upload a revised .cpp + via Upload Cost-Based Optimizer Pass and rebuild. +

+ {% else %} +

Create / Edit Custom Optimizer

+
+
+ + +
+ +
+ + +
+ +
+ + + IF [metric] <name> <op> <val> AND|OR … THEN [action] <ActionName> + +
+
+ +
+
+ +
+
+ {% endif %} +
+
+
+ +
+ + +
+ +
+
+
+ Upload Optimizer / Action +
+
+

+ Upload optimizer configs or action definitions to extend the workbench during the demo. +

+ +
+ + + + +
+ +
+
+ + +
+ + Compiles the pass into the extension and activates it as + OPT#<pass_name>. Rebuild takes a few minutes. + +
+ + + + +
+ +
+ + +
+
+ +

+ Uploaded files are stored under uploads/. +

+
+
+
+ + +
+
+
+ Benchmarking Results +
+
+ + +
+
+ + +
+ + + +
+ +
+
+ + +
+
+
+
+
+ + + + + + + diff --git a/README.md b/README.md index 32760ee..c800c00 100644 --- a/README.md +++ b/README.md @@ -78,3 +78,66 @@ Different tests can be created for DuckDB extensions. The primary way of testing ```sh make test ``` + +--- + +## Frontend (OPTBench Web UI) + +The frontend is a Flask web app that lets you explore query plans, compare optimizers, run benchmarks, and upload custom optimizer passes — all from a browser. + +### Prerequisites + +Python 3.9+ is required. Install the Python dependencies: + +```sh +pip install flask werkzeug +``` + +> **Note:** The frontend shells out to `./build/release/duckdb`, so the extension must be built first (see [Build steps](#build-steps) above). + +### Starting the server + +The `Frontend/app.py` file expects to be run from **the project root** so that relative paths to `build/`, `queries/`, and `plugins/` resolve correctly: + +```sh +cd +python Frontend/app.py +``` + +Flask will start on `http://127.0.0.1:5000` by default. + +If `LIBTORCH_DIR` is not already exported in your shell, set it before launching: + +```sh +LIBTORCH_DIR=~/libtorch python Frontend/app.py +``` + +### What the UI provides + +| Section | Description | +|---|---| +| **Query explorer** | Browse the built-in SQL+ML queries (Expedia, Flights, UC01–UC10, synthetic ranker) | +| **Plan comparison** | Side-by-side physical plan diff for any two optimizer settings | +| **Benchmark runner** | Execute a query under every active optimizer and compare runtimes | +| **Optimizer builder** | Write IF-THEN rules using the metric catalog to create custom optimizers | +| **Optimizer upload** | Upload a `.cpp` pass file; the server rebuilds the extension and activates it live | + +### Uploading a custom optimizer pass + +1. Write your pass as a self-contained `.cpp` file that calls `REGISTER_OPT_PASS(, check, apply)`. +2. Open the **Upload Pass** panel in the UI and select your `.cpp` (and an optional `.hpp` header). +3. The server drops the file into `plugins/opt_passes/`, regenerates the manifest, runs `make release`, and activates the pass under the slot name `OPT#` — no manual rebuild needed. +4. Build progress is shown in real time via the **Build Status** indicator in the UI. + +> **Allowed `#include` headers:** standard library headers (`algorithm`, `vector`, `string`, …) and project-internal headers under `duckdb/`, `optimization/`, `ml_functions/`, and `cost_model/`. System or third-party headers are rejected at upload time. + +### Production / non-debug mode + +For demos or multi-user use, run Flask behind a production WSGI server: + +```sh +pip install gunicorn +gunicorn -w 1 -b 0.0.0.0:5000 --chdir "Frontend.app:app" +``` + +> Single worker (`-w 1`) is required because the optimizer state and build status are kept in-process memory; multiple workers would not share them. diff --git a/baseline.sh b/baseline.sh new file mode 100755 index 0000000..850c45f --- /dev/null +++ b/baseline.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Resets the plugin directory to a clean baseline so the upload flow can be re-tested. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_DIR="$SCRIPT_DIR/plugins/opt_passes" + +# Remove any uploaded pass sources and rejected files +find "$PLUGIN_DIR" -maxdepth 1 -name "*.cpp" ! -name "_manifest.cpp" -delete +find "$PLUGIN_DIR" -maxdepth 1 -name "*.rejected" -delete +find "$PLUGIN_DIR/include" -maxdepth 1 -type f ! -name ".gitkeep" -delete + +# Reset manifest to empty baseline +cat > "$PLUGIN_DIR/_manifest.cpp" << 'EOF' +// AUTO-GENERATED by the frontend upload flow — do not edit by hand. +#include + +namespace duckdb { + +void LinkOptPassPlugins() { + static std::vector keep; + (void)keep; +} + +} // namespace duckdb +EOF + +echo "Baseline restored. Plugin directory:" +ls "$PLUGIN_DIR" diff --git a/demo/uploadable_passes/greedy_factorization_pass.cpp b/demo/uploadable_passes/greedy_factorization_pass.cpp new file mode 100644 index 0000000..69ff7b3 --- /dev/null +++ b/demo/uploadable_passes/greedy_factorization_pass.cpp @@ -0,0 +1,913 @@ +// Uploadable optimizer-pass plugin: greedy cost-based factorization. +// The full MLGreedyFactorizationOptimizer algorithm plus one REGISTER_OPT_PASS +// line at the bottom. Upload via the frontend; it self-registers as +// "greedy_factorization" and becomes selectable via OPT#greedy_factorization. + +#include "optimization/optimizer_pass.hpp" +#include "optimization/actions/MLGreedyFactorizationOptimizer.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/assert.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/value.hpp" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" + +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/operator/logical_comparison_join.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" + +#include "ml_functions/matrix_multiplication.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +// --------------------------------------------------------------------------- +// JoinTreeNode +// --------------------------------------------------------------------------- + +struct JoinTreeNode { + LogicalOperator *plan_node; // pointer into the live DuckDB plan tree + int side; // 0=left child of parent, 1=right child, -1=root + JoinTreeNode *parent; // nullptr for root + std::vector children; // up to 2 for a binary join + + // Populated during tree walk + idx_t num_rows; // estimated cardinality + int64_t num_features; // total input feature dims still un-reduced at this node + int64_t new_features; // model output width n (first-layer neuron count) + double cardinality_ratio; // num_rows / parent->num_rows (selectivity from this side) + int depth; // absolute depth (root = 1) + int max_depth; // deepest node in the whole tree (same for all nodes after BFS) + double cost; // current cost estimate + + // Assignment phase + int label; // 0=unassigned, 1=push mat_mul here, 2=combine children (list_add_vv) + bool processed; + bool subtree_processed; + + // Weight-split info set during apply phase + idx_t row_start; // start row in the original W for this node's feature block + idx_t row_count; // number of rows (= sum of k_i for tables on this side) +}; + +// --------------------------------------------------------------------------- +// Expression helpers (shared with MLFactorizationRewriteAction pattern) +// --------------------------------------------------------------------------- + +static bool IsMatMul(const BoundFunctionExpression &f) { + return StringUtil::CIEquals(f.function.name, "mat_mul"); +} + +static bool IsListConcat(const BoundFunctionExpression &f) { + auto name = StringUtil::Lower(f.function.name); + return name == "list_concat" || name == "list_cat" || name == "array_concat"; +} + +static bool AreConsecutivePartIndices(const std::vector &indices) { + if (indices.empty()) return false; + for (idx_t i = 1; i < indices.size(); ++i) { + if (indices[i] != indices[i - 1] + 1) return false; + } + return true; +} + +static int64_t GetStaticListLength(const unique_ptr &expr) { + if (!expr) return -1; + if (expr->return_type.id() == LogicalTypeId::ARRAY) + return static_cast(ArrayType::GetSize(expr->return_type)); + if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) + return static_cast(ArrayType::GetSize(cast.child->return_type)); + } + if (expr->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = expr->Cast(); + auto name = StringUtil::Lower(bfe.function.name); + if (name == "list_value" || name == "array_value") + return static_cast(bfe.children.size()); + } + return -1; +} + +// Collect every table_index referenced by BoundColumnRefExpressions in expr. +static void CollectTableIndexes(Expression &expr, + std::unordered_set &out) { + auto owned_copy = expr.Copy(); + ExpressionIterator::EnumerateExpression( + owned_copy, [&](Expression &sub) { + if (sub.type == ExpressionType::BOUND_COLUMN_REF) + out.insert(sub.Cast().binding.table_index); + }); +} + +// Recursively flatten list_concat(list_concat(A,B),C) -> [{A,k_A},{B,k_B},{C,k_C}] +struct FlatPart { + Expression *expr; + int64_t length; +}; + +static void FlattenListConcatImpl(Expression *expr, std::vector &out) { + if (!expr) return; + + // Unwrap implicit cast DOUBLE[N] -> DOUBLE[] inserted by DuckDB + Expression *inner = expr; + if (inner->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = inner->Cast(); + if (cast.child) inner = cast.child.get(); + } + + if (inner->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = inner->Cast(); + if (IsListConcat(bfe) && bfe.children.size() == 2) { + FlattenListConcatImpl(bfe.children[0].get(), out); + FlattenListConcatImpl(bfe.children[1].get(), out); + return; + } + } + + // Leaf: compute static length and record + // Use original expr (not inner) so GetStaticListLength can unwrap the cast itself + unique_ptr dummy; // we only need length from it + int64_t len = -1; + if (expr->return_type.id() == LogicalTypeId::ARRAY) + len = static_cast(ArrayType::GetSize(expr->return_type)); + else if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) + len = static_cast(ArrayType::GetSize(cast.child->return_type)); + } + out.push_back({expr, len}); +} + +static std::vector FlattenListConcat(Expression &expr) { + std::vector parts; + FlattenListConcatImpl(&expr, parts); + return parts; +} + +static unique_ptr BuildConcatFromParts(const BoundFunctionExpression &concat_proto, + const std::vector &parts, + const std::vector &indices) { + D_ASSERT(!indices.empty()); + auto expr = parts[indices[0]].expr->Copy(); + for (idx_t i = 1; i < indices.size(); ++i) { + vector> children; + children.push_back(std::move(expr)); + children.push_back(parts[indices[i]].expr->Copy()); + expr = make_uniq( + LogicalType::LIST(LogicalType::DOUBLE), + concat_proto.function, + std::move(children), + nullptr); + } + return expr; +} + +// --------------------------------------------------------------------------- +// Weight file helper +// --------------------------------------------------------------------------- + +static std::string WriteSplitWeightCSV(const std::vector &weights, idx_t n, + idx_t row_start, idx_t row_count, + const std::string &base_path, + const std::string &suffix) { + std::size_t h = std::hash{}( + base_path + "|" + std::to_string(row_start) + "|" + std::to_string(row_count)); + std::string out_path = "/tmp/matmul_greedy_" + std::to_string(h) + suffix + ".csv"; + { + std::ifstream probe(out_path); + if (probe.good()) return out_path; + } + std::ofstream f(out_path); + if (!f.is_open()) + throw InternalException("MLGreedyFactorization: cannot write weight file: %s", out_path.c_str()); + for (idx_t row = row_start; row < row_start + row_count; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) f << ','; + f << weights[row * n + col]; + } + f << '\n'; + } + return out_path; +} + +static std::string WriteCustomWeightCSV(const std::vector &weights, idx_t k, idx_t n, + const std::string &base_path, + const std::string &suffix, + const std::string &cache_key) { + std::size_t h = std::hash{}( + base_path + "|" + cache_key + "|" + std::to_string(k) + "|" + std::to_string(n)); + std::string out_path = "/tmp/matmul_greedy_" + std::to_string(h) + suffix + ".csv"; + { + std::ifstream probe(out_path); + if (probe.good()) return out_path; + } + + std::ofstream f(out_path); + if (!f.is_open()) + throw InternalException("MLGreedyFactorization: cannot write weight file: %s", out_path.c_str()); + for (idx_t row = 0; row < k; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) f << ','; + f << weights[row * n + col]; + } + f << '\n'; + } + return out_path; +} + +// --------------------------------------------------------------------------- +// Expression builders +// --------------------------------------------------------------------------- + +static unique_ptr BuildMatMulExpr(ClientContext &ctx, + unique_ptr input_expr, + const std::string &W_path, + const std::string &mode_str, + const std::vector &W_flat, + idx_t k, idx_t n, + ML_MatrixMultiply::MulMode mode) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = catalog.GetEntry(ctx, DEFAULT_SCHEMA, "mat_mul"); + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 3 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1].id() == LogicalTypeId::VARCHAR && + fn.arguments[2].id() == LogicalTypeId::VARCHAR) { + chosen = &fn; + break; + } + } + if (!chosen) + throw InternalException("MLGreedyFactorization: mat_mul overload not found"); + + auto bind_info = make_uniq(W_flat, k, n, W_path, true, mode); + vector> children; + children.push_back(std::move(input_expr)); + children.push_back(make_uniq(Value(W_path))); + children.push_back(make_uniq(Value(mode_str))); + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), std::move(bind_info)); +} + +static unique_ptr BuildListAddVVExpr(ClientContext &ctx, + unique_ptr left, + unique_ptr right) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = catalog.GetEntry(ctx, DEFAULT_SCHEMA, "list_add_vv"); + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 2 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1] == LogicalType::LIST(LogicalType::DOUBLE)) { + chosen = &fn; + break; + } + } + if (!chosen) + throw InternalException("MLGreedyFactorization: list_add_vv overload not found"); + vector> children; + children.push_back(std::move(left)); + children.push_back(std::move(right)); + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), nullptr); +} + +// --------------------------------------------------------------------------- +// InsertProjectionOnSide +// --------------------------------------------------------------------------- + +static idx_t GetLogicalWidth(LogicalOperator &op) { + if (op.types.empty()) op.ResolveOperatorTypes(); + return op.types.size(); +} + +static bool InsertProjectionOnSide(LogicalComparisonJoin &join, int side_idx, + Expression &extra_expr, idx_t &new_col_idx, + idx_t &new_table_idx) { + D_ASSERT(side_idx == 0 || side_idx == 1); + auto &child_ptr = join.children[side_idx]; + if (!child_ptr) return false; + + auto side_tables = child_ptr->GetTableIndex(); + if (side_tables.size() != 1) return false; + + const idx_t table_index = side_tables[0]; + const idx_t child_width = GetLogicalWidth(*child_ptr); + if (child_width == 0) return false; + + vector> select_list; + select_list.reserve(child_width + 1); + for (idx_t i = 0; i < child_width; ++i) + select_list.push_back(make_uniq(child_ptr->types[i], i)); + select_list.push_back(extra_expr.Copy()); + new_col_idx = child_width; + + auto new_proj = make_uniq(table_index, std::move(select_list)); + new_proj->children.push_back(std::move(child_ptr)); + child_ptr = std::move(new_proj); + + // If the join already projects a subset of child columns, explicitly expose + // the appended column so parent expressions can bind to it. + if (side_idx == 0) { + if (!join.left_projection_map.empty()) { + join.left_projection_map.push_back(new_col_idx); + } + } else { + if (!join.right_projection_map.empty()) { + join.right_projection_map.push_back(new_col_idx); + } + } + + new_table_idx = table_index; + return true; +} + +// --------------------------------------------------------------------------- +// Join tree construction +// --------------------------------------------------------------------------- + +static idx_t EstimateCardinality(LogicalOperator &op) { + if (op.estimated_cardinality > 0) + return op.estimated_cardinality; + // Fallback: product of children cardinalities (pessimistic cross-join) + if (op.children.size() == 2) { + idx_t l = EstimateCardinality(*op.children[0]); + idx_t r = EstimateCardinality(*op.children[1]); + return std::max(1, l * r); + } + return 1; +} + +// Walk the DuckDB plan tree and collect all LogicalComparisonJoin nodes, +// building a parallel JoinTreeNode tree. Returns the root JoinTreeNode. +// all_nodes is populated in post-order (children before parents). +static JoinTreeNode *BuildJoinTree(LogicalOperator *op, + JoinTreeNode *parent, + int side, + int64_t new_features, + std::vector> &storage) { + if (!op) return nullptr; + + // Recurse into children first (post-order) + if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + auto node = make_uniq(); + node->plan_node = op; + node->side = side; + node->parent = parent; + node->new_features = new_features; + node->label = 0; + node->processed = false; + node->subtree_processed = false; + node->row_start = 0; + node->row_count = 0; + node->depth = 0; // filled by CalculateDepths + node->max_depth = 0; + node->cost = 0.0; + + idx_t card = EstimateCardinality(*op); + node->num_rows = card; + + if (parent) { + node->cardinality_ratio = (parent->num_rows > 0) + ? static_cast(card) / static_cast(parent->num_rows) + : 1.0; + } else { + node->cardinality_ratio = 1.0; + } + + JoinTreeNode *raw = node.get(); + + // Build children recursively + if (op->children.size() >= 1) { + JoinTreeNode *left_child = BuildJoinTree(op->children[0].get(), raw, 0, new_features, storage); + if (left_child) raw->children.push_back(left_child); + } + if (op->children.size() >= 2) { + JoinTreeNode *right_child = BuildJoinTree(op->children[1].get(), raw, 1, new_features, storage); + if (right_child) raw->children.push_back(right_child); + } + + storage.push_back(std::move(node)); + return raw; + } + + // Not a join — recurse into children looking for joins below + for (idx_t i = 0; i < op->children.size(); ++i) { + int child_side = (op->children.size() == 2) ? (int)i : side; + JoinTreeNode *found = BuildJoinTree(op->children[i].get(), parent, child_side, new_features, storage); + if (found) return found; // return first found (handles single-path plans) + } + return nullptr; +} + +// --------------------------------------------------------------------------- +// Depth computation +// --------------------------------------------------------------------------- + +static int CalculateDepths(JoinTreeNode *root, + std::vector> &all_nodes) { + int max_depth = 0; + std::queue> q; + q.push({root, 1}); + while (!q.empty()) { + auto [node, d] = q.front(); q.pop(); + node->depth = d; + if (d > max_depth) max_depth = d; + for (auto *child : node->children) q.push({child, d + 1}); + } + for (auto &n : all_nodes) n->max_depth = max_depth; + return max_depth; +} + +// --------------------------------------------------------------------------- +// Feature-part attribution: which flat parts belong to which join-tree side +// --------------------------------------------------------------------------- + +// Returns true if the given LogicalOperator subtree contains table_index t. +static bool SubtreeContainsTable(LogicalOperator *op, idx_t table_index) { + if (!op) return false; + auto tables = op->GetTableIndex(); + for (idx_t t : tables) + if (t == table_index) return true; + for (auto &child : op->children) + if (SubtreeContainsTable(child.get(), table_index)) return true; + return false; +} + +// For a given JoinTreeNode, return which side (0 or 1) the table lives on. +// Returns -1 if not found in either child subtree. +static int TableSideOfNode(JoinTreeNode *node, idx_t table_index) { + if (!node || node->plan_node->children.size() < 2) return -1; + if (SubtreeContainsTable(node->plan_node->children[0].get(), table_index)) return 0; + if (SubtreeContainsTable(node->plan_node->children[1].get(), table_index)) return 1; + return -1; +} + +// --------------------------------------------------------------------------- +// Cost model +// --------------------------------------------------------------------------- + +static double ComputeCost(const JoinTreeNode &v) { + if (v.num_features == 0 || v.depth == 0 || v.max_depth == 0) + return 1.0; // safe fallback above threshold + + double var2 = static_cast(v.num_features); + double var4 = static_cast(v.new_features); + double var5 = static_cast(v.max_depth); + double var6 = var4 / var2; + double var7 = var4 / (var2 * v.depth); + double var8 = static_cast(v.num_rows) * var4 / var2; + double var9 = static_cast(v.num_rows) * var4 / (var2 * v.depth); + double var10 = var2 - var4; + double var11 = static_cast(v.depth) / static_cast(v.max_depth); + double var12 = v.cardinality_ratio; + double d4 = std::pow(static_cast(v.depth), 4.0); + + return -4.62412178e-04 * var2 + - 6.99311355e-04 * var4 + - 4.17964429e-03 * var5 + + 1.57416551e-03 * var6 + - 3.69161585e-04 * var7 + - 2.94571978e-05 * var8 + + 6.07897302e-05 * var9 + + 2.36899161e-04 * var10 + - 4.63010544e-02 * var12 + + 2.12526468e-03 * d4 + + 1.40041493e-04 * std::pow(var11, 4.0) + - 3.86043424e-02 * var11 * var12; +} + +// --------------------------------------------------------------------------- +// Greedy algorithm helpers +// --------------------------------------------------------------------------- + +static void ProcessSubtree(JoinTreeNode *node) { + if (node->subtree_processed) return; + std::vector stack = {node}; + while (!stack.empty()) { + JoinTreeNode *cur = stack.back(); stack.pop_back(); + cur->processed = true; + cur->subtree_processed = true; + for (auto *child : cur->children) + if (!child->subtree_processed) stack.push_back(child); + } + if (node->parent) node->parent->subtree_processed = true; +} + +static void UpdateParentFeatures(JoinTreeNode *parent) { + int64_t total = 0; + for (auto *child : parent->children) + total += (child->label != 0) ? child->new_features : child->num_features; + parent->num_features = total; +} + +// --------------------------------------------------------------------------- +// Greedy assignment +// --------------------------------------------------------------------------- + +static void PerformGreedy(std::vector> &all_nodes) { + constexpr double COST_THRESHOLD = 0.5; + + // Prefer deeper joins first. This avoids selecting a parent early and + // marking its entire subtree processed before child joins can be labeled. + using Entry = std::tuple; + std::priority_queue, std::greater> heap; + + for (auto &n : all_nodes) { + n->cost = ComputeCost(*n); + if (n->cost < COST_THRESHOLD) { + int depth_priority = n->max_depth - n->depth; // smaller => deeper + heap.push({depth_priority, n->cost, n.get()}); + } + } + + while (!heap.empty()) { + auto [depth_priority, current_cost, v] = heap.top(); heap.pop(); + (void)depth_priority; + + if (v->processed) continue; + if (current_cost != v->cost) continue; // stale entry + + if (v->children.size() >= 2 && + v->children[0]->label != 0 && v->children[1]->label != 0) { + v->label = 2; + } else if (current_cost < COST_THRESHOLD) { + v->label = 1; + } else { + break; + } + + ProcessSubtree(v); + + if (v->parent) { + double old_cost = v->parent->cost; + UpdateParentFeatures(v->parent); + double new_cost = ComputeCost(*v->parent); + if (new_cost != old_cost) { + v->parent->cost = new_cost; + if (new_cost < COST_THRESHOLD) { + int parent_depth_priority = v->parent->max_depth - v->parent->depth; + heap.push({parent_depth_priority, new_cost, v->parent}); + } + } + } + } + + // Final pass: promote nodes whose both children are labeled to label=2 + for (auto &n : all_nodes) { + if (n->children.size() >= 2 && + n->children[0]->label != 0 && n->children[1]->label != 0) + n->label = 2; + } +} + +// --------------------------------------------------------------------------- +// check +// --------------------------------------------------------------------------- + +// Returns true if the plan contains a projection over a multi-join with +// mat_mul(list_concat(...), W) where list_concat has 2+ parts. +static bool HasGreedyPattern(LogicalOperator *op) { + if (!op) return false; + + if (op->type == LogicalOperatorType::LOGICAL_PROJECTION) { + // Check if any child is a comparison join + bool has_join_child = false; + for (auto &child : op->children) + if (child && child->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) + has_join_child = true; + + if (has_join_child) { + auto &proj = op->Cast(); + for (auto &expr : proj.expressions) { + if (!expr || expr->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &f = expr->Cast(); + if (!IsMatMul(f) || f.children.empty()) continue; + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) continue; + // Check that the argument is a list_concat with 2+ parts + auto &arg = f.children[0]; + if (!arg || arg->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &lc = arg->Cast(); + if (IsListConcat(lc)) { + auto parts = FlattenListConcat(*arg); + if (parts.size() >= 2) return true; + } + } + } + } + + for (auto &child : op->children) + if (HasGreedyPattern(child.get())) return true; + + return false; +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// Public interface +// --------------------------------------------------------------------------- + +bool MLGreedyFactorizationOptimizer::check(OptimizerExtensionInput &input, + unique_ptr &plan) { + return plan && HasGreedyPattern(plan.get()); +} + +bool MLGreedyFactorizationOptimizer::apply(OptimizerExtensionInput &input, + unique_ptr &plan) { + if (!plan) return false; + + // Bottom-up: rewrite children first + bool changed = false; + for (auto &child : plan->children) + changed |= apply(input, child); + + if (plan->type != LogicalOperatorType::LOGICAL_PROJECTION) return changed; + auto &proj = plan->Cast(); + if (proj.children.empty() || !proj.children[0] || + proj.children[0]->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) + return changed; + + // Find the mat_mul(list_concat(...), W) expression in the projection + BoundFunctionExpression *mat_mul_expr = nullptr; + unique_ptr *mat_mul_slot = nullptr; + for (auto &expr_ptr : proj.expressions) { + if (!expr_ptr || expr_ptr->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &f = expr_ptr->Cast(); + if (!IsMatMul(f) || f.children.empty()) continue; + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) continue; + auto &arg = f.children[0]; + if (!arg || arg->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &lc = arg->Cast(); + if (!IsListConcat(lc)) continue; + auto parts = FlattenListConcat(*arg); + if (parts.size() < 2) continue; + mat_mul_expr = &f; + mat_mul_slot = &expr_ptr; + break; + } + if (!mat_mul_expr) return changed; + + auto *bd = dynamic_cast(mat_mul_expr->bind_info.get()); + const idx_t n = bd->n; + const std::string &W_file = bd->weights_file; + std::string mode_str; + switch (bd->mode) { + case ML_MatrixMultiply::MulMode::Dense: mode_str = "dense"; break; + case ML_MatrixMultiply::MulMode::InputSparse: mode_str = "sparse"; break; + default: mode_str = "dense"; break; + } + + // Flatten list_concat into ordered parts + auto parts = FlattenListConcat(*mat_mul_expr->children[0]); + if (parts.size() < 2) return changed; + const auto &concat_proto = mat_mul_expr->children[0]->Cast(); + + // Validate all part lengths are known + for (auto &p : parts) + if (p.length <= 0) return changed; + + std::vector part_row_prefix(parts.size() + 1, 0); + for (idx_t i = 0; i < parts.size(); ++i) { + part_row_prefix[i + 1] = part_row_prefix[i] + static_cast(parts[i].length); + } + + // --------------------------------------------------------------------------- + // Build join tree + // --------------------------------------------------------------------------- + std::vector> node_storage; + JoinTreeNode *root = BuildJoinTree(proj.children[0].get(), nullptr, -1, + static_cast(n), node_storage); + if (!root || node_storage.empty()) return changed; + + // Set num_features on each node (total k = sum of all part lengths) + int64_t total_k = 0; + for (auto &p : parts) total_k += p.length; + for (auto &node : node_storage) node->num_features = total_k; + + // Compute depths + CalculateDepths(root, node_storage); + + // --------------------------------------------------------------------------- + // Run greedy + // --------------------------------------------------------------------------- + PerformGreedy(node_storage); + + // Check if anything was labeled — if not, nothing to do + bool any_labeled = false; + for (auto &node : node_storage) + if (node->label != 0) { any_labeled = true; break; } + if (!any_labeled) return changed; + + // Conservative safety guard: if any join already carries a projection map + // (i.e., columns have been pruned/reordered upstream), skip this rewrite. + // The greedy rewrite currently appends columns on join children and assumes + // stable pass-through bindings. + for (auto &node_ptr : node_storage) { + auto &join = node_ptr->plan_node->Cast(); + if (join.HasProjectionMap()) { + return changed; + } + } + + // --------------------------------------------------------------------------- + // Apply push-downs + // Iterate nodes in storage order (post-order: children before parents). + // For each label=1 node, figure out which flat parts belong to each side, + // build a mat_mul for that side, and insert a projection on the join child. + // + // We track, for each node's plan_node (join), the column indices of inserted + // partial results so the parent can reference them. + // --------------------------------------------------------------------------- + + // Map from plan_node pointer -> {left_col_idx, right_col_idx} of inserted projections + // -1 means not yet inserted on that side. + struct InsertRecord { + idx_t left_col = idx_t(-1); + idx_t right_col = idx_t(-1); + }; + std::unordered_map insert_map; + + for (auto &node_ptr : node_storage) { + JoinTreeNode *node = node_ptr.get(); + if (node->label != 1) continue; + + auto &join = node->plan_node->Cast(); + + // Determine which flat parts land on left vs right side of this join + std::vector left_part_indices, right_part_indices; + for (int i = 0; i < (int)parts.size(); ++i) { + std::unordered_set refs; + CollectTableIndexes(*parts[i].expr, refs); + + for (idx_t t : refs) { + int s = TableSideOfNode(node, t); + if (s == 0) { left_part_indices.push_back(i); break; } + if (s == 1) { right_part_indices.push_back(i); break; } + } + } + + // Helper lambda: build mat_mul for a set of consecutive part indices + auto build_side_mat_mul = [&](const std::vector &indices, int side_id) + -> unique_ptr { + if (indices.empty()) return nullptr; + + idx_t row_count = 0; + for (auto idx : indices) row_count += static_cast(parts[idx].length); + + std::vector W_slice; + W_slice.reserve(row_count * n); + + const idx_t row_start = part_row_prefix[indices.front()]; + for (auto part_idx : indices) { + const idx_t start = part_row_prefix[part_idx]; + const idx_t end = part_row_prefix[part_idx + 1]; + for (idx_t row = start; row < end; ++row) { + auto row_begin = bd->weights.begin() + static_cast(row * n); + W_slice.insert(W_slice.end(), row_begin, row_begin + static_cast(n)); + } + } + + std::string suffix = (side_id == 0) ? "_left" : "_right"; + std::string W_path; + if (AreConsecutivePartIndices(indices)) { + W_path = WriteSplitWeightCSV(bd->weights, n, row_start, row_count, W_file, suffix); + } else { + std::ostringstream key; + key << "parts"; + for (auto idx : indices) key << "_" << idx; + W_path = WriteCustomWeightCSV(W_slice, row_count, n, W_file, suffix, key.str()); + } + + unique_ptr input_expr; + if (indices.size() == 1) { + input_expr = parts[indices[0]].expr->Copy(); + } else { + input_expr = BuildConcatFromParts(concat_proto, parts, indices); + } + + return BuildMatMulExpr(input.context, std::move(input_expr), + W_path, mode_str, W_slice, + row_count, n, bd->mode); + }; + + InsertRecord rec; + + if (!left_part_indices.empty()) { + auto mm_left = build_side_mat_mul(left_part_indices, 0); + if (mm_left) { + idx_t col_idx = 0; + idx_t table_idx = idx_t(-1); + if (InsertProjectionOnSide(join, 0, *mm_left, col_idx, table_idx)) { + rec.left_col = col_idx; + } + } + } + if (!right_part_indices.empty()) { + auto mm_right = build_side_mat_mul(right_part_indices, 1); + if (mm_right) { + idx_t col_idx = 0; + idx_t table_idx = idx_t(-1); + if (InsertProjectionOnSide(join, 1, *mm_right, col_idx, table_idx)) { + rec.right_col = col_idx; + } + } + } + + insert_map[node->plan_node] = rec; + join.ResolveOperatorTypes(); + } + + // --------------------------------------------------------------------------- + // Replace the top-level mat_mul with a tree of list_add_vv over + // BoundReferenceExpressions pointing at inserted partial results. + // Walk label=2 nodes (and the root join) to collect references. + // --------------------------------------------------------------------------- + + auto &top_join = proj.children[0]->Cast(); + top_join.ResolveOperatorTypes(); + auto top_bindings = top_join.GetColumnBindings(); + + auto find_top_index = [&](const ColumnBinding &binding) -> idx_t { + for (idx_t i = 0; i < top_bindings.size(); ++i) { + if (top_bindings[i] == binding) return i; + } + return idx_t(-1); + }; + + // Find all nodes with inserted projections and build list_add_vv chain. + // Resolve each inserted column through the top join's current binding map, + // then reference the matching position with BoundReferenceExpression. + std::vector> partial_refs; + + for (auto &node_ptr : node_storage) { + JoinTreeNode *node = node_ptr.get(); + if (node->label != 1) continue; + auto it = insert_map.find(node->plan_node); + if (it == insert_map.end()) continue; + + auto &jn = node->plan_node->Cast(); + jn.ResolveOperatorTypes(); + auto node_bindings = jn.GetColumnBindings(); + auto left_child_bindings = jn.children[0]->GetColumnBindings(); + + if (it->second.left_col != idx_t(-1) && it->second.left_col < node_bindings.size()) { + const auto node_binding = node_bindings[it->second.left_col]; + const idx_t top_idx = find_top_index(node_binding); + if (top_idx != idx_t(-1) && top_idx < top_join.types.size()) { + partial_refs.push_back( + make_uniq(top_join.types[top_idx], top_idx)); + } + } + if (it->second.right_col != idx_t(-1)) { + const idx_t node_idx = left_child_bindings.size() + it->second.right_col; + if (node_idx < node_bindings.size()) { + const auto node_binding = node_bindings[node_idx]; + const idx_t top_idx = find_top_index(node_binding); + if (top_idx != idx_t(-1) && top_idx < top_join.types.size()) { + partial_refs.push_back( + make_uniq(top_join.types[top_idx], top_idx)); + } + } + } + } + + if (partial_refs.size() < 2) return changed; // need at least 2 to combine + + // Fold into list_add_vv(list_add_vv(...), last) + unique_ptr combined = std::move(partial_refs[0]); + for (idx_t i = 1; i < partial_refs.size(); ++i) + combined = BuildListAddVVExpr(input.context, std::move(combined), std::move(partial_refs[i])); + + *mat_mul_slot = std::move(combined); + return true; +} + +} // namespace duckdb + +// --- plugin self-registration ------------------------------------------------ +REGISTER_OPT_PASS(greedy_factorization, + &duckdb::MLGreedyFactorizationOptimizer::check, + &duckdb::MLGreedyFactorizationOptimizer::apply); diff --git a/expedia.json b/expedia.json index cce837e..b53a797 100644 --- a/expedia.json +++ b/expedia.json @@ -2,13 +2,13 @@ "total_bytes_written": 0, "total_bytes_read": 0, "rows_returned": 79756, - "latency": 0.619100106, + "latency": 0.594917065, "result_set_size": 2871216, - "query_name": "SELECT\n s.prop_id,\n s.srch_id,\n tree_predict(\n list_transform(\n list_concat(\n /* 1) NUMERICALS — per-column scaling (exact order) */\n min_max_scaler(list_value(CAST(s.prop_location_score1 AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'),\n min_max_scaler(list_value(CAST(s.prop_location_score2 AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'),\n min_max_scaler(list_value(CAST(s.prop_log_historical_price AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'),\n min_max_scaler(list_value(CAST(s.price_usd AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'),\n min_max_scaler(list_value(CAST(s.orig_destination_distance AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'),\n min_max_scaler(list_value(CAST(h.prop_review_score AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'),\n min_max_scaler(list_value(CAST(h.avg_bookings_usd AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'),\n min_max_scaler(list_value(CAST(h.stdev_bookings_usd AS DOUBLE)),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'),\n\n /* 2) CATEGORICALS — one-hot, exact order of your Python */\n -- stringEncoderSpecs (string→int): TRUE\n one_hot_encode(CAST(s.position AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', TRUE),\n one_hot_encode(CAST(h.prop_country_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', TRUE),\n\n -- encoderSpecs (int→int): FALSE\n one_hot_encode(CAST(h.prop_starrating AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', FALSE),\n one_hot_encode(CAST(h.prop_brand_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', FALSE),\n one_hot_encode(CAST(h.count_clicks AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', FALSE),\n one_hot_encode(CAST(h.count_bookings AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', FALSE),\n\n -- stringEncoderSpecs (string→int): TRUE\n one_hot_encode(CAST(r.year AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', TRUE),\n one_hot_encode(CAST(r.month AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', TRUE),\n one_hot_encode(CAST(r.weekofyear AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', TRUE),\n one_hot_encode(CAST(r.time AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', TRUE),\n one_hot_encode(CAST(r.site_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', TRUE),\n one_hot_encode(CAST(r.visitor_location_country_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', TRUE),\n one_hot_encode(CAST(r.srch_destination_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', TRUE),\n\n -- encoderSpecs (int→int): FALSE\n one_hot_encode(CAST(r.srch_length_of_stay AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', FALSE),\n one_hot_encode(CAST(r.srch_booking_window AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', FALSE),\n one_hot_encode(CAST(r.srch_adults_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_children_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_room_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_saturday_night_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', FALSE),\n one_hot_encode(CAST(r.random_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', FALSE)\n ),\n x -> CAST(x AS FLOAT) -- tree_predict expects LIST\n ),\n '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt',\n FALSE\n )\nFROM Expedia_S_listings_extension2 AS s\nJOIN Expedia_R1_hotels2 AS h ON s.prop_id = h.prop_id\nJOIN Expedia_R2_searches AS r ON s.srch_id = r.srch_id\nWHERE s.prop_location_score1 > 1\n AND s.prop_location_score2 > 0.1\n AND s.prop_log_historical_price > 4\n AND h.count_bookings > 5\n AND r.srch_booking_window > 10\n AND r.srch_length_of_stay > 1;", + "query_name": "SELECT\n s.prop_id,\n s.srch_id,\n tree_predict(\n list_transform(\n list_concat(\n /* 1) NUMERICALS — per-column scaling (exact order) */\n min_max_scaler(list_value(CAST(s.prop_location_score1 AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'),\n min_max_scaler(list_value(CAST(s.prop_location_score2 AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'),\n min_max_scaler(list_value(CAST(s.prop_log_historical_price AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'),\n min_max_scaler(list_value(CAST(s.price_usd AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'),\n min_max_scaler(list_value(CAST(s.orig_destination_distance AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'),\n min_max_scaler(list_value(CAST(h.prop_review_score AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'),\n min_max_scaler(list_value(CAST(h.avg_bookings_usd AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'),\n min_max_scaler(list_value(CAST(h.stdev_bookings_usd AS DOUBLE)),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'),\n\n /* 2) CATEGORICALS — one-hot, exact order of your Python */\n -- stringEncoderSpecs (string→int): TRUE\n one_hot_encode(CAST(s.position AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', TRUE),\n one_hot_encode(CAST(h.prop_country_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', TRUE),\n\n -- encoderSpecs (int→int): FALSE\n one_hot_encode(CAST(h.prop_starrating AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', FALSE),\n one_hot_encode(CAST(h.prop_brand_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', FALSE),\n one_hot_encode(CAST(h.count_clicks AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', FALSE),\n one_hot_encode(CAST(h.count_bookings AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', FALSE),\n\n -- stringEncoderSpecs (string→int): TRUE\n one_hot_encode(CAST(r.year AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', TRUE),\n one_hot_encode(CAST(r.month AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', TRUE),\n one_hot_encode(CAST(r.weekofyear AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', TRUE),\n one_hot_encode(CAST(r.time AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', TRUE),\n one_hot_encode(CAST(r.site_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', TRUE),\n one_hot_encode(CAST(r.visitor_location_country_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', TRUE),\n one_hot_encode(CAST(r.srch_destination_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', TRUE),\n\n -- encoderSpecs (int→int): FALSE\n one_hot_encode(CAST(r.srch_length_of_stay AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', FALSE),\n one_hot_encode(CAST(r.srch_booking_window AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', FALSE),\n one_hot_encode(CAST(r.srch_adults_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_children_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_room_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', FALSE),\n one_hot_encode(CAST(r.srch_saturday_night_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', FALSE),\n one_hot_encode(CAST(r.random_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', FALSE)\n ),\n x -> CAST(x AS FLOAT) -- tree_predict expects LIST\n ),\n '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt',\n FALSE\n )\nFROM Expedia_S_listings_extension2 AS s\nJOIN Expedia_R1_hotels2 AS h ON s.prop_id = h.prop_id\nJOIN Expedia_R2_searches AS r ON s.srch_id = r.srch_id\nWHERE s.prop_location_score1 > 1\n AND s.prop_location_score2 > 0.1\n AND s.prop_log_historical_price > 4\n AND h.count_bookings > 5\n AND r.srch_booking_window > 10\n AND r.srch_length_of_stay > 1;", "blocked_thread_time": 0.0, - "system_peak_buffer_memory": 41269248, + "system_peak_buffer_memory": 41287680, "system_peak_temp_dir_size": 0, - "cpu_time": 4.15678951, + "cpu_time": 4.310444025000001, "extra_info": {}, "cumulative_cardinality": 637172, "cumulative_rows_scanned": 7586096, @@ -18,12 +18,12 @@ "total_bytes_read": 0, "result_set_size": 2871216, "operator_name": "PROJECTION", - "cpu_time": 4.15678951, + "cpu_time": 4.310444025000001, "extra_info": { "Projections": [ "prop_id", "srch_id", - "tree_predict(list_transform(list_concat(min_max_scaler(list_value(prop_location_score1), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'), min_max_scaler(list_value(prop_location_score2), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'), min_max_scaler(list_value(prop_log_historical_price), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'), min_max_scaler(list_value(price_usd), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'), min_max_scaler(list_value(orig_destination_distance), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'), min_max_scaler(list_value(CAST(prop_review_score AS DOUBLE)), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'), min_max_scaler(list_value(avg_bookings_usd), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'), min_max_scaler(list_value(stdev_bookings_usd), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'), one_hot_encode(position, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', true), one_hot_encode(prop_country_id, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', true), one_hot_encode(CAST(prop_starrating AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', false), one_hot_encode(CAST(prop_brand_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', false), one_hot_encode(CAST(count_clicks AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', false), one_hot_encode(CAST(count_bookings AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', false), one_hot_encode(year, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', true), one_hot_encode(month, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', true), one_hot_encode(weekofyear, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', true), one_hot_encode(time, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', true), one_hot_encode(site_id, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', true), one_hot_encode(visitor_location_country_id, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', true), one_hot_encode(srch_destination_id, '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', true), one_hot_encode(CAST(srch_length_of_stay AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', false), one_hot_encode(CAST(srch_booking_window AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', false), one_hot_encode(CAST(srch_adults_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', false), one_hot_encode(CAST(srch_children_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', false), one_hot_encode(CAST(srch_room_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', false), one_hot_encode(CAST(srch_saturday_night_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', false), one_hot_encode(CAST(random_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', false))), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt', false)" + "tree_predict(list_transform(list_concat(min_max_scaler(list_value(prop_location_score1), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'), min_max_scaler(list_value(prop_location_score2), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'), min_max_scaler(list_value(prop_log_historical_price), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'), min_max_scaler(list_value(price_usd), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'), min_max_scaler(list_value(orig_destination_distance), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'), min_max_scaler(list_value(CAST(prop_review_score AS DOUBLE)), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'), min_max_scaler(list_value(avg_bookings_usd), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'), min_max_scaler(list_value(stdev_bookings_usd), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'), one_hot_encode(position, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', true), one_hot_encode(prop_country_id, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', true), one_hot_encode(CAST(prop_starrating AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', false), one_hot_encode(CAST(prop_brand_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', false), one_hot_encode(CAST(count_clicks AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', false), one_hot_encode(CAST(count_bookings AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', false), one_hot_encode(year, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', true), one_hot_encode(month, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', true), one_hot_encode(weekofyear, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', true), one_hot_encode(time, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', true), one_hot_encode(site_id, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', true), one_hot_encode(visitor_location_country_id, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', true), one_hot_encode(srch_destination_id, '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', true), one_hot_encode(CAST(srch_length_of_stay AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', false), one_hot_encode(CAST(srch_booking_window AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', false), one_hot_encode(CAST(srch_adults_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', false), one_hot_encode(CAST(srch_children_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', false), one_hot_encode(CAST(srch_room_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', false), one_hot_encode(CAST(srch_saturday_night_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', false), one_hot_encode(CAST(random_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', false))), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt', false)" ], "Estimated Cardinality": "7220" }, @@ -32,14 +32,14 @@ "operator_cardinality": 79756, "cumulative_rows_scanned": 7586096, "operator_rows_scanned": 0, - "operator_timing": 3.9706238020000004, + "operator_timing": 4.160415685, "children": [ { "total_bytes_written": 0, "total_bytes_read": 0, "result_set_size": 26798016, "operator_name": "HASH_JOIN", - "cpu_time": 0.18616570799999999, + "cpu_time": 0.15002833999999998, "extra_info": { "Join Type": "INNER", "Conditions": "srch_id = srch_id", @@ -50,14 +50,14 @@ "operator_cardinality": 79756, "cumulative_rows_scanned": 7586096, "operator_rows_scanned": 0, - "operator_timing": 0.027968030999999997, + "operator_timing": 0.023002200999999996, "children": [ { "total_bytes_written": 0, "total_bytes_read": 0, "result_set_size": 35619696, "operator_name": "HASH_JOIN", - "cpu_time": 0.153371983, + "cpu_time": 0.12518312399999998, "extra_info": { "Join Type": "INNER", "Conditions": "prop_id = prop_id", @@ -68,14 +68,14 @@ "operator_cardinality": 212022, "cumulative_rows_scanned": 7549075, "operator_rows_scanned": 0, - "operator_timing": 0.040581439999999996, + "operator_timing": 0.045933729000000006, "children": [ { "total_bytes_written": 0, "total_bytes_read": 0, "result_set_size": 21772608, "operator_name": "SEQ_SCAN ", - "cpu_time": 0.11156367, + "cpu_time": 0.07825701199999997, "extra_info": { "Table": "Expedia_S_listings_extension2", "Type": "Sequential Scan", @@ -101,7 +101,7 @@ "operator_cardinality": 247416, "cumulative_rows_scanned": 7537136, "operator_rows_scanned": 7537136, - "operator_timing": 0.11156367, + "operator_timing": 0.07825701199999997, "children": [] }, { @@ -109,7 +109,7 @@ "total_bytes_read": 0, "result_set_size": 688320, "operator_name": "SEQ_SCAN ", - "cpu_time": 0.0012268730000000003, + "cpu_time": 0.000992383, "extra_info": { "Table": "Expedia_R1_hotels2", "Type": "Sequential Scan", @@ -132,7 +132,7 @@ "operator_cardinality": 7170, "cumulative_rows_scanned": 11939, "operator_rows_scanned": 11939, - "operator_timing": 0.0012268730000000003, + "operator_timing": 0.000992383, "children": [] } ] @@ -142,7 +142,7 @@ "total_bytes_read": 0, "result_set_size": 2033568, "operator_name": "SEQ_SCAN ", - "cpu_time": 0.004825693999999999, + "cpu_time": 0.001843015, "extra_info": { "Table": "Expedia_R2_searches", "Type": "Sequential Scan", @@ -174,7 +174,7 @@ "operator_cardinality": 11052, "cumulative_rows_scanned": 37021, "operator_rows_scanned": 37021, - "operator_timing": 0.004825693999999999, + "operator_timing": 0.001843015, "children": [] } ] diff --git a/plugins/opt_passes/.gitkeep b/plugins/opt_passes/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/plugins/opt_passes/_manifest.cpp b/plugins/opt_passes/_manifest.cpp new file mode 100644 index 0000000..8bcdb47 --- /dev/null +++ b/plugins/opt_passes/_manifest.cpp @@ -0,0 +1,13 @@ +// AUTO-GENERATED by the frontend upload flow — do not edit by hand. +#include + +extern "C" void optpass_anchor_greedy_factorization(); + +namespace duckdb { + +void LinkOptPassPlugins() { + static std::vector keep; + keep.push_back(reinterpret_cast(&optpass_anchor_greedy_factorization)); +} + +} // namespace duckdb diff --git a/plugins/opt_passes/include/.gitkeep b/plugins/opt_passes/include/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/queries/expedia_query.sql b/queries/expedia_query.sql index 0918b3c..7bdfd25 100644 --- a/queries/expedia_query.sql +++ b/queries/expedia_query.sql @@ -1,12 +1,12 @@ -- Attach the database file and use its schema -ATTACH '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/data/expedia_imbridge2.db' +ATTACH '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/data/expedia_imbridge2.db' AS expedia_db (READ_ONLY); SET schema 'expedia_db.main'; -- Optimizer Config PRAGMA enable_profiling; SET enable_profiling = 'json'; -SET profiling_output = '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/expedia.json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/expedia.json'; -- PRAGMA profiling_output='/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/plans/profile.json'; -- PRAGMA disable_optimizer; -- PRAGMA explain_output='json'; @@ -29,54 +29,54 @@ SELECT list_concat( /* 1) NUMERICALS — per-column scaling (exact order) */ min_max_scaler(list_value(CAST(s.prop_location_score1 AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score1.txt'), min_max_scaler(list_value(CAST(s.prop_location_score2 AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_location_score2.txt'), min_max_scaler(list_value(CAST(s.prop_log_historical_price AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_log_historical_price.txt'), min_max_scaler(list_value(CAST(s.price_usd AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/price_usd.txt'), min_max_scaler(list_value(CAST(s.orig_destination_distance AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/orig_destination_distance.txt'), min_max_scaler(list_value(CAST(h.prop_review_score AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/prop_review_score.txt'), min_max_scaler(list_value(CAST(h.avg_bookings_usd AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/avg_bookings_usd.txt'), min_max_scaler(list_value(CAST(h.stdev_bookings_usd AS DOUBLE)), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/scaler_files/stdev_bookings_usd.txt'), /* 2) CATEGORICALS — one-hot, exact order of your Python */ -- stringEncoderSpecs (string→int): TRUE - one_hot_encode(CAST(s.position AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', TRUE), - one_hot_encode(CAST(h.prop_country_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', TRUE), + one_hot_encode(CAST(s.position AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/position.csv', TRUE), + one_hot_encode(CAST(h.prop_country_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_country_id.csv', TRUE), -- encoderSpecs (int→int): FALSE - one_hot_encode(CAST(h.prop_starrating AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', FALSE), - one_hot_encode(CAST(h.prop_brand_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', FALSE), - one_hot_encode(CAST(h.count_clicks AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', FALSE), - one_hot_encode(CAST(h.count_bookings AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', FALSE), + one_hot_encode(CAST(h.prop_starrating AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_starrating.csv', FALSE), + one_hot_encode(CAST(h.prop_brand_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/prop_brand_bool.csv', FALSE), + one_hot_encode(CAST(h.count_clicks AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_clicks.csv', FALSE), + one_hot_encode(CAST(h.count_bookings AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/count_bookings.csv', FALSE), -- stringEncoderSpecs (string→int): TRUE - one_hot_encode(CAST(r.year AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', TRUE), - one_hot_encode(CAST(r.month AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', TRUE), - one_hot_encode(CAST(r.weekofyear AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', TRUE), - one_hot_encode(CAST(r.time AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', TRUE), - one_hot_encode(CAST(r.site_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', TRUE), - one_hot_encode(CAST(r.visitor_location_country_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', TRUE), - one_hot_encode(CAST(r.srch_destination_id AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', TRUE), + one_hot_encode(CAST(r.year AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/year.csv', TRUE), + one_hot_encode(CAST(r.month AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/month.csv', TRUE), + one_hot_encode(CAST(r.weekofyear AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/weekofyear.csv', TRUE), + one_hot_encode(CAST(r.time AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/time.csv', TRUE), + one_hot_encode(CAST(r.site_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/site_id.csv', TRUE), + one_hot_encode(CAST(r.visitor_location_country_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/visitor_location_country_id.csv', TRUE), + one_hot_encode(CAST(r.srch_destination_id AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_destination_id.csv', TRUE), -- encoderSpecs (int→int): FALSE - one_hot_encode(CAST(r.srch_length_of_stay AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', FALSE), - one_hot_encode(CAST(r.srch_booking_window AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', FALSE), - one_hot_encode(CAST(r.srch_adults_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', FALSE), - one_hot_encode(CAST(r.srch_children_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', FALSE), - one_hot_encode(CAST(r.srch_room_count AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', FALSE), - one_hot_encode(CAST(r.srch_saturday_night_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', FALSE), - one_hot_encode(CAST(r.random_bool AS VARCHAR), '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', FALSE) + one_hot_encode(CAST(r.srch_length_of_stay AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_length_of_stay.csv', FALSE), + one_hot_encode(CAST(r.srch_booking_window AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_booking_window.csv', FALSE), + one_hot_encode(CAST(r.srch_adults_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_adults_count.csv', FALSE), + one_hot_encode(CAST(r.srch_children_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_children_count.csv', FALSE), + one_hot_encode(CAST(r.srch_room_count AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_room_count.csv', FALSE), + one_hot_encode(CAST(r.srch_saturday_night_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/srch_saturday_night_bool.csv', FALSE), + one_hot_encode(CAST(r.random_bool AS VARCHAR), '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/onehotencoders/random_bool.csv', FALSE) ), x -> CAST(x AS FLOAT) -- tree_predict expects LIST ), - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt', + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/expedia/tree/0.txt', FALSE ) FROM Expedia_S_listings_extension2 AS s diff --git a/queries/factorization_demo.sql b/queries/factorization_demo.sql new file mode 100644 index 0000000..b78fbb1 --- /dev/null +++ b/queries/factorization_demo.sql @@ -0,0 +1,75 @@ +-- MLFactorizationRewriteAction demo +-- +-- Schema: left_feats has 3-dim embeddings, right_feats has 2-dim embeddings. +-- The join produces a 5-dim feature vector that is multiplied by W (5x2). +-- +-- Optimization (enable_extended_optimizer = '4') splits W into: +-- W_top (3x2) pushed to left_feats before the join +-- W_bottom (2x2) pushed to right_feats before the join +-- and replaces the single mat_mul with list_add_vv of the two partial results. +-- +-- Run: +-- duckdb < queries/factorization_demo.sql + +LOAD cactusdb; + +-- Disable DuckDB's own rewrites so the logical plan stays readable. +SET disabled_optimizers = 'expression_rewriter, empty_result_pullup, + cte_filter_pusher, regex_range, in_clause, join_order, deliminator, + unnest_rewriter, unused_columns, statistics_propagation, + common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, + top_n, build_side_probe_side, compressed_materialization, duplicate_groups, + reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, + sum_rewriter, late_materialization, cte_inlining'; + +-- ----------------------------------------------------------------------- +-- Tables: note DOUBLE[3] and DOUBLE[2] — fixed-size ARRAY types so the +-- optimizer can read k_left = 3 from the type at plan time. +-- ----------------------------------------------------------------------- +CREATE OR REPLACE TABLE left_feats (id INTEGER, features DOUBLE[3]); +CREATE OR REPLACE TABLE right_feats (id INTEGER, features DOUBLE[2]); + +INSERT INTO left_feats VALUES (1, [1.0, 2.0, 3.0]), + (2, [4.0, 5.0, 6.0]); + +INSERT INTO right_feats VALUES (1, [0.1, 0.2]), + (2, [0.3, 0.4]); + +-- ----------------------------------------------------------------------- +-- Weight matrix W (5 x 2), written to /tmp so mat_mul can read it. +-- Row-major: first 3 rows belong to left side, last 2 to right side. +-- ----------------------------------------------------------------------- +COPY ( + SELECT * FROM (VALUES + (1.0, 0.0), + (0.0, 1.0), + (1.0, 1.0), + (0.5, 0.0), + (0.0, 0.5) + ) t(c1, c2) +) TO '/tmp/demo_W_5x2.csv' (FORMAT CSV, HEADER FALSE); + +-- ----------------------------------------------------------------------- +-- 1) Baseline result (no optimizer) — used to verify correctness later. +-- ----------------------------------------------------------------------- +SET enable_extended_optimizer = ''; + +SELECT + l.id, + mat_mul(list_concat(l.features, r.features), '/tmp/demo_W_5x2.csv', 'dense') AS result +FROM left_feats l +JOIN right_feats r ON l.id = r.id +ORDER BY l.id; + +-- ----------------------------------------------------------------------- +-- 2) Same query with the factorization optimizer (option '4'). +-- The terminal will print the logical plan BEFORE and AFTER the rewrite. +-- ----------------------------------------------------------------------- +SET enable_extended_optimizer = '4'; + +SELECT + l.id, + mat_mul(list_concat(l.features, r.features), '/tmp/demo_W_5x2.csv', 'dense') AS result +FROM left_feats l +JOIN right_feats r ON l.id = r.id +ORDER BY l.id; diff --git a/queries/flights_query.sql b/queries/flights_query.sql new file mode 100644 index 0000000..c85225e --- /dev/null +++ b/queries/flights_query.sql @@ -0,0 +1,90 @@ +-- Q_Flights: route attribute prediction (codeshare) via decision forest +-- Joins routes + airlines + source/destination airports, then runs forest inference. +-- Run setup_flights_db_local.py once to create flights_imbridge2.db first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/data/flights_imbridge2.db' + AS flights_db (READ_ONLY); +SET schema 'flights_db.main'; + +PRAGMA enable_profiling; +SET enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/flights.json'; + +LOAD 'cactusdb'; +SET threads = 8; + +SELECT + r.airlineid, + r.sairportid, + r.dairportid, + decision_forest_predict( + list_transform( + list_concat( + /* 1) NUMERICALS — per-column min-max scaling (exact training order) */ + min_max_scaler(list_value(CAST(s.slatitude AS DOUBLE)), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/scaler_files/slatitude.txt'), + min_max_scaler(list_value(CAST(s.slongitude AS DOUBLE)), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/scaler_files/slongitude.txt'), + min_max_scaler(list_value(CAST(d.dlatitude AS DOUBLE)), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/scaler_files/dlatitude.txt'), + min_max_scaler(list_value(CAST(d.dlongitude AS DOUBLE)), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/scaler_files/dlongitude.txt'), + + /* 2) CATEGORICALS — one-hot encoding (exact training order) */ + -- name1: integer-valued airline attribute (is_string=FALSE) + one_hot_encode(CAST(a.name1 AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/name1.csv', + FALSE), + -- name2, name4, active: 't'/'f' booleans (is_string=TRUE) + one_hot_encode(CAST(a.name2 AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/name2.csv', + TRUE), + one_hot_encode(CAST(a.name4 AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/name4.csv', + TRUE), + one_hot_encode(CAST(a.acountry AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/acountry.csv', + TRUE), + one_hot_encode(CAST(a.active AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/active.csv', + TRUE), + one_hot_encode(CAST(s.scity AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/scity.csv', + TRUE), + one_hot_encode(CAST(s.scountry AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/scountry.csv', + TRUE), + -- stimezone: integer timezone offset (is_string=FALSE) + one_hot_encode(CAST(s.stimezone AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/stimezone.csv', + FALSE), + one_hot_encode(CAST(s.sdst AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/sdst.csv', + TRUE), + one_hot_encode(CAST(d.dcity AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/dcity.csv', + TRUE), + one_hot_encode(CAST(d.dcountry AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/dcountry.csv', + TRUE), + -- dtimezone: integer timezone offset (is_string=FALSE) + one_hot_encode(CAST(d.dtimezone AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/dtimezone.csv', + FALSE), + one_hot_encode(CAST(d.ddst AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/onehotencoders/ddst.csv', + TRUE) + ), + x -> CAST(x AS FLOAT) -- decision_forest_predict expects LIST + ), + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_dot_trees_custom', + 6756, -- total features: 4 numerical + 6752 one-hot categorical + TRUE -- classification: predicting codeshare (binary) + ) AS codeshare +FROM Flights_S_routes_extension2 AS r +JOIN Flights_R1_airlines2 AS a ON r.airlineid = a.airlineid +JOIN Flights_R2_sairports AS s ON r.sairportid = s.sairportid +JOIN Flights_R3_dairports AS d ON r.dairportid = d.dairportid +WHERE a.name2 = 't' + AND a.name4 = 't' + AND a.name1 > 2.8; diff --git a/queries/forest_narrow_baseline.sql b/queries/forest_narrow_baseline.sql new file mode 100644 index 0000000..d0345a0 --- /dev/null +++ b/queries/forest_narrow_baseline.sql @@ -0,0 +1,26 @@ +LOAD cactusdb; +SET threads = 8; +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; +PRAGMA enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/plans/forest_narrow_baseline.json'; + +WITH feature_rows AS ( + SELECT + id, + list(CAST(i AS FLOAT)) AS features + FROM ( + SELECT * FROM range(0, 500) AS ids(id), + range(0, 20) AS r(i) + ) + GROUP BY id +) +SELECT + id, + decision_forest_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_narrow_1600', + 20, + TRUE + ) AS prediction +FROM feature_rows +ORDER BY id; diff --git a/queries/forest_narrow_optimized.sql b/queries/forest_narrow_optimized.sql new file mode 100644 index 0000000..03f358e --- /dev/null +++ b/queries/forest_narrow_optimized.sql @@ -0,0 +1,27 @@ +LOAD cactusdb; +SET threads = 8; +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; +SET enable_extended_optimizer = 'RULE##DecisionForestUDF2RelationRewriteAction'; +PRAGMA enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/plans/forest_narrow_optimized.json'; + +WITH feature_rows AS ( + SELECT + id, + list(CAST(i AS FLOAT)) AS features + FROM ( + SELECT * FROM range(0, 500) AS ids(id), + range(0, 20) AS r(i) + ) + GROUP BY id +) +SELECT + id, + decision_forest_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_narrow_1600', + 20, + TRUE + ) AS prediction +FROM feature_rows +ORDER BY id; diff --git a/queries/forest_query.sql b/queries/forest_query.sql index a54da06..2334973 100644 --- a/queries/forest_query.sql +++ b/queries/forest_query.sql @@ -2,8 +2,10 @@ -Load cactusdb; +LOAD cactusdb; +SET enable_extended_optimizer = 'RULE##DecisionForestUDF2RelationRewriteAction'; +EXPLAIN -- Build 3 rows of 6756-dimensional feature vectors WITH feature_rows AS ( SELECT @@ -22,7 +24,7 @@ SELECT features, decision_forest_predict( features, - '/home/local/ASUAD/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_dot_trees_custom', + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_dot_trees_custom', 6756, TRUE ) diff --git a/queries/forest_stress_baseline.sql b/queries/forest_stress_baseline.sql new file mode 100644 index 0000000..94968a5 --- /dev/null +++ b/queries/forest_stress_baseline.sql @@ -0,0 +1,26 @@ +LOAD cactusdb; +SET threads = 8; +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; +PRAGMA enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/plans/forest_stress_baseline.json'; + +WITH feature_rows AS ( + SELECT + id, + list(CAST(i AS FLOAT)) AS features + FROM ( + SELECT * FROM range(0, 500) AS ids(id), + range(0, 6756) AS r(i) + ) + GROUP BY id +) +SELECT + id, + decision_forest_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_stress_1600', + 6756, + TRUE + ) AS prediction +FROM feature_rows +ORDER BY id; diff --git a/queries/forest_stress_optimized.sql b/queries/forest_stress_optimized.sql new file mode 100644 index 0000000..8638eef --- /dev/null +++ b/queries/forest_stress_optimized.sql @@ -0,0 +1,27 @@ +LOAD cactusdb; +SET threads = 8; +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; +SET enable_extended_optimizer = 'RULE##DecisionForestUDF2RelationRewriteAction'; +PRAGMA enable_profiling = 'json'; +SET profiling_output = '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/plans/forest_stress_optimized.json'; + +WITH feature_rows AS ( + SELECT + id, + list(CAST(i AS FLOAT)) AS features + FROM ( + SELECT * FROM range(0, 500) AS ids(id), + range(0, 6756) AS r(i) + ) + GROUP BY id +) +SELECT + id, + decision_forest_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/cactusdb-duckdb-extension/resources/imbridgextenx/flights/rf_stress_1600', + 6756, + TRUE + ) AS prediction +FROM feature_rows +ORDER BY id; diff --git a/queries/synthetic_query/synthetic_query.sql b/queries/synthetic_query/synthetic_query.sql index b9205d5..22255e9 100644 --- a/queries/synthetic_query/synthetic_query.sql +++ b/queries/synthetic_query/synthetic_query.sql @@ -5,7 +5,7 @@ SET schema 'synthetic_db.synthetic'; SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, limit_pushdown, top_n, build_side_probe_side, compressed_materialization, duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, materialized_cte, sum_rewriter, late_materialization, cte_inlining'; -SET enable_extended_optimizer = '2'; +SET enable_extended_optimizer = '4'; SELECT s.impression_id AS impression_id, diff --git a/queries/uc01_query.sql b/queries/uc01_query.sql new file mode 100644 index 0000000..71bbe1b --- /dev/null +++ b/queries/uc01_query.sql @@ -0,0 +1,74 @@ +-- Q_UC01: KMeans++ customer-segmentation inference with min-max scaling. +-- Features: avg orders/year (frequency) + avg return ratio per customer. +-- Run resources/uc01/setup_uc01.py once to create all artifacts first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc01/uc01.db' + AS uc01_db (READ_ONLY); +SET schema 'uc01_db.main'; + +LOAD 'cactusdb'; + +WITH table_groups AS ( + -- Join orders + line items + returns, group per (customer, order) + SELECT + MIN(EXTRACT(YEAR FROM CAST(date AS DATE))) AS invoice_year, + SUM(quantity * price) AS row_price, + SUM(COALESCE(or_return_quantity, 0) * price) AS return_row_price, + o_customer_sk AS customer_id, + o_order_id AS order_id + FROM lineitem li + LEFT JOIN order_returns oret + ON li.li_order_id = oret.or_order_id + AND li.li_product_id = oret.or_product_id + JOIN "order" ord ON li.li_order_id = ord.o_order_id + GROUP BY o_customer_sk, o_order_id +), +table_ratios AS ( + SELECT + customer_id, + AVG(return_row_price / NULLIF(row_price, 0)) AS return_ratio + FROM table_groups + GROUP BY customer_id +), +table_frequency_groups AS ( + SELECT + customer_id, + invoice_year, + COUNT(DISTINCT order_id) AS orders_per_year + FROM table_groups + GROUP BY customer_id, invoice_year +), +table_frequency AS ( + SELECT + customer_id, + AVG(orders_per_year) AS frequency + FROM table_frequency_groups + GROUP BY customer_id +), +features_raw AS ( + SELECT + r.customer_id, + CAST(f.frequency AS DOUBLE) AS frequency, + CAST(r.return_ratio AS DOUBLE) AS return_ratio + FROM table_ratios r + JOIN table_frequency f ON r.customer_id = f.customer_id +), +features AS ( + SELECT + customer_id, + -- min-max scale [frequency, return_ratio] using the fitted scaler + min_max_scaler( + list_value(frequency, return_ratio), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc01/uc01_scaler.csv' + ) AS features + FROM features_raw +) +SELECT + customer_id, + -- assign to nearest KMeans centroid (0-based cluster index) + kmeans_predict( + features, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc01/uc01_centroids.csv' + ) AS cluster_id +FROM features +ORDER BY customer_id; diff --git a/queries/uc03_query.sql b/queries/uc03_query.sql new file mode 100644 index 0000000..50c2e41 --- /dev/null +++ b/queries/uc03_query.sql @@ -0,0 +1,75 @@ +-- Q_UC03: DNN inference over encoded store-department features +-- Architecture: Input(80) --ReLU--> 64 --ReLU--> 32 --Sigmoid--> 1 +-- Run resources/uc03/setup_uc03.py once to create all artifacts first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/store_dept.db' + AS uc03_db (READ_ONLY); +SET schema 'uc03_db.main'; + +LOAD 'cactusdb'; + +WITH base AS ( + SELECT + store, + department, + num_of_week, + one_hot_encode( + CAST(store AS VARCHAR), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/encoders/store_encoder.csv', + FALSE + ) AS store_encoded, + one_hot_encode( + department, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/encoders/dept_encoder.csv', + TRUE + ) AS dept_encoded, + list_value(CAST(num_of_week / 156.0 AS DOUBLE)) AS week_norm + FROM store_dept +), +feat AS ( + SELECT + store, + department, + num_of_week, + list_concat(store_encoded, dept_encoded, week_norm) AS features + FROM base +) +SELECT + store, + department, + num_of_week, + -- Layer 1: features -> mat_mul(W1) -> mat_add(b1) -> relu [80 -> 64] + -- Layer 2: h1 -> mat_mul(W2) -> mat_add(b2) -> relu [64 -> 32] + -- Layer 3: h2 -> mat_mul(W3) -> mat_add(b3) -> sigmoid -> scalar [32 -> 1] + list_extract( + sigmoid( + mat_add( + mat_mul( + relu( + mat_add( + mat_mul( + relu( + mat_add( + mat_mul( + features, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_W1.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_b1.csv' + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_W2.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_b2.csv' + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_W3.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc03/nn_params/uc03_b3.csv' + ) + ), + 1 + ) AS prediction +FROM feat; diff --git a/queries/uc04_query.sql b/queries/uc04_query.sql new file mode 100644 index 0000000..3e4e855 --- /dev/null +++ b/queries/uc04_query.sql @@ -0,0 +1,46 @@ +-- Q_UC04: Naive Bayes spam inference over tokenized reviews. +-- Pipeline: clean_text -> tokenize_text -> stem_token -> log-sum scoring +-- Run resources/uc04/setup_uc04.py once to create all artifacts first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc04/uc04.db' + AS uc04_db (READ_ONLY); +SET schema 'uc04_db.main'; + +LOAD 'cactusdb'; + +WITH tokens AS ( + -- UDFs: clean (strip non-alpha) -> tokenize (split) -> stem (suffix strip) + SELECT + r.id, + stem_token(tok) AS token + FROM review r, + UNNEST(tokenize_text(clean_text(r.review_text))) AS t(tok) + WHERE tok <> '' +), +token_scores AS ( + -- Accumulate log-likelihood for each class per document + SELECT + t.id, + SUM(LN(COALESCE(m.spam_prob, 1e-9))) AS log_spam_score, + SUM(LN(COALESCE(m.non_spam_prob, 1e-9))) AS log_non_spam_score + FROM tokens t + LEFT JOIN uc04_model m ON m.token = t.token + GROUP BY t.id +), +base_priors AS ( + -- Class priors computed from labeled training set + SELECT + LN(SUM(CASE WHEN spam = 1 THEN 1 ELSE 0 END) * 1.0 / COUNT(*)) AS log_spam_prior, + LN(SUM(CASE WHEN spam = 0 THEN 1 ELSE 0 END) * 1.0 / COUNT(*)) AS log_non_spam_prior + FROM uc04_train_preprocessed +) +SELECT + s.id, + CASE + WHEN s.log_spam_score + p.log_spam_prior + > s.log_non_spam_score + p.log_non_spam_prior + THEN 1 ELSE 0 + END AS predicted_spam +FROM token_scores s +CROSS JOIN base_priors p +ORDER BY s.id; diff --git a/queries/uc08_query.sql b/queries/uc08_query.sql new file mode 100644 index 0000000..163f325 --- /dev/null +++ b/queries/uc08_query.sql @@ -0,0 +1,97 @@ +-- Q_UC08: DNN inference over order-lineitem-product features; softmax output. +-- Architecture: Input(4) --ReLU--> 64 --ReLU--> 128 --Softmax--> 384 +-- features = [quantity, scan_count, weekday, dept_label] +-- Run resources/uc08/setup_uc08.py once to create all artifacts first. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/uc08.db' + AS uc08_db (READ_ONLY); +SET schema 'uc08_db.main'; + +LOAD 'cactusdb'; + +WITH ord AS ( + SELECT o_order_id, CAST(date AS TIMESTAMP) AS ts + FROM "order" +), +ord2 AS ( + SELECT o_order_id, ts, dayofweek(ts) AS weekday + FROM ord +), +j1 AS ( + SELECT + o.o_order_id, o.ts, o.weekday, + li.li_product_id, li.quantity + FROM ord2 o + JOIN lineitem li ON o.o_order_id = li.li_order_id +), +j2 AS ( + SELECT + j1.o_order_id, j1.ts, j1.weekday, + j1.quantity, p.department + FROM j1 + JOIN product p ON j1.li_product_id = p.p_product_id +), +agg AS ( + SELECT + o_order_id, + ts AS date, + department, + quantity, + SUM(quantity) AS scan_count, + MIN(weekday) AS weekday + FROM j2 + GROUP BY o_order_id, ts, department, quantity +), +feat AS ( + SELECT + o_order_id, + date, + -- 4-dim feature: [quantity, scan_count, weekday, dept_label] + -- dept_label via argmax of one-hot encoding = integer index 0..46 + list_value( + CAST(quantity AS DOUBLE), + CAST(scan_count AS DOUBLE), + CAST(weekday AS DOUBLE), + CAST(ml_argmax(one_hot_encode( + department, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/encoders/dept_label_encoder.csv', + TRUE + )) AS DOUBLE) + ) AS features + FROM agg +) +SELECT + o_order_id, + date, + -- Layer 1: features -> mat_mul(W1) -> mat_add(b1) -> relu [4 -> 64] + -- Layer 2: h1 -> mat_mul(W2) -> mat_add(b2) -> relu [64 -> 128] + -- Layer 3: h2 -> mat_mul(W3) -> mat_add(b3) -> softmax [128 -> 384] + cdb_softmax( + mat_add( + mat_mul( + relu( + mat_add( + mat_mul( + relu( + mat_add( + mat_mul( + features, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_W1.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_b1.csv' + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_W2.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_b2.csv' + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_W3.csv', + 'dense' + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_b3.csv' + ) + ) AS prediction +FROM feat; diff --git a/queries/uc08_query_factorizable.sql b/queries/uc08_query_factorizable.sql new file mode 100644 index 0000000..ffc3304 --- /dev/null +++ b/queries/uc08_query_factorizable.sql @@ -0,0 +1,63 @@ +-- Q_UC08 (factorizable form) +-- Single-layer: mat_mul(list_concat(LEFT_3, RIGHT_47), W1_wide[50x64]) +-- Right join side uses the full 47-dim one-hot encoding of department. +-- +-- With enable_extended_optimizer='4', the factorization action will: +-- 1. Detect mat_mul(list_concat(LEFT_3, RIGHT_47), W1_wide) over the COMPARISON_JOIN. +-- 2. Split W1_wide into W1_left (3x64) and W1_right (47x64). +-- 3. Push mat_mul(left_features, W1_left) into a projection on the left side (2.1M rows). +-- 4. Push mat_mul(right_features, W1_right) into a projection on the right side (707 rows). +-- 5. Replace the original mat_mul with list_add_vv(#left_col, #right_col). +-- +-- Speedup: mat_mul(47-dim, W1_right[47x64]) runs 707 times instead of 2.1M times. + +ATTACH '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/uc08.db' + AS uc08_db (READ_ONLY); +SET schema 'uc08_db.main'; + +SET disabled_optimizers='expression_rewriter, empty_result_pullup, cte_filter_pusher, + regex_range, in_clause, join_order, deliminator, unnest_rewriter, unused_columns, + statistics_propagation, common_subexpressions, common_aggregate, column_lifetime, + limit_pushdown, top_n, build_side_probe_side, compressed_materialization, + duplicate_groups, reorder_filter, sampling_pushdown, join_filter_pushdown, + materialized_cte, sum_rewriter, late_materialization, cte_inlining'; + +SET enable_extended_optimizer = '4'; + +LOAD 'cactusdb'; + +SELECT + a.o_order_id, + a.date, + mat_mul( + list_concat( + -- LEFT_3: [quantity, scan_count, weekday] from order+lineitem agg + list_value( + CAST(a.quantity AS DOUBLE), + CAST(a.scan_count AS DOUBLE), + CAST(a.weekday AS DOUBLE) + ), + -- RIGHT_47: full one-hot dept vector from product (47-dim) + -- Factorization pushes this mat_mul to the product scan (707 rows). + one_hot_encode( + p.department, + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/encoders/dept_label_encoder.csv', + TRUE + ) + ), + '/home/asurite.ad.asu.edu/jrtandel/Documents/OPTBench/resources/uc08/nn_params/uc08_W1_wide.csv', + 'dense' + ) AS h1 +FROM ( + SELECT + o.o_order_id, + CAST(o.date AS DATE) AS date, + li.li_product_id, + li.quantity, + SUM(li.quantity) AS scan_count, + MIN(dayofweek(CAST(o.date AS TIMESTAMP))) AS weekday + FROM order_tbl o + JOIN lineitem li ON o.o_order_id = li.li_order_id + GROUP BY o.o_order_id, o.date, li.li_product_id, li.quantity +) a +JOIN product p ON a.li_product_id = p.p_product_id; diff --git a/src/cactusdb_extension.cpp b/src/cactusdb_extension.cpp index 186be78..0b562c4 100644 --- a/src/cactusdb_extension.cpp +++ b/src/cactusdb_extension.cpp @@ -2,6 +2,7 @@ #include "duckdb.hpp" #include "functions.hpp" #include "optimization/register_optimizers.hpp" +#include "optimization/optimizer_pass.hpp" #include "duckdb/main/database.hpp" // OpenSSL linked through vcpkg @@ -23,6 +24,8 @@ static void LoadInternal(ExtensionLoader &loader) { RegisterMLScalarFunctions(loader); RegisterCactusOptimizers(db); + RegisterOptimizerPassListFunction(loader); + LinkOptPassPlugins(); // pulls uploaded plugin objects into the static link } diff --git a/src/include/ml_functions/kmeans.hpp b/src/include/ml_functions/kmeans.hpp new file mode 100644 index 0000000..d59998a --- /dev/null +++ b/src/include/ml_functions/kmeans.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +#include "duckdb.hpp" +#include "duckdb/function/function.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +// kmeans_predict(feature_list LIST, centroids_file VARCHAR) -> INTEGER +// Loads a CSV of K centroids (K rows, D columns) at bind time and returns +// the 0-based index of the nearest centroid for each feature vector. +struct ML_KMeansPredict { + struct BindData final : public FunctionData { + std::vector> centroids; // [K][D] + idx_t K = 0; + idx_t D = 0; + std::string file_path; + + BindData() = default; + BindData(std::vector> c, idx_t k, idx_t d, std::string p) + : centroids(std::move(c)), K(k), D(d), file_path(std::move(p)) {} + + unique_ptr Copy() const override { + return make_uniq(centroids, K, D, file_path); + } + bool Equals(const FunctionData &o) const override { + return file_path == o.Cast().file_path; + } + }; + + static unique_ptr Bind(ClientContext &context, + ScalarFunction &bound_function, + vector> &args); + + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + + static void Register(ExtensionLoader &loader); +}; + +} // namespace duckdb diff --git a/src/include/ml_functions/list_add_vv.hpp b/src/include/ml_functions/list_add_vv.hpp new file mode 100644 index 0000000..418c463 --- /dev/null +++ b/src/include/ml_functions/list_add_vv.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/function/function.hpp" + +namespace duckdb { + +// Element-wise addition of two LIST vectors at runtime. +// result[j] = lhs[j] + rhs[j] for each element j in each row. +// Used by MLFactorizationRewriteAction to combine the two partial +// mat_mul results from each side of a join. +struct ML_ListAddVV { + static void Register(ExtensionLoader &loader); + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); +}; + +} // namespace duckdb diff --git a/src/include/ml_functions/one_hot_encoder.hpp b/src/include/ml_functions/one_hot_encoder.hpp index 17b8c8e..5ca2070 100644 --- a/src/include/ml_functions/one_hot_encoder.hpp +++ b/src/include/ml_functions/one_hot_encoder.hpp @@ -28,10 +28,14 @@ static void one_hot_encoder_bind_constructor( if (is_string) { while (std::getline(in, line)) { - std::istringstream ss(line); - std::string valStr, posStr; - if (!std::getline(ss, valStr, ',')) continue; - if (!std::getline(ss, posStr, ',')) continue; + // Split on the LAST comma so category names containing commas work. + auto sep = line.rfind(','); + if (sep == std::string::npos) continue; + std::string valStr = line.substr(0, sep); + std::string posStr = line.substr(sep + 1); + auto l = posStr.find_first_not_of(" \t\r\n"); + if (l == std::string::npos) continue; + posStr = posStr.substr(l); int position = std::stoi(posStr); str_map[valStr] = position; } diff --git a/src/include/ml_functions/relu.hpp b/src/include/ml_functions/relu.hpp new file mode 100644 index 0000000..84a431c --- /dev/null +++ b/src/include/ml_functions/relu.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include + +#include "duckdb.hpp" +#include "duckdb/function/function.hpp" + +namespace duckdb { + +struct ML_Relu { + static unique_ptr Bind(ClientContext &context, + ScalarFunction &bound_function, + vector> &args); + + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + + static void Register(ExtensionLoader &loader); +}; + +} // namespace duckdb diff --git a/src/include/ml_functions/text_functions.hpp b/src/include/ml_functions/text_functions.hpp new file mode 100644 index 0000000..969b765 --- /dev/null +++ b/src/include/ml_functions/text_functions.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/function/function.hpp" + +namespace duckdb { + +// clean_text(VARCHAR) -> VARCHAR +// Strips non-alphabetic characters and collapses whitespace. +struct ML_CleanText { + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + static void Register(ExtensionLoader &loader); +}; + +// tokenize_text(VARCHAR) -> LIST +// Splits a cleaned string on whitespace; returns only non-empty tokens. +struct ML_TokenizeText { + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + static void Register(ExtensionLoader &loader); +}; + +// stem_token(VARCHAR) -> VARCHAR +// Simple suffix-stripping stemmer (handles -ing, -ed, -er, -ion, -es, -s). +struct ML_StemToken { + static void Execute(DataChunk &args, ExpressionState &state, Vector &result); + static void Register(ExtensionLoader &loader); +}; + +} // namespace duckdb diff --git a/src/include/optimization/actions/MLFactorizationRewriteAction.hpp b/src/include/optimization/actions/MLFactorizationRewriteAction.hpp new file mode 100644 index 0000000..b9661ce --- /dev/null +++ b/src/include/optimization/actions/MLFactorizationRewriteAction.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include "duckdb.hpp" + +namespace duckdb { + +// MLFactorizationRewriteAction +// +// Targets: +// PROJECTION( mat_mul(list_concat(L, R), W_file, mode) ) +// COMPARISON_JOIN +// left_child <- L references only columns from here +// right_child <- R references only columns from here +// +// Rewrites to: +// PROJECTION( list_add_vv(ref_left_result, ref_right_result) ) +// COMPARISON_JOIN +// PROJECTION( mat_mul(L, W_top_file, mode) ) <- pushed to left child +// left_child +// PROJECTION( mat_mul(R, W_bottom_file, mode) ) <- pushed to right child +// right_child +// +// W_top = rows [0, k_left) of W (shape k_left x n) +// W_bottom = rows [k_left, k) of W (shape k_right x n) +// +// k_left is read from the static type of L: +// - DOUBLE[N] array column -> ArrayType::GetSize() +// - list_value(c1, c2, ...) -> child count +// - LIST column -> optimization skipped (unknown at plan time) +class MLFactorizationRewriteAction { +public: + static bool check(OptimizerExtensionInput &input, unique_ptr &plan); + static bool apply(OptimizerExtensionInput &input, unique_ptr &plan); +}; + +} // namespace duckdb diff --git a/src/include/optimization/actions/MLGreedyFactorizationOptimizer.hpp b/src/include/optimization/actions/MLGreedyFactorizationOptimizer.hpp new file mode 100644 index 0000000..cef9248 --- /dev/null +++ b/src/include/optimization/actions/MLGreedyFactorizationOptimizer.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "duckdb.hpp" + +namespace duckdb { + +class MLGreedyFactorizationOptimizer { +public: + static bool check(OptimizerExtensionInput &input, unique_ptr &plan); + static bool apply(OptimizerExtensionInput &input, unique_ptr &plan); +}; + +} // namespace duckdb diff --git a/src/include/optimization/dynamic_rule.hpp b/src/include/optimization/dynamic_rule.hpp new file mode 100644 index 0000000..063aa6c --- /dev/null +++ b/src/include/optimization/dynamic_rule.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/optimizer/optimizer_extension.hpp" + +#include +#include + +namespace duckdb { + +// A single metric comparison parsed from a frontend rule, e.g. +// join_cardinality_ratio > 1e-5 +// is_or carries the logic operator that joins this condition to the previous +// one (false = AND, true = OR); the first condition's is_or is ignored. +struct RuleCondition { + std::string metric; + std::string op; // > < >= <= = == != + double value = 0.0; + bool is_or = false; +}; + +// A user-authored optimizer rule serialized over the enable_extended_optimizer +// setting as: RULE## +// conditions: " " joined by " AND " / " OR " +// actions: comma-separated rewrite-action class names +// +// Conditions are evaluated against MetricCatalog for the query's schema; if +// they pass, the named actions are dispatched in order. +class DynamicRule { +public: + std::vector conditions; + std::vector actions; + + static bool IsRuleString(const std::string &s) { + return s.rfind("RULE#", 0) == 0; + } + + static DynamicRule Parse(const std::string &rule_str); + + bool EvaluateConditions(const std::string &schema) const; + void Apply(OptimizerExtensionInput &input, unique_ptr &plan) const; +}; + +} // namespace duckdb diff --git a/src/include/optimization/metric_catalog.hpp b/src/include/optimization/metric_catalog.hpp index 6f90436..3e1e2a5 100644 --- a/src/include/optimization/metric_catalog.hpp +++ b/src/include/optimization/metric_catalog.hpp @@ -62,37 +62,102 @@ class MetricCatalog { return schema->Get(metric_name, default_value); } + // Resolve a metric trying the query's schema first, then the "_default" + // schema, then the supplied fallback. Used by the dynamic rule evaluator + // where GetQuerySchema() may return an unexpected name (e.g. "main"). + double GetMetricOrDefault(const string &schema_name, + const string &metric_name, + double default_value) const { + if (auto *s = GetSchemaMetrics(schema_name); s && s->Has(metric_name)) { + return s->Get(metric_name); + } + if (auto *d = GetSchemaMetrics("_default"); d && d->Has(metric_name)) { + return d->Get(metric_name); + } + return default_value; + } + private: MetricCatalog() { Initialize(); } + // Populate a schema set with the frontend metric names used by the + // optimizer-creator rules (must match the keys in app.py METRICS). + static void AddFrontendMetrics(SchemaMetricSet &s) { + s.values["join_cardinality_ratio"] = 0.0002; + s.values["feature_nonzero_ratio"] = 0.002; + s.values["feature_width"] = 150; + s.values["card_search_impressions"] = 500000; + s.values["card_listing_metadata"] = 200000; + s.values["ml_flops"] = 280e6; // conservative default — below 5B threshold + } + void Initialize() { - // ---- Define metrics for each synthetic schema here ---- - // Example schema 1: synthetic dense features + // ---- Synthetic schemas (in-memory; GetQuerySchema returns just the schema name) ---- { SchemaMetricSet s; - s.values["join_ratio"] = 5e-5; // big joins - s.values["sparsity"] = 0.20; // 95% non-zero -> dense + s.values["join_ratio"] = 5e-5; + s.values["sparsity"] = 0.20; + s.values["ml_flops"] = 280e6; + AddFrontendMetrics(s); schema_metrics["synthetic"] = std::move(s); } - - // Example schema 2: synthetic sparse features { SchemaMetricSet s; s.values["join_ratio"] = 5e-5; - s.values["sparsity"] = 0.10; // 10% non-zero -> sparse + s.values["sparsity"] = 0.10; + s.values["ml_flops"] = 280e6; + AddFrontendMetrics(s); schema_metrics["syn_sparse"] = std::move(s); } - // Add more schemas as needed: - // { - // SchemaMetricSet s; - // s.values["join_ratio"] = ...; - // s.values["sparsity"] = ...; - // s.values["some_other_metric"] = ...; - // schema_metrics["my_schema_name"] = std::move(s); - // } + // ---- UC08: order-feature DNN, 47-dim one-hot right side, fan-out=3001x ---- + // ml_flops = 6.79B → above 5B threshold → factorization candidate + { + SchemaMetricSet s; + s.values["join_ratio"] = 0.0003; + s.values["sparsity"] = 0.021; + s.values["ml_flops"] = 6.79e9; + s.values["join_cardinality_ratio"] = 0.0003; + schema_metrics["uc08_db.main"] = std::move(s); + } + + // ---- UC03: store-department DNN, low ml_flops ---- + { + SchemaMetricSet s; + s.values["join_ratio"] = 0.0; + s.values["sparsity"] = 0.025; + s.values["ml_flops"] = 280e6; + schema_metrics["uc03_db.main"] = std::move(s); + } + + // ---- Expedia: decision-tree inference, multi-join ---- + { + SchemaMetricSet s; + s.values["join_ratio"] = 1e-4; + s.values["sparsity"] = 0.15; + s.values["ml_flops"] = 1.2e9; + schema_metrics["expedia_db.main"] = std::move(s); + } + + // ---- Flights: decision-forest inference, multi-join ---- + { + SchemaMetricSet s; + s.values["join_ratio"] = 1e-4; + s.values["sparsity"] = 0.12; + s.values["ml_flops"] = 2.1e9; + schema_metrics["flights_db.main"] = std::move(s); + } + + // ---- Fallback: any schema not listed above ---- + { + SchemaMetricSet s; + s.values["join_ratio"] = 5e-5; + s.values["sparsity"] = 0.20; + AddFrontendMetrics(s); + schema_metrics["_default"] = std::move(s); + } } private: diff --git a/src/include/optimization/optimizer_pass.hpp b/src/include/optimization/optimizer_pass.hpp new file mode 100644 index 0000000..4db3bff --- /dev/null +++ b/src/include/optimization/optimizer_pass.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/optimizer/optimizer_extension.hpp" + +#include +#include +#include + +namespace duckdb { + +class ExtensionLoader; + +// A self-contained optimizer pass uploaded as a plugin. The function-pointer +// signature deliberately matches the static check/apply methods every existing +// rewrite-action class already exposes (e.g. MLGreedyFactorizationOptimizer), so +// an existing pass becomes a plugin by adding a single REGISTER_OPT_PASS line. +struct OptimizerPass { + std::string name; + bool (*check)(OptimizerExtensionInput &, unique_ptr &); + bool (*apply)(OptimizerExtensionInput &, unique_ptr &); +}; + +// Process-wide registry of optimizer passes, populated by static initializers in +// compiled-in plugin sources (see REGISTER_OPT_PASS). Selected at query time via +// the "OPT#" enable_extended_optimizer setting. +class OptimizerPassRegistry { +public: + static OptimizerPassRegistry &Get(); // Meyers singleton + + void Register(OptimizerPass pass); // last-wins on duplicate name + const OptimizerPass *Find(const std::string &name) const; // nullptr if absent + std::vector Names() const; // registered names, sorted + +private: + std::unordered_map passes_; +}; + +// Registers the cactusdb_list_opt_passes() table function, which returns the +// names currently in OptimizerPassRegistry so the frontend can list the passes +// that are actually compiled in (single source of truth, no Python drift). +void RegisterOptimizerPassListFunction(ExtensionLoader &loader); + +// Called once at extension load. Its body lives in the AUTO-GENERATED +// plugins/opt_passes/_manifest.cpp, which references each plugin's anchor symbol +// (see REGISTER_OPT_PASS). That reference is what forces the static linker to +// retain each plugin object file — and therefore run its REGISTER_OPT_PASS +// initializer — when cactusdb is linked into the duckdb CLI as a static archive +// (object files with only static initializers are otherwise dropped). Always +// defined (the baseline manifest has an empty body). +void LinkOptPassPlugins(); + +} // namespace duckdb + +// Self-registration. Given a bare identifier IDENT (the pass name), this: +// 1. defines an extern "C" anchor symbol the generated manifest references +// (forces archive retention — see LinkOptPassPlugins), and +// 2. defines a file-scope object whose ctor registers the pass at load time, +// using the stringified IDENT as the registry name. +// Usage: +// REGISTER_OPT_PASS(greedy_factorization, &MyPass::check, &MyPass::apply); +#define REGISTER_OPT_PASS(IDENT, CHECK_FN, APPLY_FN) \ + extern "C" void optpass_anchor_##IDENT() {} \ + namespace { \ + struct OptPassReg_##IDENT { \ + OptPassReg_##IDENT() { \ + ::duckdb::OptimizerPassRegistry::Get().Register({#IDENT, (CHECK_FN), (APPLY_FN)}); \ + } \ + } g_optpassreg_##IDENT; \ + } diff --git a/src/ml_functions/CMakeLists.txt b/src/ml_functions/CMakeLists.txt index 3d9987f..d7fb609 100644 --- a/src/ml_functions/CMakeLists.txt +++ b/src/ml_functions/CMakeLists.txt @@ -5,12 +5,16 @@ set(ML_FUNCTIONS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/one_hot_encoder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/decision_tree.cpp ${CMAKE_CURRENT_SOURCE_DIR}/matrix_addition.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/list_add_vv.cpp ${CMAKE_CURRENT_SOURCE_DIR}/argmax.cpp ${CMAKE_CURRENT_SOURCE_DIR}/softmax.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sigmoid.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/relu.cpp ${CMAKE_CURRENT_SOURCE_DIR}/torch_dnn.cpp ${CMAKE_CURRENT_SOURCE_DIR}/decision_forest.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ml_decision_forest_table.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/text_functions.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/kmeans.cpp ) set(EXTENSION_SOURCES diff --git a/src/ml_functions/kmeans.cpp b/src/ml_functions/kmeans.cpp new file mode 100644 index 0000000..a990e24 --- /dev/null +++ b/src/ml_functions/kmeans.cpp @@ -0,0 +1,135 @@ +#include "ml_functions/kmeans.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/types/vector.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +// ─── Bind ───────────────────────────────────────────────────────────────────── + +unique_ptr ML_KMeansPredict::Bind(ClientContext &context, + ScalarFunction &, + vector> &args) { + if (args.size() != 2) { + throw BinderException("kmeans_predict: expects 2 arguments: features LIST, centroids_file VARCHAR"); + } + + auto v_path = ExpressionExecutor::EvaluateScalar(context, *args[1]); + if (v_path.IsNull()) throw BinderException("kmeans_predict: centroids_file is NULL"); + const std::string path = v_path.ToString(); + + if (!std::filesystem::exists(path)) { + throw InvalidInputException("kmeans_predict: file not found: %s", path); + } + + // Parse CSV: one centroid per line, values comma-separated + std::vector> centroids; + std::ifstream f(path); + std::string line; + while (std::getline(f, line)) { + if (line.empty()) continue; + std::vector row; + std::istringstream iss(line); + std::string cell; + while (std::getline(iss, cell, ',')) { + row.push_back(std::stod(cell)); + } + if (!row.empty()) centroids.push_back(std::move(row)); + } + + if (centroids.empty()) { + throw InvalidInputException("kmeans_predict: no centroids found in %s", path); + } + const idx_t K = centroids.size(); + const idx_t D = centroids[0].size(); + for (idx_t k = 1; k < K; ++k) { + if (centroids[k].size() != D) { + throw InvalidInputException("kmeans_predict: centroid dimension mismatch at row %llu", (unsigned long long)k); + } + } + + return make_uniq(std::move(centroids), K, D, path); +} + +// ─── Execute ────────────────────────────────────────────────────────────────── + +void ML_KMeansPredict::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + const auto &info = state.expr.Cast().bind_info->Cast(); + const idx_t K = info.K; + const idx_t D = info.D; + + // Input feature list + Vector &feat = args.data[0]; + UnifiedVectorFormat feat_uvf; + feat.ToUnifiedFormat(count, feat_uvf); + const auto *entries = UnifiedVectorFormat::GetData(feat_uvf); + auto &child = ListVector::GetEntry(feat); + UnifiedVectorFormat child_uvf; + child.ToUnifiedFormat(ListVector::GetListSize(feat), child_uvf); + const double *feat_data = UnifiedVectorFormat::GetData(child_uvf); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto *out = FlatVector::GetData(result); + auto &out_valid = FlatVector::Validity(result); + + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = feat_uvf.sel->get_index(i); + if (!feat_uvf.validity.RowIsValid(sel)) { + out_valid.Set(i, false); + continue; + } + + const auto le = entries[sel]; + if (le.length != D) { + throw InvalidInputException( + "kmeans_predict: feature length %llu does not match centroid dimension %llu", + (unsigned long long)le.length, (unsigned long long)D); + } + + const double *x = feat_data + le.offset; + + double best_dist = std::numeric_limits::max(); + int32_t best_k = 0; + for (idx_t k = 0; k < K; ++k) { + double dist = 0.0; + for (idx_t d = 0; d < D; ++d) { + double diff = x[d] - info.centroids[k][d]; + dist += diff * diff; + } + if (dist < best_dist) { + best_dist = dist; + best_k = static_cast(k); + } + } + out[i] = best_k; + } +} + +// ─── Register ───────────────────────────────────────────────────────────────── + +void ML_KMeansPredict::Register(ExtensionLoader &loader) { + ScalarFunction fn( + "kmeans_predict", + {LogicalType::LIST(LogicalType::DOUBLE), LogicalType::VARCHAR}, + LogicalType::INTEGER, + Execute + ); + fn.bind = Bind; + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + +} // namespace duckdb diff --git a/src/ml_functions/list_add_vv.cpp b/src/ml_functions/list_add_vv.cpp new file mode 100644 index 0000000..e6185db --- /dev/null +++ b/src/ml_functions/list_add_vv.cpp @@ -0,0 +1,100 @@ +#include "ml_functions/list_add_vv.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/vector_operations/generic_executor.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" + +namespace duckdb { + +void ML_ListAddVV::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + result.SetVectorType(VectorType::FLAT_VECTOR); + + Vector &lhs = args.data[0]; + Vector &rhs = args.data[1]; + + // Unify top-level list entries + UnifiedVectorFormat lhs_uvf, rhs_uvf; + lhs.ToUnifiedFormat(count, lhs_uvf); + rhs.ToUnifiedFormat(count, rhs_uvf); + + const auto *lhs_entries = UnifiedVectorFormat::GetData(lhs_uvf); + const auto *rhs_entries = UnifiedVectorFormat::GetData(rhs_uvf); + + // Unify child (element) vectors + auto &lhs_child = ListVector::GetEntry(lhs); + auto &rhs_child = ListVector::GetEntry(rhs); + + const idx_t lhs_total = ListVector::GetListSize(lhs); + const idx_t rhs_total = ListVector::GetListSize(rhs); + + UnifiedVectorFormat lhs_child_uvf, rhs_child_uvf; + lhs_child.ToUnifiedFormat(lhs_total, lhs_child_uvf); + rhs_child.ToUnifiedFormat(rhs_total, rhs_child_uvf); + + const double *lhs_data = UnifiedVectorFormat::GetData(lhs_child_uvf); + const double *rhs_data = UnifiedVectorFormat::GetData(rhs_child_uvf); + + // Pre-scan: count total valid output elements + idx_t total_out = 0; + for (idx_t i = 0; i < count; ++i) { + idx_t li = lhs_uvf.sel->get_index(i); + idx_t ri = rhs_uvf.sel->get_index(i); + if (lhs_uvf.validity.RowIsValid(li) && rhs_uvf.validity.RowIsValid(ri)) { + total_out += lhs_entries[li].length; + } + } + + auto *res_entries = FlatVector::GetData(result); + auto &res_child = ListVector::GetEntry(result); + auto &res_valid = FlatVector::Validity(result); + + ListVector::Reserve(result, total_out); + ListVector::SetListSize(result, total_out); + double *out = FlatVector::GetData(res_child); + + idx_t cursor = 0; + for (idx_t i = 0; i < count; ++i) { + idx_t li = lhs_uvf.sel->get_index(i); + idx_t ri = rhs_uvf.sel->get_index(i); + + if (!lhs_uvf.validity.RowIsValid(li) || !rhs_uvf.validity.RowIsValid(ri)) { + res_entries[i] = {0, 0}; + res_valid.Set(i, false); + continue; + } + + const auto &le = lhs_entries[li]; + const auto &re = rhs_entries[ri]; + + if (le.length != re.length) { + throw InvalidInputException( + "list_add_vv: length mismatch at row %llu (%llu vs %llu)", + (unsigned long long)i, + (unsigned long long)le.length, + (unsigned long long)re.length); + } + + res_entries[i] = {cursor, le.length}; + res_valid.Set(i, true); + + const double *lp = lhs_data + le.offset; + const double *rp = rhs_data + re.offset; + for (idx_t j = 0; j < le.length; ++j) { + out[cursor + j] = lp[j] + rp[j]; + } + cursor += le.length; + } +} + +void ML_ListAddVV::Register(ExtensionLoader &loader) { + ScalarFunction fn( + "list_add_vv", + {LogicalType::LIST(LogicalType::DOUBLE), LogicalType::LIST(LogicalType::DOUBLE)}, + LogicalType::LIST(LogicalType::DOUBLE), + Execute); + loader.RegisterFunction(fn); +} + +} // namespace duckdb diff --git a/src/ml_functions/ml_functions_scalar.cpp b/src/ml_functions/ml_functions_scalar.cpp index 7c9d539..e287128 100644 --- a/src/ml_functions/ml_functions_scalar.cpp +++ b/src/ml_functions/ml_functions_scalar.cpp @@ -4,12 +4,16 @@ #include "ml_functions/one_hot_encoder.hpp" #include "ml_functions/decision_tree.hpp" #include "ml_functions/matrix_addition.hpp" +#include "ml_functions/list_add_vv.hpp" #include "ml_functions/argmax.hpp" #include "ml_functions/softmax.hpp" #include "ml_functions/sigmoid.hpp" +#include "ml_functions/relu.hpp" #include "ml_functions/torch_dnn.hpp" #include "ml_functions/decision_forest.hpp" #include "ml_functions/ml_decision_forest_table.hpp" +#include "ml_functions/text_functions.hpp" +#include "ml_functions/kmeans.hpp" namespace duckdb { @@ -20,12 +24,18 @@ void RegisterMLScalarFunctions(ExtensionLoader &loader) { ML_OneHotEncoder::Register(loader); ML_DecisionTree::Register(loader); ML_MatrixAddition::Register(loader); + ML_ListAddVV::Register(loader); ML_Argmax::Register(loader); ML_Softmax::Register(loader); ML_Sigmoid::Register(loader); + ML_Relu::Register(loader); ML_TorchDNN::Register(loader); ML_DecisionForest::Register(loader); ML_ForestTable::Register(loader); + ML_CleanText::Register(loader); + ML_TokenizeText::Register(loader); + ML_StemToken::Register(loader); + ML_KMeansPredict::Register(loader); // RegisterSoftmax(loader); // RegisterSigmoid(loader); } diff --git a/src/ml_functions/relu.cpp b/src/ml_functions/relu.cpp new file mode 100644 index 0000000..704a322 --- /dev/null +++ b/src/ml_functions/relu.cpp @@ -0,0 +1,90 @@ +#include "ml_functions/relu.hpp" + +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/common/vector.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +unique_ptr ML_Relu::Bind(ClientContext &context, + ScalarFunction &bound_function, + vector> &args) { + if (args.size() != 1) { + throw BinderException("relu: expects exactly 1 argument: LIST"); + } + return nullptr; +} + +void ML_Relu::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + Vector &in = args.data[0]; + UnifiedVectorFormat in_uvf; + in.ToUnifiedFormat(count, in_uvf); + const auto *entries = UnifiedVectorFormat::GetData(in_uvf); + + auto &child = ListVector::GetEntry(in); + const idx_t total_elems = ListVector::GetListSize(in); + UnifiedVectorFormat child_uvf; + child.ToUnifiedFormat(total_elems, child_uvf); + const double *child_data = UnifiedVectorFormat::GetData(child_uvf); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto *res_entries = FlatVector::GetData(result); + auto &res_child = ListVector::GetEntry(result); + auto &res_valid = FlatVector::Validity(result); + + ListVector::Reserve(result, total_elems); + + idx_t out_cursor = 0; + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + + if (!in_uvf.validity.RowIsValid(sel)) { + res_entries[i] = {0, 0}; + res_valid.Set(i, false); + continue; + } + + const auto le = entries[sel]; + const idx_t len = le.length; + const idx_t off = le.offset; + + if (len == 0) { + throw InvalidInputException("relu: empty list at row %llu", (unsigned long long)i); + } + + using RowVec = Eigen::Matrix; + Eigen::Map row(child_data + off, (Eigen::Index)len); + // ReLU: max(0, x) element-wise + RowVec out_row = row.array().max(0.0).matrix(); + + const idx_t prev_total = ListVector::GetListSize(result); + ListVector::SetListSize(result, prev_total + len); + double *out = FlatVector::GetData(res_child); + + res_entries[i].offset = out_cursor; + res_entries[i].length = len; + std::memcpy(out + out_cursor, out_row.data(), sizeof(double) * (size_t)len); + out_cursor += len; + } +} + +void ML_Relu::Register(ExtensionLoader &loader) { + ScalarFunction fn( + "relu", + { LogicalType::LIST(LogicalType::DOUBLE) }, + LogicalType::LIST(LogicalType::DOUBLE), + Execute + ); + fn.bind = Bind; + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + +} // namespace duckdb diff --git a/src/ml_functions/text_functions.cpp b/src/ml_functions/text_functions.cpp new file mode 100644 index 0000000..245c6c9 --- /dev/null +++ b/src/ml_functions/text_functions.cpp @@ -0,0 +1,205 @@ +#include "ml_functions/text_functions.hpp" + +#include +#include +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/types/vector.hpp" +#include "duckdb/common/vector_operations/vector_operations.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +// ─── clean_text ─────────────────────────────────────────────────────────────── +// Replaces any character that is not a-z/A-Z/space with a space, then +// collapses runs of whitespace into a single space and trims ends. + +void ML_CleanText::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + Vector &in = args.data[0]; + UnifiedVectorFormat in_uvf; + in.ToUnifiedFormat(count, in_uvf); + const string_t *in_data = UnifiedVectorFormat::GetData(in_uvf); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto &res_valid = FlatVector::Validity(result); + + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + if (!in_uvf.validity.RowIsValid(sel)) { + res_valid.Set(i, false); + FlatVector::GetData(result)[i] = string_t(); + continue; + } + + std::string s = in_data[sel].GetString(); + // Replace non-alpha with space + for (char &c : s) { + if (!std::isalpha((unsigned char)c)) c = ' '; + } + // Collapse whitespace + std::string out; + out.reserve(s.size()); + bool prev_space = true; // leading trim + for (char c : s) { + if (c == ' ') { + if (!prev_space) out += ' '; + prev_space = true; + } else { + out += c; + prev_space = false; + } + } + // Trailing trim + if (!out.empty() && out.back() == ' ') out.pop_back(); + + FlatVector::GetData(result)[i] = StringVector::AddString(result, out); + } +} + +void ML_CleanText::Register(ExtensionLoader &loader) { + ScalarFunction fn("clean_text", {LogicalType::VARCHAR}, LogicalType::VARCHAR, Execute); + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + + +// ─── tokenize_text ──────────────────────────────────────────────────────────── +// Splits a string on whitespace and returns LIST of tokens. + +void ML_TokenizeText::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + Vector &in = args.data[0]; + UnifiedVectorFormat in_uvf; + in.ToUnifiedFormat(count, in_uvf); + const string_t *in_data = UnifiedVectorFormat::GetData(in_uvf); + + // Pre-compute tokens for every row + std::vector> all_tokens(count); + idx_t total = 0; + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + if (!in_uvf.validity.RowIsValid(sel)) continue; + + std::istringstream iss(in_data[sel].GetString()); + std::string tok; + while (iss >> tok) { + if (!tok.empty()) { + all_tokens[i].push_back(std::move(tok)); + ++total; + } + } + } + + result.SetVectorType(VectorType::FLAT_VECTOR); + ListVector::Reserve(result, total); + + auto *entries = FlatVector::GetData(result); + auto &child = ListVector::GetEntry(result); + auto &res_valid = FlatVector::Validity(result); + auto *child_data = FlatVector::GetData(child); + + idx_t offset = 0; + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + if (!in_uvf.validity.RowIsValid(sel)) { + res_valid.Set(i, false); + entries[i] = {0, 0}; + continue; + } + const auto &toks = all_tokens[i]; + entries[i] = {offset, toks.size()}; + for (const auto &t : toks) { + child_data[offset] = StringVector::AddString(child, t); + ++offset; + } + } + ListVector::SetListSize(result, offset); +} + +void ML_TokenizeText::Register(ExtensionLoader &loader) { + ScalarFunction fn( + "tokenize_text", + {LogicalType::VARCHAR}, + LogicalType::LIST(LogicalType::VARCHAR), + Execute + ); + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + + +// ─── stem_token ─────────────────────────────────────────────────────────────── +// Simple suffix-stripping stemmer. Handles the most common English endings. +// Minimum stem length of 3 is preserved to avoid over-stemming. + +static std::string simple_stem(std::string w) { + const idx_t min_stem = 3; + auto try_strip = [&](const char *suffix, idx_t suffix_len) -> bool { + if (w.size() <= min_stem + suffix_len) return false; + if (w.compare(w.size() - suffix_len, suffix_len, suffix) == 0) { + w.resize(w.size() - suffix_len); + return true; + } + return false; + }; + // Order matters: longer suffixes first + if (try_strip("ational", 7)) { w += "ate"; return w; } + if (try_strip("tional", 6)) { w += "tion"; return w; } + if (try_strip("ization", 7)) { w += "ize"; return w; } + if (try_strip("ation", 5)) { w += "ate"; return w; } + if (try_strip("alism", 5)) { w += "al"; return w; } + if (try_strip("ness", 4)) return w; + if (try_strip("ment", 4)) return w; + if (try_strip("ful", 3)) return w; + if (try_strip("ing", 3)) return w; + if (try_strip("ion", 3)) return w; + if (try_strip("ies", 3)) { w += 'y'; return w; } + if (try_strip("ied", 3)) { w += 'y'; return w; } + if (try_strip("ess", 3)) return w; + if (try_strip("ers", 3)) return w; + if (try_strip("er", 2)) return w; + if (try_strip("ed", 2)) return w; + if (try_strip("es", 2)) return w; + // Final -s (not -ss) + if (w.size() > min_stem + 1 && w.back() == 's' && *(w.end()-2) != 's') { + w.pop_back(); + } + return w; +} + +void ML_StemToken::Execute(DataChunk &args, ExpressionState &state, Vector &result) { + const idx_t count = args.size(); + + Vector &in = args.data[0]; + UnifiedVectorFormat in_uvf; + in.ToUnifiedFormat(count, in_uvf); + const string_t *in_data = UnifiedVectorFormat::GetData(in_uvf); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto &res_valid = FlatVector::Validity(result); + + for (idx_t i = 0; i < count; ++i) { + const idx_t sel = in_uvf.sel->get_index(i); + if (!in_uvf.validity.RowIsValid(sel)) { + res_valid.Set(i, false); + FlatVector::GetData(result)[i] = string_t(); + continue; + } + std::string stemmed = simple_stem(in_data[sel].GetString()); + FlatVector::GetData(result)[i] = StringVector::AddString(result, stemmed); + } +} + +void ML_StemToken::Register(ExtensionLoader &loader) { + ScalarFunction fn("stem_token", {LogicalType::VARCHAR}, LogicalType::VARCHAR, Execute); + fn.stability = FunctionStability::VOLATILE; + loader.RegisterFunction(fn); +} + +} // namespace duckdb diff --git a/src/optimization/CMakeLists.txt b/src/optimization/CMakeLists.txt index adaf6d7..f2fb32a 100644 --- a/src/optimization/CMakeLists.txt +++ b/src/optimization/CMakeLists.txt @@ -4,13 +4,19 @@ list(APPEND EXTENSION_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/replace_binding_rule.cpp ${CMAKE_CURRENT_SOURCE_DIR}/register_optimizers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_optimizer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_rule.cpp ${CMAKE_CURRENT_SOURCE_DIR}/catalog.cpp - + ${CMAKE_CURRENT_SOURCE_DIR}/optimizer_pass.cpp + #All Actions ${CMAKE_CURRENT_SOURCE_DIR}/actions/MatMulDense2SparseRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/MultiLayerUDF2TorchNNRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/DecisionForestUDF2RelationRewriteAction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLDecompositionPushdownRewriteAction.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/actions/MLFactorizationRewriteAction.cpp + # MLGreedyFactorizationOptimizer.cpp is intentionally NOT compiled in: + # the baseline ships without the greedy cost-based pass. It is added at + # runtime via the frontend upload -> plugins/opt_passes/ -> rebuild flow. #Feature Extraction Helpers ${CMAKE_CURRENT_SOURCE_DIR}/feature_extraction/query2vec/query_annotation.cpp diff --git a/src/optimization/actions/DecisionForestUDF2RelationRewriteAction.cpp b/src/optimization/actions/DecisionForestUDF2RelationRewriteAction.cpp index 410ded4..a4d8ec3 100644 --- a/src/optimization/actions/DecisionForestUDF2RelationRewriteAction.cpp +++ b/src/optimization/actions/DecisionForestUDF2RelationRewriteAction.cpp @@ -11,6 +11,8 @@ #include "duckdb/function/aggregate_function.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_operator_expression.hpp" #include "duckdb/planner/expression/bound_case_expression.hpp" @@ -111,6 +113,11 @@ static unique_ptr BuildForestGet(ClientContext &context, const auto get = make_uniq(table_index, std::move(tf), std::move(bind_data), std::move(return_types), std::move(names), std::move(virtual_columns)); + // Explicitly declare both columns so GetColumnBindings() returns {(0,0),(0,1)}. + // Without this, column_ids is empty and only (0,0) is visible to ColumnBindingResolver, + // causing it to fail when forest_path_proj references (0,1) = tree_path. + get->AddColumnId(0); // tree_id + get->AddColumnId(1); // tree_path std::cout << "Built forest_trees GET over path: " << forest_path << "\n"; return std::move(get); } @@ -121,15 +128,13 @@ static AggregateFunction GetAvgFloatAggregate(ClientContext &context) { auto &entry = catalog.GetEntry(context, DEFAULT_SCHEMA, "avg"); - // Pick the overload with FLOAT (or DOUBLE) argument + // DuckDB's avg only has a DOUBLE overload; the FLOAT input is cast before calling it. for (auto &fn : entry.functions.functions) { - if (fn.arguments.size() == 1 && - (fn.arguments[0].id() == LogicalTypeId::FLOAT || - fn.arguments[0].id() == LogicalTypeId::DOUBLE)) { + if (fn.arguments.size() == 1 && fn.arguments[0].id() == LogicalTypeId::DOUBLE) { return fn; } } - throw InternalException("DecisionForest rewrite: AVG(FLOAT/DOUBLE) aggregate not found"); + throw InternalException("DecisionForest rewrite: AVG(DOUBLE) aggregate not found"); } // Core rewrite: @@ -220,8 +225,21 @@ static bool RewriteForestProjectionToCrossProduct(OptimizerExtensionInput &input // Copy the original features expression as a generic bound expression unique_ptr feat_expr_original = forest_func->children[0]->Copy(); - // 5) Build right-hand side GET forest_trees(forest_path) + // 5) Build right-hand side GET forest_trees(forest_path), then project + // out tree_id so only tree_path flows into the CROSS_PRODUCT. + // Without this, DuckDB's column pruner drops tree_id at the physical + // level (since nothing references it), shifting tree_path from index + // left_cols+1 to left_cols+0 and causing an out-of-bounds access. auto right_get = BuildForestGet(context, forest_path); + // forest_trees outputs (tree_id INT=0, tree_path VARCHAR=1); keep only tree_path. + // Use BoundColumnRefExpression so DuckDB's ColumnBindingResolver correctly remaps + // the index after RemoveUnusedColumns prunes tree_id from the LogicalGet. + // BuildForestGet uses table_index=0 for the FOREST_TREES LogicalGet. + vector> forest_proj_exprs; + forest_proj_exprs.push_back( + make_uniq(LogicalType::VARCHAR, ColumnBinding(0, 1))); + auto forest_path_proj = make_uniq(99, std::move(forest_proj_exprs)); + forest_path_proj->children.push_back(std::move(right_get)); // 6) Original child of projection becomes left side of CROSS_PRODUCT if (proj.children.empty()) { @@ -230,19 +248,15 @@ static bool RewriteForestProjectionToCrossProduct(OptimizerExtensionInput &input return false; } auto left_child = std::move(proj.children[0]); + idx_t left_cols = left_child->types.size(); - // CROSS_PRODUCT(left_child, forest_trees) - auto cross = make_uniq(std::move(left_child), std::move(right_get)); + // CROSS_PRODUCT(left_child, forest_path_proj) + // Schema: [ all columns from left_child | tree_path at left_cols ] + auto cross = make_uniq(std::move(left_child), std::move(forest_path_proj)); - // After CROSS_PRODUCT schema: - // [ all columns from left_child | tree_id, tree_path ] - idx_t left_cols = cross->children[0]->types.size(); - idx_t tree_id_col_idx = left_cols + 0; - (void)tree_id_col_idx; // not used right now - idx_t tree_path_col_idx = left_cols + 1; + idx_t tree_path_col_idx = left_cols; // tree_path is the only right-side column std::cout << "CROSS_PRODUCT schema: left_cols = " << left_cols - << ", tree_id_col_idx = " << tree_id_col_idx << ", tree_path_col_idx = " << tree_path_col_idx << "\n"; // 7) Build inner PROJECTION on top of CROSS_PRODUCT @@ -262,8 +276,9 @@ static bool RewriteForestProjectionToCrossProduct(OptimizerExtensionInput &input continue; } auto &orig = *proj.expressions[i]; - inner_proj_exprs.push_back( - make_uniq(orig.return_type, next_col_idx)); + // Copy the original expression directly — it already carries the correct + // BoundColumnRefExpression binding from the binder, which survives column pruning. + inner_proj_exprs.push_back(proj.expressions[i]->Copy()); group_types.push_back(orig.return_type); next_col_idx++; } @@ -276,13 +291,7 @@ static bool RewriteForestProjectionToCrossProduct(OptimizerExtensionInput &input // idx_t left_cols = cross->children[0]->types.size(); // idx_t tree_path_col_idx = left_cols + 1; -// 1) Figure out which column is "features" on the left side. -// In your current setup, the projection child is [id, features], -// so features is column index 1. -// For now we hard-code that, since this rule is specialized to your pattern. -idx_t features_col_idx = 1; - -// 2) Lookup tree_predict(...) function from the catalog +// 1) Lookup tree_predict(...) function from the catalog auto &catalog = Catalog::GetSystemCatalog(context); auto &tree_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, "tree_predict"); @@ -308,19 +317,17 @@ ScalarFunction tree_func = *tree_func_ptr; // 3) Build children for tree_predict(features, tree_path, false) -// features: directly reference the LIST(FLOAT) column from CROSS_PRODUCT -auto feat_ref_for_child = - make_uniq(LogicalType::LIST(LogicalType::FLOAT), - features_col_idx); -auto feat_ref_for_bind = - make_uniq(LogicalType::LIST(LogicalType::FLOAT), - features_col_idx); +// features: reuse the original BoundColumnRefExpression from the forest UDF call — +// it already carries the correct binding for features in the left child subplan. +auto feat_ref_for_child = feat_expr_original->Copy(); +auto feat_ref_for_bind = feat_expr_original->Copy(); -// tree_path: VARCHAR column from CROSS_PRODUCT +// tree_path: forest_path_proj has table_index=99, outputs tree_path at column 0. +// Use BoundColumnRefExpression so ColumnBindingResolver remaps it correctly. auto tree_path_ref_for_child = - make_uniq(LogicalType::VARCHAR, tree_path_col_idx); + make_uniq(LogicalType::VARCHAR, ColumnBinding(99, 0)); auto tree_path_ref_for_bind = - make_uniq(LogicalType::VARCHAR, tree_path_col_idx); + make_uniq(LogicalType::VARCHAR, ColumnBinding(99, 0)); // has_missing: FALSE literal auto has_missing_child = @@ -353,7 +360,7 @@ auto tree_pred_expr = make_uniq( inner_proj_exprs.push_back(std::move(tree_pred_expr)); // Build LogicalProjection with the new API (table_index + select_list) - idx_t inner_table_index = 0; + idx_t inner_table_index = 98; // unique index; must not collide with FOREST_TREES(0) or outer proj auto inner_proj = make_uniq(inner_table_index, std::move(inner_proj_exprs)); inner_proj->children.push_back(std::move(cross)); @@ -361,24 +368,30 @@ auto tree_pred_expr = make_uniq( // 8) Build AGGREGATE on top of inner_proj: // SELECT group_keys..., AVG(pred) // - // LogicalAggregate takes a select_list of expressions, where: - // - [0..group_count-1] are group keys - // - [group_count..] are BoundAggregateExpression(s) - vector> aggregate_select; + // LogicalAggregate: group keys go into aggregate->groups, + // only BoundAggregateExpression(s) go in the select_list/expressions. + // Mixing BoundReferenceExpression into expressions causes a type-resolution crash. - // group keys: references [0..group_types.size()-1] of inner_proj output + // group keys for aggregate->groups — reference inner_proj output by ColumnBinding + vector> group_exprs; for (idx_t i = 0; i < group_types.size(); i++) { - aggregate_select.push_back( - make_uniq(group_types[i], i)); + group_exprs.push_back( + make_uniq(group_types[i], ColumnBinding(inner_table_index, i))); } // prediction column index in inner_proj idx_t pred_idx = group_types.size(); - // children for AVG: just reference the prediction column + // children for AVG: reference prediction column via ColumnBinding. + // DuckDB's AVG only has a DOUBLE overload; cast FLOAT → DOUBLE explicitly + // so the executor doesn't see a type mismatch at runtime. vector> avg_children; - avg_children.push_back( - make_uniq(tree_func.return_type, pred_idx)); + { + auto pred_ref = make_uniq( + tree_func.return_type, ColumnBinding(inner_table_index, pred_idx)); + avg_children.push_back( + BoundCastExpression::AddCastToType(context, std::move(pred_ref), LogicalType::DOUBLE)); + } // lookup AVG(FLOAT/DOUBLE) auto avg_fun = GetAvgFloatAggregate(context); @@ -393,15 +406,18 @@ auto tree_pred_expr = make_uniq( avg_fun, std::move(avg_children), std::move(avg_filter), std::move(avg_bind_info), agg_type); - // append AVG(pred) to select list - aggregate_select.push_back(std::move(avg_agg)); + // select_list contains only aggregate expressions + vector> agg_select; + agg_select.push_back(std::move(avg_agg)); - // group_index = 0, aggregate_index = group_count - idx_t group_index = 0; - idx_t aggregate_index = group_types.size(); + // group_index and aggregate_index are table-level binding indices; + // use distinct values that won't collide with inner_proj (0) or original projection. + idx_t group_index = 100; + idx_t aggregate_index = 101; auto aggregate = make_uniq( - group_index, aggregate_index, std::move(aggregate_select)); + group_index, aggregate_index, std::move(agg_select)); + aggregate->groups = std::move(group_exprs); aggregate->children.push_back(std::move(inner_proj)); // 9) Final projection on top: @@ -409,23 +425,23 @@ auto tree_pred_expr = make_uniq( // - forest_result = (is_classification ? (avg > 0 ? 1 : 0) : avg) vector> final_proj_exprs; - // group keys: references [0..group_types.size()-1] of aggregate output + // group keys from aggregate output — reference by ColumnBinding(group_index, i) for (idx_t i = 0; i < group_types.size(); i++) { final_proj_exprs.push_back( - make_uniq(group_types[i], i)); + make_uniq(group_types[i], ColumnBinding(group_index, i))); } - // avg_pred is at col index = group_types.size() - idx_t avg_idx = group_types.size(); + // avg result is the first (and only) aggregate expression — ColumnBinding(aggregate_index, 0) auto avg_ref = - make_uniq(avg_type, avg_idx); + make_uniq(avg_type, ColumnBinding(aggregate_index, 0)); unique_ptr forest_result_expr; if (is_classification) { // CASE WHEN avg > 0 THEN 1 ELSE 0 - auto zero_val = Value::FLOAT(0.0f); - auto zero_expr = make_uniq(zero_val); + // avg_type is DOUBLE (AVG(FLOAT) returns DOUBLE in DuckDB), so both sides + // of the comparison must be DOUBLE to avoid a bound-expression type mismatch. + auto zero_expr = make_uniq(Value(0).DefaultCastAs(avg_type)); auto cond = make_uniq( ExpressionType::COMPARE_GREATERTHAN, avg_ref->Copy(), diff --git a/src/optimization/actions/MLFactorizationRewriteAction.cpp b/src/optimization/actions/MLFactorizationRewriteAction.cpp new file mode 100644 index 0000000..b3f2e3a --- /dev/null +++ b/src/optimization/actions/MLFactorizationRewriteAction.cpp @@ -0,0 +1,514 @@ +#include "optimization/actions/MLFactorizationRewriteAction.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/assert.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/value.hpp" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" + +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/operator/logical_comparison_join.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" + +#include "ml_functions/matrix_multiplication.hpp" + +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +// --------------------------------------------------------------------------- +// Small plan/expression helpers +// --------------------------------------------------------------------------- + +static idx_t GetLogicalWidth(LogicalOperator &op) { + if (op.types.empty()) { + op.ResolveOperatorTypes(); + } + return op.types.size(); +} + +static bool IsMatMul(const BoundFunctionExpression &f) { + return StringUtil::CIEquals(f.function.name, "mat_mul"); +} + +static bool IsListConcat(const BoundFunctionExpression &f) { + auto name = StringUtil::Lower(f.function.name); + return name == "list_concat" || name == "list_cat" || name == "array_concat"; +} + +// Collect every table_index referenced by BoundColumnRefExpressions in expr. +static void CollectTableIndexes(const unique_ptr &expr, + std::unordered_set &out) { + if (!expr) { + return; + } + ExpressionIterator::EnumerateExpression( + const_cast &>(expr), [&](Expression &sub) { + if (sub.type == ExpressionType::BOUND_COLUMN_REF) { + out.insert(sub.Cast().binding.table_index); + } + }); +} + +// True when every table index in expr_tables appears in side_tables, +// and expr_tables is non-empty (i.e. the expression actually references something). +static bool TablesOnlyInSide(const std::unordered_set &expr_tables, + const vector &side_tables) { + if (expr_tables.empty()) { + return false; + } + for (idx_t t : expr_tables) { + if (std::find(side_tables.begin(), side_tables.end(), t) == side_tables.end()) { + return false; + } + } + return true; +} + +// Return the static element count of a list-producing expression, or -1 if unknown. +// DOUBLE[N] column -> N (ARRAY type carries size in the type) +// cast(DOUBLE[N] -> DOUBLE[]) -> N (DuckDB inserts this cast before list_concat) +// list_value(a, b, c) -> 3 (inline list literal; child count = element count) +// LIST -> -1 (variable length, unknown at plan time) +static int64_t GetStaticListLength(const unique_ptr &expr) { + if (!expr) { + return -1; + } + // Direct ARRAY type (e.g. DOUBLE[3] column ref used without casting). + if (expr->return_type.id() == LogicalTypeId::ARRAY) { + return static_cast(ArrayType::GetSize(expr->return_type)); + } + // DuckDB silently casts DOUBLE[N] → DOUBLE[] before passing to list_concat. + // Look through the cast to recover the original array size. + if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) { + return static_cast(ArrayType::GetSize(cast.child->return_type)); + } + } + // Inline list literal: list_value(col1, col2, ...) — each child is one element. + if (expr->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = expr->Cast(); + auto name = StringUtil::Lower(bfe.function.name); + if (name == "list_value" || name == "array_value") { + return static_cast(bfe.children.size()); + } + } + return -1; +} + +// --------------------------------------------------------------------------- +// Weight file helpers +// --------------------------------------------------------------------------- + +// Write a contiguous slice of a row-major weight matrix (rows [row_start, row_start+row_count)) +// to a CSV file under /tmp. The path is deterministic so the same split is never written twice. +static std::string WriteSplitWeightCSV(const std::vector &weights, idx_t n, + idx_t row_start, idx_t row_count, + const std::string &base_path, + const std::string &suffix) { + // Deterministic filename: hash of base_path + split parameters. + std::size_t h = std::hash{}( + base_path + "|" + std::to_string(row_start) + "|" + std::to_string(row_count)); + std::string out_path = "/tmp/matmul_fact_" + std::to_string(h) + suffix + ".csv"; + + // Idempotent: skip writing if the file already exists. + { + std::ifstream probe(out_path); + if (probe.good()) { + return out_path; + } + } + + std::ofstream f(out_path); + if (!f.is_open()) { + throw InternalException("MLFactorization: cannot write split weight file: %s", + out_path.c_str()); + } + + for (idx_t row = row_start; row < row_start + row_count; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) { + f << ','; + } + f << weights[row * n + col]; + } + f << '\n'; + } + + return out_path; +} + +// --------------------------------------------------------------------------- +// Expression builders +// --------------------------------------------------------------------------- + +// Build a new mat_mul(input, W_path, mode_str) BoundFunctionExpression with +// a pre-populated BindData containing the split weights. Looks up the scalar +// function from the catalog so we get the correct function pointer. +static unique_ptr BuildMatMulExpr(ClientContext &ctx, + unique_ptr input_expr, + const std::string &W_path, + const std::string &mode_str, + const std::vector &W_flat, + idx_t k, idx_t n, + ML_MatrixMultiply::MulMode mode) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = + catalog.GetEntry(ctx, DEFAULT_SCHEMA, "mat_mul"); + + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 3 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1].id() == LogicalTypeId::VARCHAR && + fn.arguments[2].id() == LogicalTypeId::VARCHAR) { + chosen = &fn; + break; + } + } + if (!chosen) { + throw InternalException( + "MLFactorization: mat_mul(LIST, VARCHAR, VARCHAR) overload not found"); + } + + auto bind_info = make_uniq(W_flat, k, n, W_path, true, mode); + + vector> children; + children.push_back(std::move(input_expr)); + children.push_back(make_uniq(Value(W_path))); + children.push_back(make_uniq(Value(mode_str))); + + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), + std::move(bind_info)); +} + +// Build list_add_vv(left_ref, right_ref). +static unique_ptr BuildListAddVVExpr(ClientContext &ctx, + unique_ptr left_ref, + unique_ptr right_ref) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = + catalog.GetEntry(ctx, DEFAULT_SCHEMA, "list_add_vv"); + + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 2 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1] == LogicalType::LIST(LogicalType::DOUBLE)) { + chosen = &fn; + break; + } + } + if (!chosen) { + throw InternalException( + "MLFactorization: list_add_vv(LIST, LIST) overload not found"); + } + + vector> children; + children.push_back(std::move(left_ref)); + children.push_back(std::move(right_ref)); + + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), nullptr); +} + +// --------------------------------------------------------------------------- +// Plan mutation helper +// --------------------------------------------------------------------------- + +// Wrap join.children[side_idx] in a new LogicalProjection that passes through +// all existing columns and appends extra_expr as an additional column. +// Sets new_col_idx to the index of extra_expr in the new child's output. +// Returns false if the child has multiple table indexes (can't safely rewrite). +static bool InsertProjectionOnSide(LogicalComparisonJoin &join, int side_idx, + Expression &extra_expr, idx_t &new_col_idx) { + D_ASSERT(side_idx == 0 || side_idx == 1); + auto &child_ptr = join.children[side_idx]; + if (!child_ptr) { + return false; + } + + auto side_tables = child_ptr->GetTableIndex(); + if (side_tables.size() != 1) { + return false; + } + const idx_t table_index = side_tables[0]; + + const idx_t child_width = GetLogicalWidth(*child_ptr); + if (child_width == 0) { + return false; + } + + vector> select_list; + select_list.reserve(child_width + 1); + + // Pass-through all existing columns by position. + for (idx_t i = 0; i < child_width; ++i) { + select_list.push_back(make_uniq(child_ptr->types[i], i)); + } + + // Append the new expression; record its position in the output. + select_list.push_back(extra_expr.Copy()); + new_col_idx = child_width; + + // Wrap the existing child in a projection that inherits its table_index, + // so bindings from above that reference this side remain valid. + auto new_proj = make_uniq(table_index, std::move(select_list)); + new_proj->children.push_back(std::move(child_ptr)); + child_ptr = std::move(new_proj); + + return true; +} + +// --------------------------------------------------------------------------- +// Pattern detection (used by both check and apply) +// --------------------------------------------------------------------------- + +// Recursively search expr for a sub-expression matching: +// mat_mul( list_concat(L, R), W_file, mode ) +// where L references only left_tables, R only right_tables, and k_left > 0. +// Returns a pointer to the unique_ptr holding the matching mat_mul +// so the caller can replace it in-place, or nullptr if not found. +static unique_ptr *FindFactorizableMatMul(unique_ptr &expr, + const vector &left_tables, + const vector &right_tables) { + if (!expr || expr->expression_class != ExpressionClass::BOUND_FUNCTION) { + return nullptr; + } + auto &f = expr->Cast(); + + // Check whether THIS node is the mat_mul(list_concat(L, R), W) pattern. + if (IsMatMul(f) && !f.children.empty()) { + auto *bd = dynamic_cast(f.bind_info.get()); + if (bd && bd->has_bound_weights) { + auto &first_child = f.children[0]; + if (first_child && + first_child->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &lc = first_child->Cast(); + if (IsListConcat(lc) && lc.children.size() == 2) { + std::unordered_set left_refs, right_refs; + CollectTableIndexes(lc.children[0], left_refs); + CollectTableIndexes(lc.children[1], right_refs); + int64_t k_left = GetStaticListLength(lc.children[0]); + if (k_left > 0 && + TablesOnlyInSide(left_refs, left_tables) && + TablesOnlyInSide(right_refs, right_tables)) { + return &expr; // caller owns this unique_ptr and may replace it + } + } + } + } + } + + // Recurse into all children of this bound function. + for (auto &child : f.children) { + auto *result = FindFactorizableMatMul(child, left_tables, right_tables); + if (result) { + return result; + } + } + return nullptr; +} + +// Thin wrapper used by check() — const-safe via const_cast (read-only traversal). +static bool MatchesFactorizationPattern(const unique_ptr &expr, + const vector &left_tables, + const vector &right_tables) { + return FindFactorizableMatMul( + const_cast &>(expr), left_tables, right_tables) != + nullptr; +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// check +// --------------------------------------------------------------------------- + +bool MLFactorizationRewriteAction::check(OptimizerExtensionInput &input, + unique_ptr &plan) { + if (!plan) { + return false; + } + + bool found = false; + + if (plan->type == LogicalOperatorType::LOGICAL_PROJECTION && + plan->children.size() == 1 && + plan->children[0] && + plan->children[0]->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + + auto &join = plan->children[0]->Cast(); + auto left_tables = join.children[0]->GetTableIndex(); + auto right_tables = join.children[1]->GetTableIndex(); + + auto &proj = plan->Cast(); + for (auto &expr : proj.expressions) { + if (MatchesFactorizationPattern(expr, left_tables, right_tables)) { + found = true; + break; + } + } + } + + for (auto &child : plan->children) { + found |= check(input, child); + } + + return found; +} + +// --------------------------------------------------------------------------- +// apply +// --------------------------------------------------------------------------- + +bool MLFactorizationRewriteAction::apply(OptimizerExtensionInput &input, + unique_ptr &plan) { + if (!plan) { + return false; + } + + bool changed = false; + + // Bottom-up: rewrite children first. + for (auto &child : plan->children) { + changed |= apply(input, child); + } + + // Only act on a projection whose direct child is a comparison join. + if (plan->type != LogicalOperatorType::LOGICAL_PROJECTION) { + return changed; + } + auto &proj = plan->Cast(); + if (proj.children.size() != 1 || !proj.children[0] || + proj.children[0]->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + return changed; + } + + auto &join = proj.children[0]->Cast(); + auto left_tables = join.children[0]->GetTableIndex(); + auto right_tables = join.children[1]->GetTableIndex(); + + // Both join sides must each have exactly one table index so we can safely + // insert a projection on each side. + if (left_tables.size() != 1 || right_tables.size() != 1) { + return changed; + } + + for (auto &expr_ptr : proj.expressions) { + // Recursively locate the mat_mul(list_concat(L,R), W) sub-expression. + // target_ptr points to the unique_ptr that holds it, so we + // can replace it in-place without disturbing the rest of the DNN chain. + auto *target_ptr = FindFactorizableMatMul(expr_ptr, left_tables, right_tables); + if (!target_ptr) { + continue; + } + + // ---- Unpack the matched mat_mul sub-expression ---- + auto &f = (*target_ptr)->Cast(); + auto &lc = f.children[0]->Cast(); // list_concat + auto *bd = dynamic_cast(f.bind_info.get()); + + const int64_t k_left_signed = GetStaticListLength(lc.children[0]); + D_ASSERT(k_left_signed > 0); + const idx_t k_left = static_cast(k_left_signed); + const idx_t k_right = bd->k - k_left; + const idx_t n = bd->n; + + if (k_right == 0 || k_left + k_right != bd->k) { + continue; // sanity: split must be non-trivial + } + + // ---- Split the weight matrix ---- + // weights is flat row-major [k * n]: row i starts at weights[i*n]. + // W_top = rows [0, k_left) + // W_bottom = rows [k_left, k) + std::vector W_top(bd->weights.begin(), + bd->weights.begin() + (ptrdiff_t)(k_left * n)); + std::vector W_bottom(bd->weights.begin() + (ptrdiff_t)(k_left * n), + bd->weights.end()); + + std::string W_top_path = WriteSplitWeightCSV(bd->weights, n, 0, k_left, bd->weights_file, "_left"); + std::string W_bottom_path = WriteSplitWeightCSV(bd->weights, n, k_left, k_right, bd->weights_file, "_right"); + + // Mode string for the reconstructed mat_mul children. + std::string mode_str; + switch (bd->mode) { + case ML_MatrixMultiply::MulMode::Dense: mode_str = "dense"; break; + case ML_MatrixMultiply::MulMode::InputSparse: mode_str = "sparse"; break; + default: mode_str = "dense"; break; + } + + // ---- Build partial mat_mul expressions ---- + // L and R are copies of list_concat's children; they carry the correct + // BoundColumnRefExpression bindings (T_left / T_right) and need no rewrite. + auto mat_mul_left = BuildMatMulExpr(input.context, + lc.children[0]->Copy(), + W_top_path, mode_str, + W_top, k_left, n, bd->mode); + + auto mat_mul_right = BuildMatMulExpr(input.context, + lc.children[1]->Copy(), + W_bottom_path, mode_str, + W_bottom, k_right, n, bd->mode); + + // ---- Record widths BEFORE any modification ---- + const idx_t L_width = GetLogicalWidth(*join.children[0]); + const idx_t R_width = GetLogicalWidth(*join.children[1]); + + // ---- Insert projections on each join side ---- + idx_t new_left_col = 0; + idx_t new_right_col = 0; + + if (!InsertProjectionOnSide(join, 0, *mat_mul_left, new_left_col)) { + continue; + } + if (!InsertProjectionOnSide(join, 1, *mat_mul_right, new_right_col)) { + // Left side already modified; keep going — the result is suboptimal + // but not incorrect (the new projection is benign). + continue; + } + + // ---- Compute global column positions in the JOIN output ---- + // After insertions: + // Left side occupies indices [0, L_width] (L_width+1 cols, new at L_width) + // Right side occupies indices [L_width+1, L_width+1+R_width] (R_width+1 cols, new at end) + const idx_t left_global_idx = L_width; + const idx_t right_global_idx = L_width + 1 + R_width; + + // ---- Replace the mat_mul sub-expression with list_add_vv(ref_left, ref_right) ---- + // *target_ptr is the unique_ptr holding the mat_mul node; replacing it + // patches it in-place so any enclosing relu/mat_add/softmax chain is + // preserved untouched. + auto left_ref = make_uniq( + LogicalType::LIST(LogicalType::DOUBLE), left_global_idx); + auto right_ref = make_uniq( + LogicalType::LIST(LogicalType::DOUBLE), right_global_idx); + + *target_ptr = BuildListAddVVExpr(input.context, + std::move(left_ref), + std::move(right_ref)); + + changed = true; + break; // one factorization per projection per pass + } + + return changed; +} + +} // namespace duckdb diff --git a/src/optimization/actions/MLGreedyFactorizationOptimizer.cpp b/src/optimization/actions/MLGreedyFactorizationOptimizer.cpp new file mode 100644 index 0000000..9b79ab1 --- /dev/null +++ b/src/optimization/actions/MLGreedyFactorizationOptimizer.cpp @@ -0,0 +1,902 @@ +#include "optimization/actions/MLGreedyFactorizationOptimizer.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/assert.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/value.hpp" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" + +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/operator/logical_comparison_join.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" + +#include "ml_functions/matrix_multiplication.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +// --------------------------------------------------------------------------- +// JoinTreeNode +// --------------------------------------------------------------------------- + +struct JoinTreeNode { + LogicalOperator *plan_node; // pointer into the live DuckDB plan tree + int side; // 0=left child of parent, 1=right child, -1=root + JoinTreeNode *parent; // nullptr for root + std::vector children; // up to 2 for a binary join + + // Populated during tree walk + idx_t num_rows; // estimated cardinality + int64_t num_features; // total input feature dims still un-reduced at this node + int64_t new_features; // model output width n (first-layer neuron count) + double cardinality_ratio; // num_rows / parent->num_rows (selectivity from this side) + int depth; // absolute depth (root = 1) + int max_depth; // deepest node in the whole tree (same for all nodes after BFS) + double cost; // current cost estimate + + // Assignment phase + int label; // 0=unassigned, 1=push mat_mul here, 2=combine children (list_add_vv) + bool processed; + bool subtree_processed; + + // Weight-split info set during apply phase + idx_t row_start; // start row in the original W for this node's feature block + idx_t row_count; // number of rows (= sum of k_i for tables on this side) +}; + +// --------------------------------------------------------------------------- +// Expression helpers (shared with MLFactorizationRewriteAction pattern) +// --------------------------------------------------------------------------- + +static bool IsMatMul(const BoundFunctionExpression &f) { + return StringUtil::CIEquals(f.function.name, "mat_mul"); +} + +static bool IsListConcat(const BoundFunctionExpression &f) { + auto name = StringUtil::Lower(f.function.name); + return name == "list_concat" || name == "list_cat" || name == "array_concat"; +} + +static bool AreConsecutivePartIndices(const std::vector &indices) { + if (indices.empty()) return false; + for (idx_t i = 1; i < indices.size(); ++i) { + if (indices[i] != indices[i - 1] + 1) return false; + } + return true; +} + +static int64_t GetStaticListLength(const unique_ptr &expr) { + if (!expr) return -1; + if (expr->return_type.id() == LogicalTypeId::ARRAY) + return static_cast(ArrayType::GetSize(expr->return_type)); + if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) + return static_cast(ArrayType::GetSize(cast.child->return_type)); + } + if (expr->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = expr->Cast(); + auto name = StringUtil::Lower(bfe.function.name); + if (name == "list_value" || name == "array_value") + return static_cast(bfe.children.size()); + } + return -1; +} + +// Collect every table_index referenced by BoundColumnRefExpressions in expr. +static void CollectTableIndexes(Expression &expr, + std::unordered_set &out) { + auto owned_copy = expr.Copy(); + ExpressionIterator::EnumerateExpression( + owned_copy, [&](Expression &sub) { + if (sub.type == ExpressionType::BOUND_COLUMN_REF) + out.insert(sub.Cast().binding.table_index); + }); +} + +// Recursively flatten list_concat(list_concat(A,B),C) -> [{A,k_A},{B,k_B},{C,k_C}] +struct FlatPart { + Expression *expr; + int64_t length; +}; + +static void FlattenListConcatImpl(Expression *expr, std::vector &out) { + if (!expr) return; + + // Unwrap implicit cast DOUBLE[N] -> DOUBLE[] inserted by DuckDB + Expression *inner = expr; + if (inner->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = inner->Cast(); + if (cast.child) inner = cast.child.get(); + } + + if (inner->expression_class == ExpressionClass::BOUND_FUNCTION) { + auto &bfe = inner->Cast(); + if (IsListConcat(bfe) && bfe.children.size() == 2) { + FlattenListConcatImpl(bfe.children[0].get(), out); + FlattenListConcatImpl(bfe.children[1].get(), out); + return; + } + } + + // Leaf: compute static length and record + // Use original expr (not inner) so GetStaticListLength can unwrap the cast itself + unique_ptr dummy; // we only need length from it + int64_t len = -1; + if (expr->return_type.id() == LogicalTypeId::ARRAY) + len = static_cast(ArrayType::GetSize(expr->return_type)); + else if (expr->expression_class == ExpressionClass::BOUND_CAST) { + auto &cast = expr->Cast(); + if (cast.child && cast.child->return_type.id() == LogicalTypeId::ARRAY) + len = static_cast(ArrayType::GetSize(cast.child->return_type)); + } + out.push_back({expr, len}); +} + +static std::vector FlattenListConcat(Expression &expr) { + std::vector parts; + FlattenListConcatImpl(&expr, parts); + return parts; +} + +static unique_ptr BuildConcatFromParts(const BoundFunctionExpression &concat_proto, + const std::vector &parts, + const std::vector &indices) { + D_ASSERT(!indices.empty()); + auto expr = parts[indices[0]].expr->Copy(); + for (idx_t i = 1; i < indices.size(); ++i) { + vector> children; + children.push_back(std::move(expr)); + children.push_back(parts[indices[i]].expr->Copy()); + expr = make_uniq( + LogicalType::LIST(LogicalType::DOUBLE), + concat_proto.function, + std::move(children), + nullptr); + } + return expr; +} + +// --------------------------------------------------------------------------- +// Weight file helper +// --------------------------------------------------------------------------- + +static std::string WriteSplitWeightCSV(const std::vector &weights, idx_t n, + idx_t row_start, idx_t row_count, + const std::string &base_path, + const std::string &suffix) { + std::size_t h = std::hash{}( + base_path + "|" + std::to_string(row_start) + "|" + std::to_string(row_count)); + std::string out_path = "/tmp/matmul_greedy_" + std::to_string(h) + suffix + ".csv"; + { + std::ifstream probe(out_path); + if (probe.good()) return out_path; + } + std::ofstream f(out_path); + if (!f.is_open()) + throw InternalException("MLGreedyFactorization: cannot write weight file: %s", out_path.c_str()); + for (idx_t row = row_start; row < row_start + row_count; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) f << ','; + f << weights[row * n + col]; + } + f << '\n'; + } + return out_path; +} + +static std::string WriteCustomWeightCSV(const std::vector &weights, idx_t k, idx_t n, + const std::string &base_path, + const std::string &suffix, + const std::string &cache_key) { + std::size_t h = std::hash{}( + base_path + "|" + cache_key + "|" + std::to_string(k) + "|" + std::to_string(n)); + std::string out_path = "/tmp/matmul_greedy_" + std::to_string(h) + suffix + ".csv"; + { + std::ifstream probe(out_path); + if (probe.good()) return out_path; + } + + std::ofstream f(out_path); + if (!f.is_open()) + throw InternalException("MLGreedyFactorization: cannot write weight file: %s", out_path.c_str()); + for (idx_t row = 0; row < k; ++row) { + for (idx_t col = 0; col < n; ++col) { + if (col > 0) f << ','; + f << weights[row * n + col]; + } + f << '\n'; + } + return out_path; +} + +// --------------------------------------------------------------------------- +// Expression builders +// --------------------------------------------------------------------------- + +static unique_ptr BuildMatMulExpr(ClientContext &ctx, + unique_ptr input_expr, + const std::string &W_path, + const std::string &mode_str, + const std::vector &W_flat, + idx_t k, idx_t n, + ML_MatrixMultiply::MulMode mode) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = catalog.GetEntry(ctx, DEFAULT_SCHEMA, "mat_mul"); + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 3 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1].id() == LogicalTypeId::VARCHAR && + fn.arguments[2].id() == LogicalTypeId::VARCHAR) { + chosen = &fn; + break; + } + } + if (!chosen) + throw InternalException("MLGreedyFactorization: mat_mul overload not found"); + + auto bind_info = make_uniq(W_flat, k, n, W_path, true, mode); + vector> children; + children.push_back(std::move(input_expr)); + children.push_back(make_uniq(Value(W_path))); + children.push_back(make_uniq(Value(mode_str))); + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), std::move(bind_info)); +} + +static unique_ptr BuildListAddVVExpr(ClientContext &ctx, + unique_ptr left, + unique_ptr right) { + auto &catalog = Catalog::GetSystemCatalog(ctx); + auto &entry = catalog.GetEntry(ctx, DEFAULT_SCHEMA, "list_add_vv"); + const ScalarFunction *chosen = nullptr; + for (auto &fn : entry.functions.functions) { + if (fn.arguments.size() == 2 && + fn.arguments[0] == LogicalType::LIST(LogicalType::DOUBLE) && + fn.arguments[1] == LogicalType::LIST(LogicalType::DOUBLE)) { + chosen = &fn; + break; + } + } + if (!chosen) + throw InternalException("MLGreedyFactorization: list_add_vv overload not found"); + vector> children; + children.push_back(std::move(left)); + children.push_back(std::move(right)); + return make_uniq(LogicalType::LIST(LogicalType::DOUBLE), + *chosen, std::move(children), nullptr); +} + +// --------------------------------------------------------------------------- +// InsertProjectionOnSide +// --------------------------------------------------------------------------- + +static idx_t GetLogicalWidth(LogicalOperator &op) { + if (op.types.empty()) op.ResolveOperatorTypes(); + return op.types.size(); +} + +static bool InsertProjectionOnSide(LogicalComparisonJoin &join, int side_idx, + Expression &extra_expr, idx_t &new_col_idx, + idx_t &new_table_idx) { + D_ASSERT(side_idx == 0 || side_idx == 1); + auto &child_ptr = join.children[side_idx]; + if (!child_ptr) return false; + + auto side_tables = child_ptr->GetTableIndex(); + if (side_tables.size() != 1) return false; + + const idx_t table_index = side_tables[0]; + const idx_t child_width = GetLogicalWidth(*child_ptr); + if (child_width == 0) return false; + + vector> select_list; + select_list.reserve(child_width + 1); + for (idx_t i = 0; i < child_width; ++i) + select_list.push_back(make_uniq(child_ptr->types[i], i)); + select_list.push_back(extra_expr.Copy()); + new_col_idx = child_width; + + auto new_proj = make_uniq(table_index, std::move(select_list)); + new_proj->children.push_back(std::move(child_ptr)); + child_ptr = std::move(new_proj); + + // If the join already projects a subset of child columns, explicitly expose + // the appended column so parent expressions can bind to it. + if (side_idx == 0) { + if (!join.left_projection_map.empty()) { + join.left_projection_map.push_back(new_col_idx); + } + } else { + if (!join.right_projection_map.empty()) { + join.right_projection_map.push_back(new_col_idx); + } + } + + new_table_idx = table_index; + return true; +} + +// --------------------------------------------------------------------------- +// Join tree construction +// --------------------------------------------------------------------------- + +static idx_t EstimateCardinality(LogicalOperator &op) { + if (op.estimated_cardinality > 0) + return op.estimated_cardinality; + // Fallback: product of children cardinalities (pessimistic cross-join) + if (op.children.size() == 2) { + idx_t l = EstimateCardinality(*op.children[0]); + idx_t r = EstimateCardinality(*op.children[1]); + return std::max(1, l * r); + } + return 1; +} + +// Walk the DuckDB plan tree and collect all LogicalComparisonJoin nodes, +// building a parallel JoinTreeNode tree. Returns the root JoinTreeNode. +// all_nodes is populated in post-order (children before parents). +static JoinTreeNode *BuildJoinTree(LogicalOperator *op, + JoinTreeNode *parent, + int side, + int64_t new_features, + std::vector> &storage) { + if (!op) return nullptr; + + // Recurse into children first (post-order) + if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + auto node = make_uniq(); + node->plan_node = op; + node->side = side; + node->parent = parent; + node->new_features = new_features; + node->label = 0; + node->processed = false; + node->subtree_processed = false; + node->row_start = 0; + node->row_count = 0; + node->depth = 0; // filled by CalculateDepths + node->max_depth = 0; + node->cost = 0.0; + + idx_t card = EstimateCardinality(*op); + node->num_rows = card; + + if (parent) { + node->cardinality_ratio = (parent->num_rows > 0) + ? static_cast(card) / static_cast(parent->num_rows) + : 1.0; + } else { + node->cardinality_ratio = 1.0; + } + + JoinTreeNode *raw = node.get(); + + // Build children recursively + if (op->children.size() >= 1) { + JoinTreeNode *left_child = BuildJoinTree(op->children[0].get(), raw, 0, new_features, storage); + if (left_child) raw->children.push_back(left_child); + } + if (op->children.size() >= 2) { + JoinTreeNode *right_child = BuildJoinTree(op->children[1].get(), raw, 1, new_features, storage); + if (right_child) raw->children.push_back(right_child); + } + + storage.push_back(std::move(node)); + return raw; + } + + // Not a join — recurse into children looking for joins below + for (idx_t i = 0; i < op->children.size(); ++i) { + int child_side = (op->children.size() == 2) ? (int)i : side; + JoinTreeNode *found = BuildJoinTree(op->children[i].get(), parent, child_side, new_features, storage); + if (found) return found; // return first found (handles single-path plans) + } + return nullptr; +} + +// --------------------------------------------------------------------------- +// Depth computation +// --------------------------------------------------------------------------- + +static int CalculateDepths(JoinTreeNode *root, + std::vector> &all_nodes) { + int max_depth = 0; + std::queue> q; + q.push({root, 1}); + while (!q.empty()) { + auto [node, d] = q.front(); q.pop(); + node->depth = d; + if (d > max_depth) max_depth = d; + for (auto *child : node->children) q.push({child, d + 1}); + } + for (auto &n : all_nodes) n->max_depth = max_depth; + return max_depth; +} + +// --------------------------------------------------------------------------- +// Feature-part attribution: which flat parts belong to which join-tree side +// --------------------------------------------------------------------------- + +// Returns true if the given LogicalOperator subtree contains table_index t. +static bool SubtreeContainsTable(LogicalOperator *op, idx_t table_index) { + if (!op) return false; + auto tables = op->GetTableIndex(); + for (idx_t t : tables) + if (t == table_index) return true; + for (auto &child : op->children) + if (SubtreeContainsTable(child.get(), table_index)) return true; + return false; +} + +// For a given JoinTreeNode, return which side (0 or 1) the table lives on. +// Returns -1 if not found in either child subtree. +static int TableSideOfNode(JoinTreeNode *node, idx_t table_index) { + if (!node || node->plan_node->children.size() < 2) return -1; + if (SubtreeContainsTable(node->plan_node->children[0].get(), table_index)) return 0; + if (SubtreeContainsTable(node->plan_node->children[1].get(), table_index)) return 1; + return -1; +} + +// --------------------------------------------------------------------------- +// Cost model +// --------------------------------------------------------------------------- + +static double ComputeCost(const JoinTreeNode &v) { + if (v.num_features == 0 || v.depth == 0 || v.max_depth == 0) + return 1.0; // safe fallback above threshold + + double var2 = static_cast(v.num_features); + double var4 = static_cast(v.new_features); + double var5 = static_cast(v.max_depth); + double var6 = var4 / var2; + double var7 = var4 / (var2 * v.depth); + double var8 = static_cast(v.num_rows) * var4 / var2; + double var9 = static_cast(v.num_rows) * var4 / (var2 * v.depth); + double var10 = var2 - var4; + double var11 = static_cast(v.depth) / static_cast(v.max_depth); + double var12 = v.cardinality_ratio; + double d4 = std::pow(static_cast(v.depth), 4.0); + + return -4.62412178e-04 * var2 + - 6.99311355e-04 * var4 + - 4.17964429e-03 * var5 + + 1.57416551e-03 * var6 + - 3.69161585e-04 * var7 + - 2.94571978e-05 * var8 + + 6.07897302e-05 * var9 + + 2.36899161e-04 * var10 + - 4.63010544e-02 * var12 + + 2.12526468e-03 * d4 + + 1.40041493e-04 * std::pow(var11, 4.0) + - 3.86043424e-02 * var11 * var12; +} + +// --------------------------------------------------------------------------- +// Greedy algorithm helpers +// --------------------------------------------------------------------------- + +static void ProcessSubtree(JoinTreeNode *node) { + if (node->subtree_processed) return; + std::vector stack = {node}; + while (!stack.empty()) { + JoinTreeNode *cur = stack.back(); stack.pop_back(); + cur->processed = true; + cur->subtree_processed = true; + for (auto *child : cur->children) + if (!child->subtree_processed) stack.push_back(child); + } + if (node->parent) node->parent->subtree_processed = true; +} + +static void UpdateParentFeatures(JoinTreeNode *parent) { + int64_t total = 0; + for (auto *child : parent->children) + total += (child->label != 0) ? child->new_features : child->num_features; + parent->num_features = total; +} + +// --------------------------------------------------------------------------- +// Greedy assignment +// --------------------------------------------------------------------------- + +static void PerformGreedy(std::vector> &all_nodes) { + constexpr double COST_THRESHOLD = 0.5; + + // Prefer deeper joins first. This avoids selecting a parent early and + // marking its entire subtree processed before child joins can be labeled. + using Entry = std::tuple; + std::priority_queue, std::greater> heap; + + for (auto &n : all_nodes) { + n->cost = ComputeCost(*n); + if (n->cost < COST_THRESHOLD) { + int depth_priority = n->max_depth - n->depth; // smaller => deeper + heap.push({depth_priority, n->cost, n.get()}); + } + } + + while (!heap.empty()) { + auto [depth_priority, current_cost, v] = heap.top(); heap.pop(); + (void)depth_priority; + + if (v->processed) continue; + if (current_cost != v->cost) continue; // stale entry + + if (v->children.size() >= 2 && + v->children[0]->label != 0 && v->children[1]->label != 0) { + v->label = 2; + } else if (current_cost < COST_THRESHOLD) { + v->label = 1; + } else { + break; + } + + ProcessSubtree(v); + + if (v->parent) { + double old_cost = v->parent->cost; + UpdateParentFeatures(v->parent); + double new_cost = ComputeCost(*v->parent); + if (new_cost != old_cost) { + v->parent->cost = new_cost; + if (new_cost < COST_THRESHOLD) { + int parent_depth_priority = v->parent->max_depth - v->parent->depth; + heap.push({parent_depth_priority, new_cost, v->parent}); + } + } + } + } + + // Final pass: promote nodes whose both children are labeled to label=2 + for (auto &n : all_nodes) { + if (n->children.size() >= 2 && + n->children[0]->label != 0 && n->children[1]->label != 0) + n->label = 2; + } +} + +// --------------------------------------------------------------------------- +// check +// --------------------------------------------------------------------------- + +// Returns true if the plan contains a projection over a multi-join with +// mat_mul(list_concat(...), W) where list_concat has 2+ parts. +static bool HasGreedyPattern(LogicalOperator *op) { + if (!op) return false; + + if (op->type == LogicalOperatorType::LOGICAL_PROJECTION) { + // Check if any child is a comparison join + bool has_join_child = false; + for (auto &child : op->children) + if (child && child->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) + has_join_child = true; + + if (has_join_child) { + auto &proj = op->Cast(); + for (auto &expr : proj.expressions) { + if (!expr || expr->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &f = expr->Cast(); + if (!IsMatMul(f) || f.children.empty()) continue; + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) continue; + // Check that the argument is a list_concat with 2+ parts + auto &arg = f.children[0]; + if (!arg || arg->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &lc = arg->Cast(); + if (IsListConcat(lc)) { + auto parts = FlattenListConcat(*arg); + if (parts.size() >= 2) return true; + } + } + } + } + + for (auto &child : op->children) + if (HasGreedyPattern(child.get())) return true; + + return false; +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// Public interface +// --------------------------------------------------------------------------- + +bool MLGreedyFactorizationOptimizer::check(OptimizerExtensionInput &input, + unique_ptr &plan) { + return plan && HasGreedyPattern(plan.get()); +} + +bool MLGreedyFactorizationOptimizer::apply(OptimizerExtensionInput &input, + unique_ptr &plan) { + if (!plan) return false; + + // Bottom-up: rewrite children first + bool changed = false; + for (auto &child : plan->children) + changed |= apply(input, child); + + if (plan->type != LogicalOperatorType::LOGICAL_PROJECTION) return changed; + auto &proj = plan->Cast(); + if (proj.children.empty() || !proj.children[0] || + proj.children[0]->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) + return changed; + + // Find the mat_mul(list_concat(...), W) expression in the projection + BoundFunctionExpression *mat_mul_expr = nullptr; + unique_ptr *mat_mul_slot = nullptr; + for (auto &expr_ptr : proj.expressions) { + if (!expr_ptr || expr_ptr->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &f = expr_ptr->Cast(); + if (!IsMatMul(f) || f.children.empty()) continue; + auto *bd = dynamic_cast(f.bind_info.get()); + if (!bd || !bd->has_bound_weights) continue; + auto &arg = f.children[0]; + if (!arg || arg->expression_class != ExpressionClass::BOUND_FUNCTION) continue; + auto &lc = arg->Cast(); + if (!IsListConcat(lc)) continue; + auto parts = FlattenListConcat(*arg); + if (parts.size() < 2) continue; + mat_mul_expr = &f; + mat_mul_slot = &expr_ptr; + break; + } + if (!mat_mul_expr) return changed; + + auto *bd = dynamic_cast(mat_mul_expr->bind_info.get()); + const idx_t n = bd->n; + const std::string &W_file = bd->weights_file; + std::string mode_str; + switch (bd->mode) { + case ML_MatrixMultiply::MulMode::Dense: mode_str = "dense"; break; + case ML_MatrixMultiply::MulMode::InputSparse: mode_str = "sparse"; break; + default: mode_str = "dense"; break; + } + + // Flatten list_concat into ordered parts + auto parts = FlattenListConcat(*mat_mul_expr->children[0]); + if (parts.size() < 2) return changed; + const auto &concat_proto = mat_mul_expr->children[0]->Cast(); + + // Validate all part lengths are known + for (auto &p : parts) + if (p.length <= 0) return changed; + + std::vector part_row_prefix(parts.size() + 1, 0); + for (idx_t i = 0; i < parts.size(); ++i) { + part_row_prefix[i + 1] = part_row_prefix[i] + static_cast(parts[i].length); + } + + // --------------------------------------------------------------------------- + // Build join tree + // --------------------------------------------------------------------------- + std::vector> node_storage; + JoinTreeNode *root = BuildJoinTree(proj.children[0].get(), nullptr, -1, + static_cast(n), node_storage); + if (!root || node_storage.empty()) return changed; + + // Set num_features on each node (total k = sum of all part lengths) + int64_t total_k = 0; + for (auto &p : parts) total_k += p.length; + for (auto &node : node_storage) node->num_features = total_k; + + // Compute depths + CalculateDepths(root, node_storage); + + // --------------------------------------------------------------------------- + // Run greedy + // --------------------------------------------------------------------------- + PerformGreedy(node_storage); + + // Check if anything was labeled — if not, nothing to do + bool any_labeled = false; + for (auto &node : node_storage) + if (node->label != 0) { any_labeled = true; break; } + if (!any_labeled) return changed; + + // Conservative safety guard: if any join already carries a projection map + // (i.e., columns have been pruned/reordered upstream), skip this rewrite. + // The greedy rewrite currently appends columns on join children and assumes + // stable pass-through bindings. + for (auto &node_ptr : node_storage) { + auto &join = node_ptr->plan_node->Cast(); + if (join.HasProjectionMap()) { + return changed; + } + } + + // --------------------------------------------------------------------------- + // Apply push-downs + // Iterate nodes in storage order (post-order: children before parents). + // For each label=1 node, figure out which flat parts belong to each side, + // build a mat_mul for that side, and insert a projection on the join child. + // + // We track, for each node's plan_node (join), the column indices of inserted + // partial results so the parent can reference them. + // --------------------------------------------------------------------------- + + // Map from plan_node pointer -> {left_col_idx, right_col_idx} of inserted projections + // -1 means not yet inserted on that side. + struct InsertRecord { + idx_t left_col = idx_t(-1); + idx_t right_col = idx_t(-1); + }; + std::unordered_map insert_map; + + for (auto &node_ptr : node_storage) { + JoinTreeNode *node = node_ptr.get(); + if (node->label != 1) continue; + + auto &join = node->plan_node->Cast(); + + // Determine which flat parts land on left vs right side of this join + std::vector left_part_indices, right_part_indices; + for (int i = 0; i < (int)parts.size(); ++i) { + std::unordered_set refs; + CollectTableIndexes(*parts[i].expr, refs); + + for (idx_t t : refs) { + int s = TableSideOfNode(node, t); + if (s == 0) { left_part_indices.push_back(i); break; } + if (s == 1) { right_part_indices.push_back(i); break; } + } + } + + // Helper lambda: build mat_mul for a set of consecutive part indices + auto build_side_mat_mul = [&](const std::vector &indices, int side_id) + -> unique_ptr { + if (indices.empty()) return nullptr; + + idx_t row_count = 0; + for (auto idx : indices) row_count += static_cast(parts[idx].length); + + std::vector W_slice; + W_slice.reserve(row_count * n); + + const idx_t row_start = part_row_prefix[indices.front()]; + for (auto part_idx : indices) { + const idx_t start = part_row_prefix[part_idx]; + const idx_t end = part_row_prefix[part_idx + 1]; + for (idx_t row = start; row < end; ++row) { + auto row_begin = bd->weights.begin() + static_cast(row * n); + W_slice.insert(W_slice.end(), row_begin, row_begin + static_cast(n)); + } + } + + std::string suffix = (side_id == 0) ? "_left" : "_right"; + std::string W_path; + if (AreConsecutivePartIndices(indices)) { + W_path = WriteSplitWeightCSV(bd->weights, n, row_start, row_count, W_file, suffix); + } else { + std::ostringstream key; + key << "parts"; + for (auto idx : indices) key << "_" << idx; + W_path = WriteCustomWeightCSV(W_slice, row_count, n, W_file, suffix, key.str()); + } + + unique_ptr input_expr; + if (indices.size() == 1) { + input_expr = parts[indices[0]].expr->Copy(); + } else { + input_expr = BuildConcatFromParts(concat_proto, parts, indices); + } + + return BuildMatMulExpr(input.context, std::move(input_expr), + W_path, mode_str, W_slice, + row_count, n, bd->mode); + }; + + InsertRecord rec; + + if (!left_part_indices.empty()) { + auto mm_left = build_side_mat_mul(left_part_indices, 0); + if (mm_left) { + idx_t col_idx = 0; + idx_t table_idx = idx_t(-1); + if (InsertProjectionOnSide(join, 0, *mm_left, col_idx, table_idx)) { + rec.left_col = col_idx; + } + } + } + if (!right_part_indices.empty()) { + auto mm_right = build_side_mat_mul(right_part_indices, 1); + if (mm_right) { + idx_t col_idx = 0; + idx_t table_idx = idx_t(-1); + if (InsertProjectionOnSide(join, 1, *mm_right, col_idx, table_idx)) { + rec.right_col = col_idx; + } + } + } + + insert_map[node->plan_node] = rec; + join.ResolveOperatorTypes(); + } + + // --------------------------------------------------------------------------- + // Replace the top-level mat_mul with a tree of list_add_vv over + // BoundReferenceExpressions pointing at inserted partial results. + // Walk label=2 nodes (and the root join) to collect references. + // --------------------------------------------------------------------------- + + auto &top_join = proj.children[0]->Cast(); + top_join.ResolveOperatorTypes(); + auto top_bindings = top_join.GetColumnBindings(); + + auto find_top_index = [&](const ColumnBinding &binding) -> idx_t { + for (idx_t i = 0; i < top_bindings.size(); ++i) { + if (top_bindings[i] == binding) return i; + } + return idx_t(-1); + }; + + // Find all nodes with inserted projections and build list_add_vv chain. + // Resolve each inserted column through the top join's current binding map, + // then reference the matching position with BoundReferenceExpression. + std::vector> partial_refs; + + for (auto &node_ptr : node_storage) { + JoinTreeNode *node = node_ptr.get(); + if (node->label != 1) continue; + auto it = insert_map.find(node->plan_node); + if (it == insert_map.end()) continue; + + auto &jn = node->plan_node->Cast(); + jn.ResolveOperatorTypes(); + auto node_bindings = jn.GetColumnBindings(); + auto left_child_bindings = jn.children[0]->GetColumnBindings(); + + if (it->second.left_col != idx_t(-1) && it->second.left_col < node_bindings.size()) { + const auto node_binding = node_bindings[it->second.left_col]; + const idx_t top_idx = find_top_index(node_binding); + if (top_idx != idx_t(-1) && top_idx < top_join.types.size()) { + partial_refs.push_back( + make_uniq(top_join.types[top_idx], top_idx)); + } + } + if (it->second.right_col != idx_t(-1)) { + const idx_t node_idx = left_child_bindings.size() + it->second.right_col; + if (node_idx < node_bindings.size()) { + const auto node_binding = node_bindings[node_idx]; + const idx_t top_idx = find_top_index(node_binding); + if (top_idx != idx_t(-1) && top_idx < top_join.types.size()) { + partial_refs.push_back( + make_uniq(top_join.types[top_idx], top_idx)); + } + } + } + } + + if (partial_refs.size() < 2) return changed; // need at least 2 to combine + + // Fold into list_add_vv(list_add_vv(...), last) + unique_ptr combined = std::move(partial_refs[0]); + for (idx_t i = 1; i < partial_refs.size(); ++i) + combined = BuildListAddVVExpr(input.context, std::move(combined), std::move(partial_refs[i])); + + *mat_mul_slot = std::move(combined); + return true; +} + +} // namespace duckdb diff --git a/src/optimization/dynamic_rule.cpp b/src/optimization/dynamic_rule.cpp new file mode 100644 index 0000000..e2dafeb --- /dev/null +++ b/src/optimization/dynamic_rule.cpp @@ -0,0 +1,164 @@ +#include "optimization/dynamic_rule.hpp" +#include "optimization/metric_catalog.hpp" + +#include "optimization/actions/MatMulDense2SparseRewriteAction.hpp" +#include "optimization/actions/MultiLayerUDF2TorchNNRewriteAction.hpp" +#include "optimization/actions/MLDecompositionPushdownRewriteAction.hpp" +#include "optimization/actions/MLFactorizationRewriteAction.hpp" +#include "optimization/actions/DecisionForestUDF2RelationRewriteAction.hpp" + +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +// Trim leading/trailing ASCII whitespace. +std::string Trim(const std::string &s) { + size_t b = s.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) { + return ""; + } + size_t e = s.find_last_not_of(" \t\r\n"); + return s.substr(b, e - b + 1); +} + +// Split on a single-character delimiter (empty fields preserved). +std::vector Split(const std::string &s, char delim) { + std::vector out; + std::string cur; + std::stringstream ss(s); + while (std::getline(ss, cur, delim)) { + out.push_back(cur); + } + return out; +} + +// Whitespace tokenizer (collapses runs of whitespace). +std::vector Tokenize(const std::string &s) { + std::vector out; + std::stringstream ss(s); + std::string tok; + while (ss >> tok) { + out.push_back(tok); + } + return out; +} + +bool CompareMetric(double lhs, const std::string &op, double rhs) { + if (op == ">") return lhs > rhs; + if (op == "<") return lhs < rhs; + if (op == ">=") return lhs >= rhs; + if (op == "<=") return lhs <= rhs; + if (op == "=" || op == "==") return lhs == rhs; + if (op == "!=") return lhs != rhs; + return false; +} + +using ActionFn = bool (*)(OptimizerExtensionInput &, unique_ptr &); + +const std::unordered_map &ActionTable() { + static const std::unordered_map table = { + {"MLDecompositionPushdownRewriteAction", &MLDecompositionPushdownRewriteAction::apply}, + {"MatMulDense2SparseRewriteAction", &MatMulDense2SparseRewriteAction::apply}, + {"MultiLayerUDF2TorchNNRewriteAction", &MultiLayerUDF2TorchNNRewriteAction::apply}, + {"MLFactorizationRewriteAction", &MLFactorizationRewriteAction::apply}, + {"DecisionForestUDF2RelationRewriteAction", &DecisionForestUDF2RelationRewriteAction::apply}, + }; + return table; +} + +} // namespace + +DynamicRule DynamicRule::Parse(const std::string &rule_str) { + DynamicRule rule; + + // Expect exactly: RULE## + auto sections = Split(rule_str, '#'); + if (sections.size() < 3 || Trim(sections[0]) != "RULE") { + return rule; // empty rule => no conditions, no actions + } + const std::string conditions_section = Trim(sections[1]); + const std::string actions_section = Trim(sections[2]); + + // --- conditions: walk whitespace tokens as [metric op value] triples, + // with AND/OR keywords in between. --- + auto tokens = Tokenize(conditions_section); + bool next_is_or = false; // logic for the upcoming condition + size_t i = 0; + while (i + 2 < tokens.size()) { + RuleCondition c; + c.metric = tokens[i]; + c.op = tokens[i + 1]; + try { + c.value = std::stod(tokens[i + 2]); + } catch (...) { + c.value = 0.0; + } + c.is_or = next_is_or; + rule.conditions.push_back(c); + i += 3; + + // optional logic keyword before the next triple + next_is_or = false; + if (i < tokens.size()) { + std::string kw = tokens[i]; + for (auto &ch : kw) { + ch = (char)std::toupper((unsigned char)ch); + } + if (kw == "AND" || kw == "OR") { + next_is_or = (kw == "OR"); + i += 1; + } + } + } + + // --- actions: comma separated --- + for (auto &a : Split(actions_section, ',')) { + std::string name = Trim(a); + if (!name.empty()) { + rule.actions.push_back(name); + } + } + + return rule; +} + +bool DynamicRule::EvaluateConditions(const std::string &schema) const { + if (conditions.empty()) { + return true; // no conditions => always apply + } + const auto &mc = MetricCatalog::Get(); + bool result = false; + for (size_t i = 0; i < conditions.size(); i++) { + const auto &c = conditions[i]; + double v = mc.GetMetricOrDefault(schema, c.metric, 0.0); + bool pass = CompareMetric(v, c.op, c.value); + if (i == 0) { + result = pass; + } else if (c.is_or) { + result = result || pass; + } else { + result = result && pass; + } + } + return result; +} + +void DynamicRule::Apply(OptimizerExtensionInput &input, unique_ptr &plan) const { + const auto &table = ActionTable(); + for (const auto &name : actions) { + auto it = table.find(name); + if (it == table.end()) { + std::cerr << "[DynamicRule] unknown action skipped: " << name << "\n"; + continue; + } + it->second(input, plan); + } +} + +} // namespace duckdb diff --git a/src/optimization/optimizer_pass.cpp b/src/optimization/optimizer_pass.cpp new file mode 100644 index 0000000..579fa75 --- /dev/null +++ b/src/optimization/optimizer_pass.cpp @@ -0,0 +1,84 @@ +#include "optimization/optimizer_pass.hpp" + +#include "duckdb/function/table_function.hpp" +#include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/common/types/data_chunk.hpp" + +#include + +namespace duckdb { + +OptimizerPassRegistry &OptimizerPassRegistry::Get() { + static OptimizerPassRegistry instance; + return instance; +} + +void OptimizerPassRegistry::Register(OptimizerPass pass) { + passes_[pass.name] = std::move(pass); // last-wins so a re-upload supersedes +} + +const OptimizerPass *OptimizerPassRegistry::Find(const std::string &name) const { + auto it = passes_.find(name); + return it == passes_.end() ? nullptr : &it->second; +} + +std::vector OptimizerPassRegistry::Names() const { + std::vector names; + names.reserve(passes_.size()); + for (const auto &kv : passes_) { + names.push_back(kv.first); + } + std::sort(names.begin(), names.end()); + return names; +} + +// --------------------------------------------------------------------------- +// cactusdb_list_opt_passes() table function +// --------------------------------------------------------------------------- + +namespace { + +struct ListPassesBindData : public TableFunctionData { + std::vector names; +}; + +struct ListPassesState : public GlobalTableFunctionState { + idx_t cursor = 0; +}; + +unique_ptr ListPassesBind(ClientContext &, TableFunctionBindInput &, + vector &return_types, vector &names) { + return_types.push_back(LogicalType::VARCHAR); + names.push_back("name"); + auto bind = make_uniq(); + bind->names = OptimizerPassRegistry::Get().Names(); + return std::move(bind); +} + +unique_ptr ListPassesInit(ClientContext &, TableFunctionInitInput &) { + return make_uniq(); +} + +void ListPassesFunction(ClientContext &, TableFunctionInput &data, DataChunk &output) { + auto &bind = data.bind_data->Cast(); + auto &state = data.global_state->Cast(); + + idx_t remaining = bind.names.size() - state.cursor; + idx_t count = std::min(STANDARD_VECTOR_SIZE, remaining); + output.SetCardinality(count); + for (idx_t i = 0; i < count; i++) { + output.data[0].SetValue(i, Value(bind.names[state.cursor + i])); + } + state.cursor += count; +} + +} // namespace + +void RegisterOptimizerPassListFunction(ExtensionLoader &loader) { + TableFunction tf("cactusdb_list_opt_passes", {}, ListPassesFunction); + tf.bind = ListPassesBind; + tf.init_global = ListPassesInit; + loader.RegisterFunction(tf); +} + +} // namespace duckdb diff --git a/src/optimization/register_optimizers.cpp b/src/optimization/register_optimizers.cpp index 594db1d..8ec4201 100644 --- a/src/optimization/register_optimizers.cpp +++ b/src/optimization/register_optimizers.cpp @@ -5,9 +5,11 @@ #include "duckdb/optimizer/optimizer_extension.hpp" #include "optimization/add5_rule.hpp" -#include "optimization/replace_binding_rule.hpp" -#include "optimization/dynamic_optimizer.hpp" +#include "optimization/replace_binding_rule.hpp" +#include "optimization/dynamic_optimizer.hpp" #include "optimization/metric_catalog.hpp" +#include "optimization/dynamic_rule.hpp" +#include "optimization/optimizer_pass.hpp" #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" @@ -23,20 +25,30 @@ #include "optimization/actions/MatMulDense2SparseRewriteAction.hpp" #include "optimization/actions/MultiLayerUDF2TorchNNRewriteAction.hpp" #include "optimization/actions/MLDecompositionPushdownRewriteAction.hpp" +#include "optimization/actions/MLFactorizationRewriteAction.hpp" +// MLGreedyFactorizationOptimizer is no longer compiled into the baseline; it is +// added at runtime via an uploaded plugin that self-registers into +// OptimizerPassRegistry and is selected with the "OPT#" setting. namespace duckdb { -//helper function to get schema name from LogicalOperator +// Return a catalog-qualified schema name ("catalog.schema") from the first +// LOGICAL_GET found in the plan tree. This lets the MetricCatalog distinguish +// uc08_db.main from uc03_db.main even though both have schema name "main". +// For the default in-memory catalog ("memory") we return just the schema name +// so that the "synthetic" entry in MetricCatalog continues to match. static string GetQuerySchema(LogicalOperator &op) { if (op.type == LogicalOperatorType::LOGICAL_GET) { auto &get = op.Cast(); auto table_entry = get.GetTable(); if (table_entry) { - // This line may or may not compile depending on visibility: - return table_entry->schema.name; - - // If that complains about access, just fall back: - // return table_entry->GetName(); // as described above + const string &schema_name = table_entry->schema.name; + const string catalog_name = table_entry->catalog.GetName(); + if (!catalog_name.empty() && + catalog_name != "memory" && catalog_name != "system") { + return catalog_name + "." + schema_name; + } + return schema_name; } return string(); } @@ -130,6 +142,33 @@ struct CactusDynamicOpt : public OptimizerExtension { in.context.TryGetCurrentSetting("enable_extended_optimizer", setting); std::string opt_setting = setting.ToString(); + // Dynamic rule path: a frontend-authored optimizer serialized as + // "RULE##". Conditions are evaluated against the + // MetricCatalog for this query's schema; matched actions are applied. + if (DynamicRule::IsRuleString(opt_setting)) { + auto rule = DynamicRule::Parse(opt_setting); + if (rule.EvaluateConditions(query_schema)) { + rule.Apply(in, plan); + } + return; + } + + // "OPT#" — run a single registered optimizer pass by name. + // Passes self-register into OptimizerPassRegistry from compiled-in plugin + // sources (plugins/opt_passes/*.cpp). An unknown name is a no-op so the + // baseline behaves correctly before a pass has been uploaded + rebuilt. + if (opt_setting.rfind("OPT#", 0) == 0) { + const std::string pass_name = opt_setting.substr(4); + if (const auto *pass = OptimizerPassRegistry::Get().Find(pass_name)) { + if (pass->check(in, plan)) { + pass->apply(in, plan); + } + } else { + std::cerr << "[OptPass] unknown pass skipped: " << pass_name << "\n"; + } + return; + } + if (opt_setting.find("1") != std::string::npos) { if (join_ratio > HIGH_JOIN_RATIO) { MLDecompositionPushdownRewriteAction::apply(in, plan); @@ -147,7 +186,16 @@ struct CactusDynamicOpt : public OptimizerExtension { MultiLayerUDF2TorchNNRewriteAction::apply(in, plan); } - // pretty = plan->ToString(); + if (opt_setting.find("4") != std::string::npos) { + if (MLFactorizationRewriteAction::check(in, plan)) { + MLFactorizationRewriteAction::apply(in, plan); + } + } + + // Flag "5" (greedy factorization) is intentionally gone from the baseline. + // The greedy pass is now an uploadable plugin selected via "OPT#greedy_factorization". + + // pretty = plan->ToString(); // std::cout << "\n[CactusDynamicOpt] Final Plan:\n" << pretty << "\n"; } @@ -158,12 +206,8 @@ struct CactusDynamicOpt1 : public OptimizerExtension { static void Run(OptimizerExtensionInput &in, unique_ptr &plan) { // DumpPlanBase64(in.context, *plan, "./plans/initial_plan.duckdbplan"); - std::string pretty = plan->ToString(); - std::cout << "\nInitial Plan:\n" << pretty << "\n"; MatMulDense2SparseRewriteAction::apply(in, plan); // DumpPlanBase64(in.context, *plan, "./plans/final_plan.duckdbplan"); - pretty = plan->ToString(); - std::cout << "\nFinal Plan:\n" << pretty << "\n"; } }; diff --git a/test/sql/greedy_factorization.test b/test/sql/greedy_factorization.test new file mode 100644 index 0000000..cf7aa99 --- /dev/null +++ b/test/sql/greedy_factorization.test @@ -0,0 +1,105 @@ +# name: test/sql/greedy_factorization.test +# description: Test MLGreedyFactorizationOptimizer (optimizer option "5") +# 3-table join with mat_mul(list_concat(t1.f, t2.f, t3.f), W, 'dense') +# group: [sql] + +require cactusdb + +statement ok +LOAD cactusdb; + +# ------------------------------------------------------------------ +# Weight matrix W: shape (6 x 2) +# t1 contributes k1=2 rows, t2 k2=2 rows, t3 k3=2 rows +# +# W = [[1, 0], <- row 0 (t1 block) +# [0, 1], <- row 1 +# [1, 0], <- row 2 (t2 block) +# [0, 1], <- row 3 +# [1, 0], <- row 4 (t3 block) +# [0, 1]] <- row 5 +# +# For input [a,b | c,d | e,f]: +# result = [a+c+e, b+d+f] +# ------------------------------------------------------------------ + +statement ok +CREATE TEMP TABLE w_greedy(c1 DOUBLE, c2 DOUBLE); + +statement ok +INSERT INTO w_greedy VALUES + (1,0),(0,1), + (1,0),(0,1), + (1,0),(0,1); + +statement ok +COPY w_greedy TO '/tmp/w_greedy.csv' (FORMAT CSV, HEADER FALSE); + +# ------------------------------------------------------------------ +# Base tables: +# t1: id + 2-element feature vector +# t2: id + 2-element feature vector +# t3: id + 2-element feature vector +# ------------------------------------------------------------------ + +statement ok +CREATE TABLE t1_greedy(id INTEGER, f DOUBLE[2]); + +statement ok +INSERT INTO t1_greedy VALUES (1, [1.0, 2.0]), (2, [3.0, 4.0]); + +statement ok +CREATE TABLE t2_greedy(id INTEGER, f DOUBLE[2]); + +statement ok +INSERT INTO t2_greedy VALUES (1, [10.0, 20.0]), (2, [30.0, 40.0]); + +statement ok +CREATE TABLE t3_greedy(id INTEGER, f DOUBLE[2]); + +statement ok +INSERT INTO t3_greedy VALUES (1, [100.0, 200.0]), (2, [300.0, 400.0]); + +# ------------------------------------------------------------------ +# Baseline: compute expected results WITHOUT optimizer +# result for id=1: [1+10+100, 2+20+200] = [111, 222] +# result for id=2: [3+30+300, 4+40+400] = [333, 444] +# ------------------------------------------------------------------ + +query I +SELECT mat_mul(list_concat(list_concat(t1.f, t2.f), t3.f), '/tmp/w_greedy.csv', 'dense') AS result +FROM t1_greedy t1 +JOIN t2_greedy t2 ON t1.id = t2.id +JOIN t3_greedy t3 ON t1.id = t3.id +ORDER BY t1.id; +---- +[111.0, 222.0] +[333.0, 444.0] + +# ------------------------------------------------------------------ +# Same query with greedy optimizer enabled — result must be identical +# ------------------------------------------------------------------ + +# Intentionally disable built-in optimizers (but not extension) so profile 5 +# runs without projection-map-producing passes. +statement ok +SET disabled_optimizers='expression_rewriter,filter_pullup,filter_pushdown,empty_result_pullup,cte_filter_pusher,regex_range,in_clause,join_order,deliminator,unnest_rewriter,unused_columns,statistics_propagation,common_subexpressions,common_aggregate,column_lifetime,limit_pushdown,top_n,build_side_probe_side,compressed_materialization,duplicate_groups,reorder_filter,sampling_pushdown,join_filter_pushdown,materialized_cte,sum_rewriter,late_materialization,cte_inlining'; + +statement ok +SET enable_extended_optimizer = '5'; + +query I +SELECT mat_mul(list_concat(list_concat(t1.f, t2.f), t3.f), '/tmp/w_greedy.csv', 'dense') AS result +FROM t1_greedy t1 +JOIN t2_greedy t2 ON t1.id = t2.id +JOIN t3_greedy t3 ON t1.id = t3.id +ORDER BY t1.id; +---- +[111.0, 222.0] +[333.0, 444.0] + +statement ok +SET enable_extended_optimizer = ''; + +statement ok +SET disabled_optimizers = '';