From 4064b724c11c0ab0b9eaa1c27e909d0e6e107c6d Mon Sep 17 00:00:00 2001 From: jiazou-bigdata Date: Tue, 16 Sep 2025 04:44:41 -0700 Subject: [PATCH 1/2] Add implementation and test cases for matrix multiplication: Implementation: src/ml_functions/ml_functions_scalar.cpp Test cases: test/sql/cactusdb.test --- CMakeLists.txt | 11 +- logfile | 16 ++ src/cactusdb_extension.cpp | 6 +- src/include/function_builder.hpp | 239 ++++++++++++++++++++++ src/include/functions.hpp | 13 ++ src/ml_functions/CMakeLists.txt | 4 + src/ml_functions/ml_functions_scalar.cpp | 242 +++++++++++++++++++++++ src/util/CMakeLists.txt | 4 + src/util/function_builder.cpp | 195 ++++++++++++++++++ test/README.md | 2 +- test/sql/cactusdb.test | 26 +++ 11 files changed, 751 insertions(+), 7 deletions(-) create mode 100644 logfile create mode 100644 src/include/function_builder.hpp create mode 100644 src/include/functions.hpp create mode 100644 src/ml_functions/CMakeLists.txt create mode 100644 src/ml_functions/ml_functions_scalar.cpp create mode 100644 src/util/CMakeLists.txt create mode 100644 src/util/function_builder.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 23b887b..dbc450d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,8 @@ set(TARGET_NAME cactusdb) # used in cmake with find_package. Feel free to remove or replace with other dependencies. # Note that it should also be removed from vcpkg.json to prevent needlessly installing it.. -## TO BE REMOVED find_package(OpenSSL REQUIRED) +find_package(Eigen3 REQUIRED) set(EXTENSION_NAME ${TARGET_NAME}_extension) @@ -18,16 +18,19 @@ set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) project(${TARGET_NAME}) include_directories(src/include) +include_directories(${EIGEN3_INCLUDE_DIR}) -## TO BE REPLACED set(EXTENSION_SOURCES src/cactusdb_extension.cpp) +add_subdirectory(src/ml_functions) +add_subdirectory(src/util) +message(STATUS "EXTENSION_SOURCES include ${EXTENSION_SOURCES}") build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) # Link OpenSSL in both the static library as the loadable extension -target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) -target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) +target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto Eigen3::Eigen) +target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto Eigen3::Eigen) install( TARGETS ${EXTENSION_NAME} diff --git a/logfile b/logfile new file mode 100644 index 0000000..d5a4842 --- /dev/null +++ b/logfile @@ -0,0 +1,16 @@ +2025-09-15 11:51:32.458 MST [77299] LOG: starting PostgreSQL 15.14 (Homebrew) on aarch64-apple-darwin23.6.0, compiled by Apple clang version 16.0.0 (clang-1600.0.26.6), 64-bit +2025-09-15 11:51:32.459 MST [77299] LOG: listening on IPv4 address "127.0.0.1", port 8888 +2025-09-15 11:51:32.459 MST [77299] LOG: listening on IPv6 address "::1", port 8888 +2025-09-15 11:51:32.460 MST [77299] LOG: listening on Unix socket "/tmp/.s.PGSQL.8888" +2025-09-15 11:51:32.466 MST [77302] LOG: database system was interrupted; last known up at 2025-09-03 13:56:52 MST +2025-09-15 11:51:32.537 MST [77302] LOG: database system was not properly shut down; automatic recovery in progress +2025-09-15 11:51:32.541 MST [77302] LOG: redo starts at 0/43401828 +2025-09-15 11:51:35.046 MST [77302] LOG: invalid record length at 0/50F774F8: wanted 24, got 0 +2025-09-15 11:51:35.046 MST [77302] LOG: redo done at 0/50F774C0 system usage: CPU: user: 0.22 s, system: 0.93 s, elapsed: 2.50 s +2025-09-15 11:51:35.051 MST [77300] LOG: checkpoint starting: end-of-recovery immediate wait +2025-09-15 11:51:35.580 MST [77300] LOG: checkpoint complete: wrote 16375 buffers (99.9%); 0 WAL file(s) added, 13 removed, 0 recycled; write=0.390 s, sync=0.005 s, total=0.533 s; sync files=64, longest=0.003 s, average=0.001 s; distance=224727 kB, estimate=224727 kB +2025-09-15 11:51:35.584 MST [77299] LOG: database system is ready to accept connections +2025-09-15 11:53:12.240 MST [77334] ERROR: relation "customers" does not exist at character 22 +2025-09-15 11:53:12.240 MST [77334] STATEMENT: SELECT count(*) FROM CUSTOMERS; +2025-09-15 11:53:12.301 MST [77334] ERROR: relation "suppliers" does not exist at character 22 +2025-09-15 11:53:12.301 MST [77334] STATEMENT: SELECT count(*) FROM SUPPLIERS; diff --git a/src/cactusdb_extension.cpp b/src/cactusdb_extension.cpp index 6a6ce64..f872cba 100644 --- a/src/cactusdb_extension.cpp +++ b/src/cactusdb_extension.cpp @@ -1,7 +1,6 @@ -#define DUCKDB_EXTENSION_MAIN - #include "cactusdb_extension.hpp" #include "duckdb.hpp" +#include "functions.hpp" // OpenSSL linked through vcpkg #include @@ -9,6 +8,9 @@ namespace duckdb { static void LoadInternal(ExtensionLoader &loader) { + + RegisterMLScalarFunctions(loader); + } void CactusdbExtension::Load(ExtensionLoader &loader) { diff --git a/src/include/function_builder.hpp b/src/include/function_builder.hpp new file mode 100644 index 0000000..ea9a8d6 --- /dev/null +++ b/src/include/function_builder.hpp @@ -0,0 +1,239 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/function/function_set.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/parser/parsed_data/create_function_info.hpp" +#include "duckdb/common/insertion_order_preserving_map.hpp" +#include "duckdb/catalog/default/default_functions.hpp" + +namespace duckdb { + +//------------------------------------------------------------------------------ +// Function Builder +//------------------------------------------------------------------------------ + +class ScalarFunctionBuilder; +class AggregateFunctionBuilder; +class MacroFunctionBuilder; + +class FunctionBuilder { +public: + template + static void RegisterScalar(ExtensionLoader &loader, const char *name, CALLBACK &&callback); + + template + static void RegisterAggregate(ExtensionLoader &loader, const char *name, CALLBACK &&callback); + + template + static void RegisterMacro(ExtensionLoader &loader, const char *name, CALLBACK &&callback); + + // TODO: + static void AddTableFunctionDocs(ExtensionLoader &loader, const char *name, const char *desc, const char *example, + const InsertionOrderPreservingMap &tags); + + static string RemoveIndentAndTrailingWhitespace(const char *str); + +private: + static void Register(ExtensionLoader &loader, const char *name, ScalarFunctionBuilder &builder); + static void Register(ExtensionLoader &loader, const char *name, AggregateFunctionBuilder &builder); + static void Register(ExtensionLoader &loader, const char *name, MacroFunctionBuilder &builder); +}; + +//------------------------------------------------------------------------------ +// Scalar Function Variant Builder +//------------------------------------------------------------------------------ + +class ScalarFunctionVariantBuilder { + friend class ScalarFunctionBuilder; + +public: + void AddParameter(const char *name, const LogicalType &type); + void SetReturnType(LogicalType type); + void SetFunction(scalar_function_t fn); + void SetInit(init_local_state_t init); + void SetBind(bind_scalar_function_t bind); + void SetDescription(const string &desc); + void SetExample(const string &ex); + +private: + explicit ScalarFunctionVariantBuilder() : function({}, LogicalTypeId::INVALID, nullptr) { + } + + ScalarFunction function; + FunctionDescription description = {}; +}; + +inline void ScalarFunctionVariantBuilder::AddParameter(const char *name, const LogicalType &type) { + function.arguments.emplace_back(type); + description.parameter_names.emplace_back(name); + description.parameter_types.emplace_back(type); +} + +inline void ScalarFunctionVariantBuilder::SetReturnType(LogicalType type) { + function.return_type = std::move(type); +} + +inline void ScalarFunctionVariantBuilder::SetFunction(scalar_function_t fn) { + function.function = fn; +} + +inline void ScalarFunctionVariantBuilder::SetInit(init_local_state_t init) { + function.init_local_state = init; +} + +inline void ScalarFunctionVariantBuilder::SetBind(bind_scalar_function_t bind) { + function.bind = bind; +} + +inline void ScalarFunctionVariantBuilder::SetDescription(const string &desc) { + description.description = desc; +} + +inline void ScalarFunctionVariantBuilder::SetExample(const string &ex) { + description.examples.emplace_back(ex); +} + +//------------------------------------------------------------------------------ +// Scalar Function Builder +//------------------------------------------------------------------------------ + +class ScalarFunctionBuilder { + friend class FunctionBuilder; + +public: + template + void AddVariant(CALLBACK &&callback); + void SetTag(const string &key, const string &value); + void SetDescription(const string &desc); + void SetExample(const string &ex); + +private: + explicit ScalarFunctionBuilder(const char *name) : set(name) { + } + + ScalarFunctionSet set; + vector descriptions = {}; + InsertionOrderPreservingMap tags = {}; + + // If not set by a variant + string default_description; + string default_example; +}; + +inline void ScalarFunctionBuilder::SetDescription(const string &desc) { + default_description = FunctionBuilder::RemoveIndentAndTrailingWhitespace(desc.c_str()); +} + +inline void ScalarFunctionBuilder::SetExample(const string &ex) { + default_example = FunctionBuilder::RemoveIndentAndTrailingWhitespace(ex.c_str()); +} + +inline void ScalarFunctionBuilder::SetTag(const string &key, const string &value) { + tags[key] = value; +} + +template +void ScalarFunctionBuilder::AddVariant(CALLBACK &&callback) { + ScalarFunctionVariantBuilder builder; + + callback(builder); + + // A return type is required + if (builder.function.return_type.id() == LogicalTypeId::INVALID) { + throw InternalException("Return type not set in ScalarFunctionBuilder::AddVariant"); + } + + // Add the new variant to the set + set.AddFunction(std::move(builder.function)); + + // Add the description + descriptions.emplace_back(std::move(builder.description)); +} + +//------------------------------------------------------------------------------ +// Macro +//------------------------------------------------------------------------------ +class MacroFunctionBuilder { + friend class FunctionBuilder; + +public: + void AddDefinition(const vector ¶meters, const string &body, const char *desc = nullptr, + const char *example = nullptr) { + macros.push_back({parameters, body, desc, example}); + } + +private: + struct MacroDef { + vector parameters; + string body; + const char *description; + const char *example; + }; + + vector macros; +}; + +//------------------------------------------------------------------------------ +// Aggregate +//------------------------------------------------------------------------------ + +class AggregateFunctionBuilder { + friend class FunctionBuilder; + +public: + void SetTag(const string &key, const string &value); + void SetDescription(const string &desc); + void SetExample(const string &ex); + void SetFunction(const AggregateFunction &function); + +private: + explicit AggregateFunctionBuilder(const char *name) : set(name) { + } + string description; + string example; + InsertionOrderPreservingMap tags; + AggregateFunctionSet set; +}; + +inline void AggregateFunctionBuilder::SetFunction(const AggregateFunction &function) { + set.AddFunction(function); +} + +inline void AggregateFunctionBuilder::SetDescription(const string &desc) { + description = desc; +} +inline void AggregateFunctionBuilder::SetExample(const string &ex) { + example = ex; +} +inline void AggregateFunctionBuilder::SetTag(const string &key, const string &value) { + tags[key] = value; +} + +//------------------------------------------------------------------------------ +// Function Builder Methods +//------------------------------------------------------------------------------ + +template +void FunctionBuilder::RegisterScalar(ExtensionLoader &loader, const char *name, CALLBACK &&callback) { + ScalarFunctionBuilder builder(name); + callback(builder); + + Register(loader, name, builder); +} + +template +void FunctionBuilder::RegisterAggregate(ExtensionLoader &loader, const char *name, CALLBACK &&callback) { + AggregateFunctionBuilder builder(name); + callback(builder); + Register(loader, name, builder); +} + +template +void FunctionBuilder::RegisterMacro(ExtensionLoader &loader, const char *name, CALLBACK &&callback) { + MacroFunctionBuilder builder; + callback(builder); + Register(loader, name, builder); +} + +} // namespace duckdb diff --git a/src/include/functions.hpp b/src/include/functions.hpp new file mode 100644 index 0000000..5f1648c --- /dev/null +++ b/src/include/functions.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "duckdb/common/typedefs.hpp" + +#include "duckdb.hpp" + +namespace duckdb { + +class ExtensionLoader; + +void RegisterMLScalarFunctions(ExtensionLoader &loader); + +} diff --git a/src/ml_functions/CMakeLists.txt b/src/ml_functions/CMakeLists.txt new file mode 100644 index 0000000..5d5c838 --- /dev/null +++ b/src/ml_functions/CMakeLists.txt @@ -0,0 +1,4 @@ +set(EXTENSION_SOURCES + ${EXTENSION_SOURCES} + ${CMAKE_CURRENT_SOURCE_DIR}/ml_functions_scalar.cpp +PARENT_SCOPE) diff --git a/src/ml_functions/ml_functions_scalar.cpp b/src/ml_functions/ml_functions_scalar.cpp new file mode 100644 index 0000000..12047cd --- /dev/null +++ b/src/ml_functions/ml_functions_scalar.cpp @@ -0,0 +1,242 @@ +#include +#include +#include "functions.hpp" +#include "function_builder.hpp" +#include "duckdb.hpp" +#include "duckdb/common/types/value.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/common/enum_util.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/function/scalar_function.hpp" +#include +#include "duckdb/common/constants.hpp" +#include "duckdb/common/types/blob.hpp" +#include "duckdb/common/vector_operations/generic_executor.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/common/vector_operations/septenary_executor.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" + +namespace duckdb { + +class MatrixData final : public FunctionData { + +public: + unsigned int num_rows_; + unsigned int num_columns_; + + bool use_gpu_; + + explicit MatrixData (int num_rows, int num_columns, bool use_gpu) { + num_rows_ = num_rows; + num_columns_ = num_columns; + use_gpu_ = use_gpu; + } + + unique_ptr Copy() const override { + return make_uniq(num_rows_, num_columns_, use_gpu_); + } + + bool Equals(const FunctionData &other) const override { + auto &other_data = other.Cast(); + return num_rows_ == other_data.num_rows_ && num_columns_ == other_data.num_columns_ + && use_gpu_ == other_data.use_gpu_; + } + +}; + + +class MatrixDataByPointer final : public FunctionData { + +public: + float* weights_; + unsigned int num_rows_; + unsigned int num_columns_; + bool use_gpu_; + + + explicit MatrixDataByPointer (float* weights, int num_rows, int num_columns, bool use_gpu) { + weights_ = new float[num_rows * num_columns]; + std::memcpy(weights_, weights, num_rows * num_columns * sizeof(float)); + num_rows_ = num_rows; + num_columns_ = num_columns; + use_gpu_ = use_gpu; + } + + unique_ptr Copy() const override { + return make_uniq(weights_, num_rows_, num_columns_, use_gpu_); + } + + bool Equals(const FunctionData &other) const override { + auto &other_data = other.Cast(); + return num_rows_ == other_data.num_rows_ && num_columns_ == other_data.num_columns_ + && weights_ == other_data.weights_ && use_gpu_ == other_data.use_gpu_; + } + +}; + + +/* +static unique_ptr MatrixBindByPointer(ClientContext &context, ScalarFunction &bound_function, + vector> &arguments) { + + const auto expr0 = ExpressionExecutor::EvaluateScalar(context, *arguments[0]); + const auto weights_value = expr0.GetPointer(); + const auto expr1 = ExpressionExecutor::EvaluateScalar(context, *arguments[1]); + const auto num_rows_value = expr1.GetValue(); + const auto expr2 = ExpressionExecutor::EvaluateScalar(context, *arguments[2]); + const auto num_columns_value = expr2.GetValue(); + const auto expr3 = ExpressionExecutor::EvaluateScalar(context, *arguments[3]); + const auto use_gpu = expr3.GetValue(); + return make_uniq((float *)weights_value, num_rows_value, num_columns_value, use_gpu); + +} + + +static unique_ptr MatrixBind(ClientContext &context, ScalarFunction &bound_function, + vector> &arguments) { + return make_uniq(8, 2, false); +} + +*/ + + +struct ML_MatrixMultiply { + +public: + + static void Execute(DataChunk &args, ExpressionState &state, Vector &result) { + + const auto count = args.size(); + std::cout << "data size" << count << std::endl; + + auto &lhs = ArrayVector::GetEntry(args.data[0]); + auto &rhs = ArrayVector::GetEntry(args.data[1]); + auto &res = ArrayVector::GetEntry(result); + + const auto &lhs_validity = FlatVector::Validity(lhs); + const auto &rhs_validity = FlatVector::Validity(rhs); + + UnifiedVectorFormat lhs_format; + UnifiedVectorFormat rhs_format; + + args.data[0].ToUnifiedFormat(count, lhs_format); + args.data[1].ToUnifiedFormat(count, rhs_format); + + auto lhs_data = FlatVector::GetData(lhs); + auto rhs_data = FlatVector::GetData(rhs); + auto res_data = FlatVector::GetData(res); + + /* + for (int i = 0; i < count; i++) { + std::cout << "lhs-"<< i << ":" << lhs.GetValue(i) << std::endl; + std::cout << "rhs-"<< i << ":"<< rhs.GetValue(i) << std::endl; + } + std::cout << "lhs type:" << int(lhs.GetVectorType()) << std::endl; + std::cout << "lhs type:" << lhs.GetType().ToString() << std::endl; + std::cout << "lhs type:" << EnumUtil::ToChars(lhs.GetType().InternalType()) << std::endl; + std::cout << "rhs type:" << int(GetTypeIdSize(lhs.GetType().InternalType())) << std::endl; + std::cout << "rhs type:" << int(rhs.GetVectorType()) << std::endl; + std::cout << "rhs type:" << EnumUtil::ToChars(rhs.GetType().InternalType()) << std::endl; + std::cout << "rhs type:" << int(GetTypeIdSize(rhs.GetType().InternalType())) << std::endl; + + auto lhs_buffer = lhs.GetBuffer(); + auto rhs_buffer = rhs.GetBuffer(); + + std::cout << "lhs buffer type:" << EnumUtil::ToChars(lhs_buffer->GetBufferType()) << std::endl; + std::cout << "rhs buffer type:" << EnumUtil::ToChars(rhs_buffer->GetBufferType()) << std::endl; + */ + + + Value v2 = args.data[2].GetValue(0); + Value v3 = args.data[3].GetValue(0); + int num_rows = IntegerValue::Get(v2); + int num_columns = IntegerValue::Get(v3); + std::cout << "num rows: " << num_rows << "; num columns: " << num_columns << std::endl; + + const auto rhs_idx = rhs_format.sel->get_index(0); + const auto right_offset = rhs_idx * num_rows; + if (!rhs_validity.CheckAllValid(right_offset + num_rows, right_offset)) { + throw InvalidInputException(StringUtil::Format("right argument can not contain NULL values")); + } + const auto rhs_data_ptr = rhs_data + right_offset; + Eigen::Map> + weight_matrix((float *)rhs_data_ptr, num_rows, num_columns); + + for (idx_t i = 0; i < count; i++) { + const auto lhs_idx = lhs_format.sel->get_index(i); + + if (!lhs_format.validity.RowIsValid(lhs_idx)) { + FlatVector::SetNull(result, i, true); + continue; + } + + const auto left_offset = lhs_idx * num_rows; + if (!lhs_validity.CheckAllValid(left_offset + num_rows, left_offset)) { + throw InvalidInputException(StringUtil::Format("left argument can not contain NULL values")); + } + + const auto result_offset = i * num_rows; + + const auto lhs_data_ptr = lhs_data + left_offset; + const auto res_data_ptr = res_data + result_offset; + + bool use_gpu = false; + + if (use_gpu) { + // TODO: implementation of matrix multiplication in GPU + throw std::runtime_error( + "GPU implementation of Matrix Multiple is not implemented."); + } else { + + Eigen::Map> + input_matrix((float *)lhs_data_ptr, 1, num_rows); + + std::cout << "matrix multiplication" << std::endl; + Eigen::Matrix + result_matrix = input_matrix * weight_matrix; + + std::cout << "moving the output" << std::endl; + //dispatch results to multiple rows + + std::memcpy ( + (float*)res_data_ptr, result_matrix.row(0).data(),num_columns); + } + + } + } + + static void RegisterByArray(ExtensionLoader &loader, int num_features, int num_neurons) { + + FunctionBuilder::RegisterScalar(loader, "matmul", [&](ScalarFunctionBuilder &func) { + func.AddVariant([&](ScalarFunctionVariantBuilder &variant) { + variant.AddParameter("input", LogicalType::ARRAY(LogicalType::FLOAT, num_features)); + variant.AddParameter("weight", LogicalType::ARRAY(LogicalType::FLOAT, num_features * num_neurons)); + variant.AddParameter("num_rows", LogicalType::INTEGER); + variant.AddParameter("num_columns", LogicalType::INTEGER); + variant.SetReturnType(LogicalType::ARRAY(LogicalType::FLOAT, num_neurons)); + variant.SetFunction(Execute); + }); + + func.SetDescription(R"( + Returns matrix results multiplying each input with a pre-defined weight matrix. + )"); + + func.SetTag("ext", "cactusdb"); + func.SetTag("category", "relation"); + }); + + } + +}; + +void RegisterMLScalarFunctions(ExtensionLoader &loader) { + + ML_MatrixMultiply::RegisterByArray(loader, 8, 2); + +} + + +} + diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt new file mode 100644 index 0000000..8c98fcf --- /dev/null +++ b/src/util/CMakeLists.txt @@ -0,0 +1,4 @@ +set(EXTENSION_SOURCES + ${EXTENSION_SOURCES} + ${CMAKE_CURRENT_SOURCE_DIR}/function_builder.cpp +PARENT_SCOPE) diff --git a/src/util/function_builder.cpp b/src/util/function_builder.cpp new file mode 100644 index 0000000..d1d0d86 --- /dev/null +++ b/src/util/function_builder.cpp @@ -0,0 +1,195 @@ +#include "function_builder.hpp" + +#include "duckdb/catalog/catalog_entry/function_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" + +namespace duckdb { + +string FunctionBuilder::RemoveIndentAndTrailingWhitespace(const char *ptr) { + string tmp; + // Replace all tabs with 4 spaces in ptr + for (const char *text = ptr; *text; text++) { + if (*text == '\t') { + tmp.append(" "); + } else { + tmp += *text; + } + } + + auto text = tmp.c_str(); + + string result; + // Skip any empty first newlines if present + while (*text == '\n') { + text++; + } + + // Track indent length + auto indent_start = text; + while (isspace(*text) && *text != '\n') { + text++; + } + auto indent_len = text - indent_start; + while (*text) { + result += *text; + if (*text++ == '\n') { + // Remove all indentation, but only if it matches the first line's indentation + bool matched_indent = true; + for (auto i = 0; i < indent_len; i++) { + if (*text != indent_start[i]) { + matched_indent = false; + break; + } + } + if (matched_indent) { + auto remaining_indent = indent_len; + while (*text && remaining_indent > 0) { + text++; + remaining_indent--; + } + } + } + } + + // Also remove any trailing whitespace + result.erase(result.find_last_not_of(" \n\r\t") + 1); + return result; +} + +void FunctionBuilder::Register(ExtensionLoader &loader, const char *name, ScalarFunctionBuilder &builder) { + // Register the function + loader.RegisterFunction(std::move(builder.set)); + auto &db = loader.GetDatabaseInstance(); + + // Also add the parameter names. We need to access the catalog entry for this. + auto &catalog = Catalog::GetSystemCatalog(db); + auto transaction = CatalogTransaction::GetSystemTransaction(db); + auto &schema = catalog.GetSchema(transaction, DEFAULT_SCHEMA); + auto catalog_entry = schema.GetEntry(transaction, CatalogType::SCALAR_FUNCTION_ENTRY, name); + if (!catalog_entry) { + // This should not happen, we just registered the function + throw InternalException("Function with name \"%s\" not found in FunctionBuilder::AddScalar", name); + } + + auto &func_entry = catalog_entry->Cast(); + + // Insert all descriptions + for (auto &desc : builder.descriptions) { + + // Add default description if none is set + if (desc.description.empty()) { + desc.description = builder.default_description; + } else { + desc.description = RemoveIndentAndTrailingWhitespace(desc.description.c_str()); + } + + // Add default example if none is set + if (desc.examples.empty()) { + desc.examples.push_back(builder.default_example); + } else { + for (auto &ex : desc.examples) { + ex = RemoveIndentAndTrailingWhitespace(ex.c_str()); + } + } + + func_entry.descriptions.push_back(desc); + } + + if (!builder.tags.empty()) { + func_entry.tags = std::move(builder.tags); + } +} + +void FunctionBuilder::Register(ExtensionLoader &loader, const char *name, AggregateFunctionBuilder &builder) { + // Register the function + loader.RegisterFunction(std::move(builder.set)); + auto &db = loader.GetDatabaseInstance(); + + // Also add the parameter names. We need to access the catalog entry for this. + auto &catalog = Catalog::GetSystemCatalog(db); + auto transaction = CatalogTransaction::GetSystemTransaction(db); + auto &schema = catalog.GetSchema(transaction, DEFAULT_SCHEMA); + auto catalog_entry = schema.GetEntry(transaction, CatalogType::AGGREGATE_FUNCTION_ENTRY, name); + if (!catalog_entry) { + // This should not happen, we just registered the function + throw InternalException("Function with name \"%s\" not found in FunctionBuilder::AddAggregate", name); + } + + auto &func_entry = catalog_entry->Cast(); + + // Insert all descriptions + const auto descr = RemoveIndentAndTrailingWhitespace(builder.description.c_str()); + const auto exampl = RemoveIndentAndTrailingWhitespace(builder.example.c_str()); + FunctionDescription function_description; + function_description.description = descr; + function_description.examples.push_back(exampl); + func_entry.descriptions.push_back(function_description); + + if (!builder.tags.empty()) { + func_entry.tags = std::move(builder.tags); + } +} + +void FunctionBuilder::Register(ExtensionLoader &loader, const char *name, MacroFunctionBuilder &builder) { + // Register the function + vector macros; + vector descriptions; + + for (auto &def : builder.macros) { + DefaultMacro macro = {}; + macro.schema = DEFAULT_SCHEMA; + macro.name = name; + macro.named_parameters[0].name = nullptr; + macro.named_parameters[0].default_value = nullptr; + macro.macro = def.body.c_str(); + for (idx_t i = 0; i < def.parameters.size(); i++) { + if (i >= 8) { + throw InternalException("Too many parameters in macro!"); + } + macro.parameters[i] = def.parameters[i].c_str(); + } + macro.parameters[def.parameters.size()] = nullptr; + macros.push_back(macro); + + FunctionDescription function_description; + if (def.description) { + function_description.description = RemoveIndentAndTrailingWhitespace(def.description); + } + if (def.example) { + function_description.examples.push_back(RemoveIndentAndTrailingWhitespace(def.example)); + } + descriptions.push_back(function_description); + } + + const auto macro_ptr = array_ptr(macros.data(), macros.size()); + const auto info = DefaultFunctionGenerator::CreateInternalMacroInfo(macro_ptr); + info->descriptions = descriptions; + + loader.RegisterFunction(*info); +} + +void FunctionBuilder::AddTableFunctionDocs(ExtensionLoader &loader, const char *name, const char *desc, + const char *example, const InsertionOrderPreservingMap &tags) { + + auto &db = loader.GetDatabaseInstance(); + auto &catalog = Catalog::GetSystemCatalog(db); + auto transaction = CatalogTransaction::GetSystemTransaction(db); + auto &schema = catalog.GetSchema(transaction, DEFAULT_SCHEMA); + auto catalog_entry = schema.GetEntry(transaction, CatalogType::TABLE_FUNCTION_ENTRY, name); + if (!catalog_entry) { + // This should not happen, we just registered the function + throw InternalException("Function with name \"%s\" not found in FunctionBuilder::AddScalar", name); + } + + auto &func_entry = catalog_entry->Cast(); + FunctionDescription function_description; + function_description.description = RemoveIndentAndTrailingWhitespace(desc); + function_description.examples.push_back(RemoveIndentAndTrailingWhitespace(example)); + func_entry.descriptions.push_back(function_description); + + for (const auto &tag : tags) { + func_entry.tags.insert(tag.first, tag.second); + } +} + +} // namespace duckdb diff --git a/test/README.md b/test/README.md index fb5e514..ba23067 100644 --- a/test/README.md +++ b/test/README.md @@ -8,4 +8,4 @@ make test or ```bash make test_debug -``` \ No newline at end of file +``` diff --git a/test/sql/cactusdb.test b/test/sql/cactusdb.test index 1fd6ce9..36cab74 100644 --- a/test/sql/cactusdb.test +++ b/test/sql/cactusdb.test @@ -3,3 +3,29 @@ # group: [sql] require cactusdb + +#Create table +statement ok +CREATE TABLE array_data (id INTEGER, features FLOAT[8]); + +#Insert data +statement ok +INSERT INTO array_data VALUES + (1, [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1]), + (2, [2.2, 2.2, 2.2, 2.2, 2.2, 2.2, 2.2, 2.2]), + (3, [3.3, 3.3, 3.3, 3.3, 3.3, 3.3, 3.3, 3.3]), + (4, [4.4, 4.4, 4.4, 4.4, 4.4, 4.4, 4.4, 4.4]), + (5, [5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5]), + (6, [6.6, 6.6, 6.6, 6.6, 6.6, 6.6, 6.6, 6.6]); + +#Test query +statement ok +SELECT id, features, FROM array_data ORDER BY id + +#Test query +statement ok +SELECT id, array_inner_product(features, array_value(1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT)), FROM array_data ORDER BY id + +#Test query +statement ok +SELECT id, matmul(features, array_value(1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT, 1.0::FLOAT), 8, 2), FROM array_data ORDER BY id From e60bf2063c0118ee990f3b7c8bd3ab6d312ce672 Mon Sep 17 00:00:00 2001 From: jiazou-bigdata Date: Tue, 16 Sep 2025 04:52:10 -0700 Subject: [PATCH 2/2] Add missing files --- src/ml_functions/ml_functions_scalar.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ml_functions/ml_functions_scalar.cpp b/src/ml_functions/ml_functions_scalar.cpp index 12047cd..82c2094 100644 --- a/src/ml_functions/ml_functions_scalar.cpp +++ b/src/ml_functions/ml_functions_scalar.cpp @@ -109,7 +109,6 @@ struct ML_MatrixMultiply { static void Execute(DataChunk &args, ExpressionState &state, Vector &result) { const auto count = args.size(); - std::cout << "data size" << count << std::endl; auto &lhs = ArrayVector::GetEntry(args.data[0]); auto &rhs = ArrayVector::GetEntry(args.data[1]); @@ -193,11 +192,9 @@ struct ML_MatrixMultiply { Eigen::Map> input_matrix((float *)lhs_data_ptr, 1, num_rows); - std::cout << "matrix multiplication" << std::endl; Eigen::Matrix result_matrix = input_matrix * weight_matrix; - std::cout << "moving the output" << std::endl; //dispatch results to multiple rows std::memcpy (