From e196cc6b376637e39214095c4d3f93511c3dd396 Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 13:25:54 +0200 Subject: [PATCH 01/12] blob conv test --- tests/test_convert_blob.py | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/test_convert_blob.py diff --git a/tests/test_convert_blob.py b/tests/test_convert_blob.py new file mode 100644 index 00000000..222ec515 --- /dev/null +++ b/tests/test_convert_blob.py @@ -0,0 +1,46 @@ +from quasardb import test_convert as m + +import numpy as np +import numpy.ma as ma +import quasardb + +from utils import assert_indexed_arrays_equal + + +def _blob_recode(values): + idx = np.array( + [ + np.datetime64("2017-01-01T00:00:00", "ns"), + np.datetime64("2017-01-01T00:00:01", "ns"), + np.datetime64("2017-01-01T00:00:02", "ns"), + np.datetime64("2017-01-01T00:00:03", "ns"), + ] + ) + return m.test_array_recode(quasardb.ColumnType.Blob, np.dtype("O"), (idx, values)) + + +def test_blob_object_array_recode_roundtrip_bytes(): + values = np.array([b"alpha", b"beta", b"\x00gamma", b"delta"], dtype=np.object_) + result = _blob_recode(values) + assert_indexed_arrays_equal((result[0], values), result) + + +def test_blob_object_array_recode_roundtrip_none_values(): + values = np.array([b"alpha", None, b"gamma", None], dtype=np.object_) + result = _blob_recode(values) + assert_indexed_arrays_equal((result[0], values), result) + + +def test_blob_object_array_recode_roundtrip_masked_values(): + values = ma.masked_array( + data=np.array([b"alpha", b"beta", b"gamma", b"delta"], dtype=np.object_), + mask=[False, True, False, True], + ) + result = _blob_recode(values) + assert_indexed_arrays_equal((result[0], values), result) + + +def test_blob_object_array_recode_roundtrip_empty_bytes(): + values = np.array([b"", b"beta", b"", b"delta"], dtype=np.object_) + result = _blob_recode(values) + assert_indexed_arrays_equal((result[0], values), result) From a55ed5a903b9e4a5cc1fbe20de79d1dc504af7e1 Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 13:47:22 +0200 Subject: [PATCH 02/12] fix --- quasardb/convert/value.hpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/quasardb/convert/value.hpp b/quasardb/convert/value.hpp index fb39e3ae..c3bc7878 100644 --- a/quasardb/convert/value.hpp +++ b/quasardb/convert/value.hpp @@ -573,8 +573,21 @@ struct value_converter template <> struct value_converter - : public value_converter -{}; +{ + using dtype = traits::pyobject_dtype; + + value_converter delegate_{}; + + inline qdb_blob_t operator()(py::object const & x) const + { + if (dtype::is_null(x)) + { + return traits::null_value(); + } + + return delegate_(py::cast(x)); + } +}; template <> struct value_converter From 948afbbf0f44a1a546198043e5adb455c1764fbe Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 13:56:46 +0200 Subject: [PATCH 03/12] fix 2 --- quasardb/masked_array.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/quasardb/masked_array.hpp b/quasardb/masked_array.hpp index 0de1479a..b8e4dbed 100644 --- a/quasardb/masked_array.hpp +++ b/quasardb/masked_array.hpp @@ -500,11 +500,25 @@ class masked_array template static qdb::mask masked_null(py::array const & xs) { - using value_type = typename Dtype::value_type; - py::array_t ret{ShapeContainer{xs.size()}}; bool * p_ret = static_cast(ret.mutable_data()); + if constexpr (std::is_same_v) + { + auto begin = static_cast(xs.data()); + auto end = begin + xs.size(); + + // NumPy object arrays store raw PyObject * values, not py::object wrappers. + for (auto cur = begin; cur != end; ++cur, ++p_ret) + { + *p_ret = (*cur == nullptr) || (*cur == Py_None); + } + + return mask::of_array(py::cast(ret)); + } + + using value_type = typename Dtype::value_type; + // The step_size is `1` for all fixed-width dtypes, but in case // of variable width dtypes, is, well, variable. py::ssize_t step_size = Dtype::stride_size(xs.itemsize()); From f501e5bac8442974a5a69ea50cb172ead1b9014d Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 14:05:05 +0200 Subject: [PATCH 04/12] fix 3 --- tests/test_convert.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_convert.cpp b/tests/test_convert.cpp index 6f920e4c..0f9d4ffe 100644 --- a/tests/test_convert.cpp +++ b/tests/test_convert.cpp @@ -79,7 +79,7 @@ struct array_recode_cdtype_dispatch; { \ auto tmp = convert::point_array(input); \ auto tmp2 = convert::point_array(tmp); \ - output = input; \ + output = std::move(tmp2); \ } \ }; From 3c0a781991501f2fa18a9a1a320b70321e23527b Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 14:53:41 +0200 Subject: [PATCH 05/12] fix 4 --- quasardb/convert/array.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quasardb/convert/array.hpp b/quasardb/convert/array.hpp index 3eff9515..597e664c 100644 --- a/quasardb/convert/array.hpp +++ b/quasardb/convert/array.hpp @@ -186,8 +186,8 @@ struct convert_array [[nodiscard]] constexpr inline auto operator()() const noexcept { - auto xform = [](value_type const & x) -> delegate_value_type { - if (To::is_null(x)) + auto xform = [](From const & x) -> delegate_value_type { + if (traits::is_null(x)) { return Delegate::null_value(); } From dc354a621fb8f255d64d024125938c89d942512d Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 15:53:12 +0200 Subject: [PATCH 06/12] result none --- tests/test_convert_blob.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_convert_blob.py b/tests/test_convert_blob.py index 222ec515..78cb9941 100644 --- a/tests/test_convert_blob.py +++ b/tests/test_convert_blob.py @@ -43,4 +43,8 @@ def test_blob_object_array_recode_roundtrip_masked_values(): def test_blob_object_array_recode_roundtrip_empty_bytes(): values = np.array([b"", b"beta", b"", b"delta"], dtype=np.object_) result = _blob_recode(values) - assert_indexed_arrays_equal((result[0], values), result) + expected = ma.masked_array( + data=np.array([None, b"beta", None, b"delta"], dtype=np.object_), + mask=[True, False, True, False], + ) + assert_indexed_arrays_equal((result[0], expected), result) From ab83d22a21053228672dc705e4cc7b7fcbf0bf0e Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 16:05:11 +0200 Subject: [PATCH 07/12] variant 1 --- quasardb/convert/array.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/quasardb/convert/array.hpp b/quasardb/convert/array.hpp index 597e664c..badeaa93 100644 --- a/quasardb/convert/array.hpp +++ b/quasardb/convert/array.hpp @@ -186,17 +186,17 @@ struct convert_array [[nodiscard]] constexpr inline auto operator()() const noexcept { - auto xform = [](From const & x) -> delegate_value_type { - if (traits::is_null(x)) + auto xform = [](delegate_value_type const & x) -> value_type { + if (Delegate::is_null(x)) { - return Delegate::null_value(); + return To::null_value(); } else { - return static_cast(x); + return static_cast(x); } }; - return ranges::views::transform(xform) | delegate(); + return delegate() | ranges::views::transform(xform); }; }; From 2546402fdebaeee8fed5d248cfa2cc178fe5e52c Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 16:48:29 +0200 Subject: [PATCH 08/12] fix bug 2 --- quasardb/numpy/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/quasardb/numpy/__init__.py b/quasardb/numpy/__init__.py index 5fead519..ac5449d6 100644 --- a/quasardb/numpy/__init__.py +++ b/quasardb/numpy/__init__.py @@ -170,6 +170,7 @@ def _coerce_dtype( "Forced dtype provided for column '%s' = %s, but that column is not found in the table. Skipping...", k, ) + continue i = offsets[k] dtype_[i] = dt From 2b234569f475d4af9973edfbf9601aec8903c7b1 Mon Sep 17 00:00:00 2001 From: vikonix Date: Tue, 7 Apr 2026 17:16:27 +0200 Subject: [PATCH 09/12] fix more --- quasardb/reader.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/quasardb/reader.cpp b/quasardb/reader.cpp index 421021b7..f183b1d3 100644 --- a/quasardb/reader.cpp +++ b/quasardb/reader.cpp @@ -42,6 +42,7 @@ namespace detail xs = convert::masked_array( ranges::views::counted(column.data.doubles, data.row_count)); break; + case qdb_ts_column_symbol: case qdb_ts_column_string: xs = convert::masked_array( ranges::views::counted(column.data.strings, data.row_count)); @@ -55,13 +56,6 @@ namespace detail ranges::views::counted(column.data.timestamps, data.row_count)); break; - case qdb_ts_column_symbol: - // This should not happen, as "symbol" is just an internal representation, and symbols - // are exposed to the user as strings. If this actually happens, it indicates either - // a bug in the bulk reader *or* a memory corruption. - throw qdb::not_implemented_exception( - "Internal error: invalid data type: symbol column type returned from bulk reader"); - case qdb_ts_column_uninitialized: throw qdb::not_implemented_exception( "Internal error: invalid data type: uninitialized column " From e0bba12f7216715ed5da08700e0d0bd7fb6379f9 Mon Sep 17 00:00:00 2001 From: vikonix Date: Wed, 8 Apr 2026 20:04:18 +0200 Subject: [PATCH 10/12] fix symbols --- quasardb/reader.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/quasardb/reader.cpp b/quasardb/reader.cpp index f183b1d3..421021b7 100644 --- a/quasardb/reader.cpp +++ b/quasardb/reader.cpp @@ -42,7 +42,6 @@ namespace detail xs = convert::masked_array( ranges::views::counted(column.data.doubles, data.row_count)); break; - case qdb_ts_column_symbol: case qdb_ts_column_string: xs = convert::masked_array( ranges::views::counted(column.data.strings, data.row_count)); @@ -56,6 +55,13 @@ namespace detail ranges::views::counted(column.data.timestamps, data.row_count)); break; + case qdb_ts_column_symbol: + // This should not happen, as "symbol" is just an internal representation, and symbols + // are exposed to the user as strings. If this actually happens, it indicates either + // a bug in the bulk reader *or* a memory corruption. + throw qdb::not_implemented_exception( + "Internal error: invalid data type: symbol column type returned from bulk reader"); + case qdb_ts_column_uninitialized: throw qdb::not_implemented_exception( "Internal error: invalid data type: uninitialized column " From 9d590290472a2077662834401df540099d701a22 Mon Sep 17 00:00:00 2001 From: vikonix Date: Wed, 8 Apr 2026 23:03:55 +0200 Subject: [PATCH 11/12] check revert 1 --- quasardb/convert/value.hpp | 6 ++++++ quasardb/masked_array.hpp | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/quasardb/convert/value.hpp b/quasardb/convert/value.hpp index c3bc7878..1722bccb 100644 --- a/quasardb/convert/value.hpp +++ b/quasardb/convert/value.hpp @@ -571,7 +571,12 @@ struct value_converter } }; +#if 1 template <> +struct value_converter + : public value_converter +{}; +#else template <> struct value_converter { using dtype = traits::pyobject_dtype; @@ -588,6 +593,7 @@ struct value_converter return delegate_(py::cast(x)); } }; +#endif template <> struct value_converter diff --git a/quasardb/masked_array.hpp b/quasardb/masked_array.hpp index b8e4dbed..71623d3b 100644 --- a/quasardb/masked_array.hpp +++ b/quasardb/masked_array.hpp @@ -500,9 +500,12 @@ class masked_array template static qdb::mask masked_null(py::array const & xs) { + using value_type = typename Dtype::value_type; + py::array_t ret{ShapeContainer{xs.size()}}; bool * p_ret = static_cast(ret.mutable_data()); +#if 0 if constexpr (std::is_same_v) { auto begin = static_cast(xs.data()); @@ -516,8 +519,7 @@ class masked_array return mask::of_array(py::cast(ret)); } - - using value_type = typename Dtype::value_type; +#endif // The step_size is `1` for all fixed-width dtypes, but in case // of variable width dtypes, is, well, variable. From cbdb5c629cdd28d881d956412b955e410dd086a1 Mon Sep 17 00:00:00 2001 From: vikonix Date: Wed, 8 Apr 2026 23:26:14 +0200 Subject: [PATCH 12/12] check revert 2 --- quasardb/convert/array.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/quasardb/convert/array.hpp b/quasardb/convert/array.hpp index badeaa93..bc365b6b 100644 --- a/quasardb/convert/array.hpp +++ b/quasardb/convert/array.hpp @@ -184,6 +184,22 @@ struct convert_array static constexpr convert_array const delegate{}; +#if 1 + [[nodiscard]] constexpr inline auto operator()() const noexcept + { + auto xform = [](value_type const & x) -> delegate_value_type { + if (To::is_null(x)) + { + return Delegate::null_value(); + } + else + { + return static_cast(x); + } + }; + return ranges::views::transform(xform) | delegate(); + }; +#else [[nodiscard]] constexpr inline auto operator()() const noexcept { auto xform = [](delegate_value_type const & x) -> value_type { @@ -198,6 +214,7 @@ struct convert_array }; return delegate() | ranges::views::transform(xform); }; +#endif }; }; // namespace qdb::convert::detail