From 69aa64615488d4a41e09108a8b8ccc773dc9c9d6 Mon Sep 17 00:00:00 2001 From: Narek Galstyan Date: Sat, 5 Aug 2023 19:34:14 +0000 Subject: [PATCH 1/6] rename pgvector's vector.h to pgvector_vector.h --- src/hnsw/build.c | 2 +- src/hnsw/insert.c | 2 +- src/hnsw/{vector.h => pgvector_vector.h} | 0 src/hnsw/scan.c | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename src/hnsw/{vector.h => pgvector_vector.h} (100%) diff --git a/src/hnsw/build.c b/src/hnsw/build.c index b2d3ea7e2..21440110b 100644 --- a/src/hnsw/build.c +++ b/src/hnsw/build.c @@ -15,9 +15,9 @@ #include "bench.h" #include "external_index.h" #include "hnsw.h" +#include "pgvector_vector.h" #include "usearch.h" #include "utils.h" -#include "vector.h" #if PG_VERSION_NUM >= 140000 #include "utils/backend_progress.h" diff --git a/src/hnsw/insert.c b/src/hnsw/insert.c index f63292ce9..327ef04a5 100644 --- a/src/hnsw/insert.c +++ b/src/hnsw/insert.c @@ -14,9 +14,9 @@ #include "external_index.h" #include "hnsw.h" #include "options.h" +#include "pgvector_vector.h" #include "usearch.h" #include "utils.h" -#include "vector.h" /* * Context delete callback for insert context diff --git a/src/hnsw/vector.h b/src/hnsw/pgvector_vector.h similarity index 100% rename from src/hnsw/vector.h rename to src/hnsw/pgvector_vector.h diff --git a/src/hnsw/scan.c b/src/hnsw/scan.c index 3bcbcfec1..3902f2e58 100644 --- a/src/hnsw/scan.c +++ b/src/hnsw/scan.c @@ -10,7 +10,7 @@ #include "external_index.h" #include "hnsw.h" #include "options.h" -#include "vector.h" +#include "pgvector_vector.h" PG_MODULE_MAGIC; From 5b1dbe1612275fbbe76ad4724698b6917d8440fa Mon Sep 17 00:00:00 2001 From: Narek Galstyan Date: Sun, 6 Aug 2023 06:16:12 +0000 Subject: [PATCH 2/6] Add casts to/from array<->vec32<->uvec8 --- sql/lanterndb.sql | 40 ++++++ src/vec_type.h | 41 ++++++ src/vector_casts.c | 338 +++++++++++++++++++++++++++++++++++++++++++++ src/vector_casts.h | 25 ++++ 4 files changed, 444 insertions(+) create mode 100644 src/vec_type.h create mode 100644 src/vector_casts.c create mode 100644 src/vector_casts.h diff --git a/sql/lanterndb.sql b/sql/lanterndb.sql index 8f6cf9e0c..f74ca4ed6 100644 --- a/sql/lanterndb.sql +++ b/sql/lanterndb.sql @@ -50,3 +50,43 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- -- CREATE TYPEs vec* + +-- Function that are generic over the family of vec types + +CREATE FUNCTION ldb_generic_vec_typmod_in(cstring[]) RETURNS integer + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +-- 8-byte unit-vector (i.e. vector with elements in range [-1, 1]) +CREATE TYPE uvec8; + +CREATE FUNCTION ldb_uvec8_in(cstring, oid, integer) RETURNS uvec8 + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +CREATE FUNCTION ldb_uvec8_out(uvec8) RETURNS cstring + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +CREATE FUNCTION ldb_uvec8_recv(internal, oid, integer) RETURNS uvec8 + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +CREATE FUNCTION ldb_uvec8_send(uvec8) RETURNS bytea + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + + +CREATE TYPE uvec8 ( + INPUT = ldb_uvec8_in, + OUTPUT = ldb_uvec8_out, + RECEIVE = ldb_uvec8_recv, + SEND = ldb_uvec8_send, + TYPMOD_IN = ldb_generic_vec_typmod_in, + STORAGE = extended +); + +-- -- Unit ector which has 1 byte elements in range [-1, 1] +CREATE FUNCTION ldb_cast_uvec8_uvec8(uvec8, integer, boolean) RETURNS uvec8 + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +-- casts + +CREATE CAST (uvec8 AS uvec8) + WITH FUNCTION ldb_cast_uvec8_uvec8(uvec8, integer, boolean) AS IMPLICIT; \ No newline at end of file diff --git a/src/vec_type.h b/src/vec_type.h new file mode 100644 index 000000000..4714f5451 --- /dev/null +++ b/src/vec_type.h @@ -0,0 +1,41 @@ +#ifndef LDB_VEC_H +#define LDB_VEC_H +#include + +typedef struct +{ + int32 vl_len_; /* varlena header (do not touch directly!) */ + uint16 dim; /* number of dimensions */ + uint16 elem_size; + char data[ FLEXIBLE_ARRAY_MEMBER ]; +} LDBVec; + +static inline LDBVec *NewLDBVec(int dim, int elem_size) +{ + LDBVec *result; + int size; + + size = sizeof(LDBVec) + dim * elem_size; + result = (LDBVec *)palloc0(size); + SET_VARSIZE(result, size); + result->dim = dim; + result->elem_size = elem_size; + + return result; +} + +/* Confined by uint16 in LDBVec structure */ +#define LDB_VEC_MAX_DIM 1 << 16 +/* + * Returns a pointer to the actual array data. + */ +#define LDBVEC_DATA_SIZE(a) (((a->dim)) * (a->elem_size)) + +/* + * Returns a pointer to the actual array data. + */ +#define LDBVEC_DATA_PTR(a) (((void *)(a->data))) + +#define DatumGetLDBVec(x) ((LDBVec *)PG_DETOAST_DATUM(x)) + +#endif // LDB_VEC_H \ No newline at end of file diff --git a/src/vector_casts.c b/src/vector_casts.c new file mode 100644 index 000000000..4397436e8 --- /dev/null +++ b/src/vector_casts.c @@ -0,0 +1,338 @@ +#include + +#include "vector_casts.h" + +#include +#include +#include +#include +#include +#include // for get_typlenbyvalalign + +#include "usearch.h" +#include "vec_type.h" + +/*** Functions generic over all vec types (uvec8, vec8, vec16, vec32) */ + +ArrayType *ldb_generic_vec_in(PG_FUNCTION_ARGS) +{ + char *str = PG_GETARG_CSTRING(0); + + // The second argument is the type OID of our newly created type + // we do not need that. For arrays, it would be the element type + // and not the array type. It would be good to have this be the + // element type for our vec types as well. But I do not think that + // is possible for custom types + // We always read the passed array element type as FLOAT4OID + // In the future, we may want to have vec64 (but unlikely, since + // postgres's own double precision works just fine) + int32 typioparam = FLOAT4OID; + + // represents the number of dimensions in the vec type: '{1,1,1}'::vec(3) + // ....^... + int32 typmod = PG_GETARG_INT32(2); + + Oid oid = fmgr_internal_function("array_in"); + ArrayType *array; + + array = (ArrayType *)DatumGetPointer(OidInputFunctionCall(oid, str, typioparam, typmod)); + // postgres does not enforce the array bounds(cite?), but we want to enforce them + /* todo:: this is a possible potential bug (or atleast very surprising behavior). + * Say we create a type and have typemod_in set up for it and provide a type for it + * e.g. '{1,2,3}::vec8(3)' for us. + * Postgres calls typemod_in and validates the '3' parameter before calling this function + * BUT, postgres does not pass typmod returned by the typmod function to this function + * as its third argument. + * So, the branch below is not triggered for the case above: even if we had passed '4' as + * type parameter, the validation below would not trigger + * Validation currently is still triggered through the cast function (cast from vec8 to vec8) + * so UX is not affected but we should probably report this or talk to postgres devs + * to see if this is expected behaviour. + */ + if(typmod != -1 && ARR_NDIM(array) != typmod) { + elog(ERROR, "vec type expected %d dimensions but provided array has %d dimensions", typmod, ARR_NDIM(array)); + } + return array; +} + +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_generic_vec_typmod_in); +/* + * Provides Type modifier with semantics similar to postgres arrays + * The only difference is that we actually enforce these constraints + * '{1,2,3}'::int[4] is valid but '{1,2,3}'::vec8(4) is not + * Postgres Docs: The type_modifier_input_function is passed the + * declared modifier(s) in the form of a cstring array. + * It must check the values for validity (throwing an error if they are wrong), + * and if they are correct, return a single non-negative integer value that + * will be stored as the column “typmod” + */ +Datum ldb_generic_vec_typmod_in(PG_FUNCTION_ARGS) +{ + ArrayType *ta = PG_GETARG_ARRAYTYPE_P(0); + int32 *tl; + int n; + + tl = ArrayGetIntegerTypmods(ta, &n); + + if(n != 1) { + elog(ERROR, "wrong number of modifiers"); + } + + if(*tl < 1) { + elog(ERROR, "vector dimensions must be at least 1"); + } + + if(*tl > LDB_VEC_MAX_DIM) { + elog(ERROR, "vector dimensions must be at most %d", LDB_VEC_MAX_DIM); + } + + return Int32GetDatum(*tl); +} + +Datum ldb_generic_vec_out(ArrayType *arr) +{ + Oid oid = fmgr_internal_function("array_out"); + char *res = OidOutputFunctionCall(oid, PointerGetDatum(arr)); + return CStringGetDatum(res); +} + +/* + * Convert LDB's 'vec*' external binary representation to the internal representation + */ +Datum ldb_generic_vec_recv(PG_FUNCTION_ARGS) +{ + StringInfo buf = (StringInfo)PG_GETARG_POINTER(0); + int32 typmod = PG_GETARG_INT32(2); + LDBVec *result; + uint16 dim; + uint16 elem_size; + elog(ERROR, "vec recv called"); + + dim = pq_getmsgint(buf, sizeof(uint16)); + elem_size = pq_getmsgint(buf, sizeof(uint16)); + + if(dim < 1 || dim > LDB_VEC_MAX_DIM) { + elog(ERROR, "received binary vec with invalid invalid dimension %d", dim); + } + + if(typmod != -1 && dim != typmod) { + elog(ERROR, "received binary vec with wrong dimension %d, expected %d", dim, typmod); + } + + result = NewLDBVec(dim, elem_size); + pq_copymsgbytes(buf, result->data, elem_size * dim); + // todo:: validate that the copy succeeded and result is not corrupted + PG_RETURN_POINTER(result); +} + +/* + * Convert LDB's 'vec*' internal representation to the external binary representation + */ +Datum ldb_generic_vec_send(PG_FUNCTION_ARGS) +{ + StringInfoData buf; + LDBVec *vec = DatumGetLDBVec(PG_GETARG_DATUM(0)); + + elog(ERROR, "vec send called"); + pq_begintypsend(&buf); + pq_sendint(&buf, vec->dim, sizeof(uint16)); + pq_sendint(&buf, vec->elem_size, sizeof(uint16)); + pq_sendbytes(&buf, vec->data, vec->elem_size * vec->dim); + PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); +} + +/******************************* UVEC8 ******************************/ + +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_uvec8_in); + +Datum ldb_uvec8_in(PG_FUNCTION_ARGS) +{ + LDBVec *res; + usearch_error_t error = NULL; + // fcinfo below is from the macro PG_FUNCTION_ARGS and passes SQL-arguments + // to the generic vec reader + ArrayType *array = ldb_generic_vec_in(fcinfo); + assert(array != NULL); + // if the element type ever changes (e.g. to FLOAT8OID), + // the usearch_cast call must be reworked. + // on 64bit systems FLOAT8OID fits inline in a Datum BUT on 32 bit systems it + // does not so when casting from FLOAT8 array, I can no longer assume + // ARR_DATA_PTR(array) is an array of element values (there will be another + // indirection via a pointer) + assert(array->elemtype == FLOAT4OID); + int ndims = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); + res = ldb_cast_array_vec(array, ndims, usearch_scalar_f8_k); + // in vec8, each vector element is one byte, hense the 1 + // res = NewLDBVec(ndims, 1); + // float *array_elems = (float *)ARR_DATA_PTR(array); + + if(error) { + elog(ERROR, "error in float downcasting: %s", error); + } + + return PointerGetDatum(res); +} + +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_uvec8_out); + +Datum ldb_uvec8_out(PG_FUNCTION_ARGS) +{ + ArrayType *res; + float *array_elems; + Datum *array_elems_datum; + int array_elems_size; + usearch_error_t error = NULL; + LDBVec *vec = DatumGetLDBVec(PG_GETARG_DATUM(0)); + assert(vec->dim != 0); + assert(vec->elem_size == 1); + array_elems_size = sizeof(float) * vec->dim; + array_elems = palloc0(array_elems_size); + array_elems_datum = palloc0(sizeof(Datum) * vec->dim); + usearch_cast(usearch_scalar_f8_k, vec->data, usearch_scalar_f32_k, array_elems, array_elems_size, vec->dim, &error); + if(error != NULL) { + elog(ERROR, "error casting: %s", error); + } + + for(int i = 0; i < vec->dim; i++) { + array_elems_datum[ i ] = Float4GetDatum(array_elems[ i ]); + } + + res = construct_array(array_elems_datum, vec->dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT); + assert(res != NULL); + + assert(res->elemtype == FLOAT4OID); + int ndims = ArrayGetNItems(ARR_NDIM(res), ARR_DIMS(res)); + assert(ndims = vec->dim); + + return ldb_generic_vec_out(res); +} + +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_uvec8_recv); +Datum ldb_uvec8_recv(PG_FUNCTION_ARGS) { return ldb_generic_vec_recv(fcinfo); } + +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_uvec8_send); +Datum ldb_uvec8_send(PG_FUNCTION_ARGS) { return ldb_generic_vec_send(fcinfo); } + +// PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector); +// Datum +// array_to_vec8(PG_FUNCTION_ARGS){} + +/************ CASTS *************/ +/* + * Casts for type vec* to type vec* + * Note: these functions are called in callsites like '{1,2,3}'::vec8(4) + * The call to this function is what throws the dimension mismatch error + * vect*_in calls have a typmod parameter but the parsed typmod result is not + * passed to them. + */ + +// dest must be provided iff from != to and it must be initialized +// and sized appropriately for 'to' +Datum ldb_cast_vec_vec(FunctionCallInfo fcinfo, int from, int to, LDBVec *dest, int ddata_size) +{ + LDBVec *src = DatumGetLDBVec(PG_GETARG_DATUM(0)); + int32 typmod = PG_GETARG_INT32(1); + usearch_error_t error = NULL; + assert((from == to) ^ (dest != NULL)); + + if(typmod != -1 && src->dim != typmod) { + elog(ERROR, "vector cast wrong dimension %d, expected %d", src->dim, typmod); + } + + if(from != to) { + assert(src->dim == dest->dim); + usearch_cast(from, src->data, to, dest->data, ddata_size, src->dim, &error); + // todo:: q:: who manages the memory of the src? + // pretty sure the caller must deal with it but would be good to know for sure from code + // or documentation + return PointerGetDatum(dest); + } + + return PointerGetDatum(src); +} + +// clang-format off +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_cast_uvec8_uvec8); +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_cast_vec32_vec32); + +Datum ldb_cast_uvec8_uvec8(PG_FUNCTION_ARGS) { return ldb_cast_vec_vec(fcinfo, usearch_scalar_f8_k, usearch_scalar_f8_k, NULL, 0); } +Datum ldb_cast_vec32_vec32(PG_FUNCTION_ARGS) { return ldb_cast_vec_vec(fcinfo, usearch_scalar_f32_k, usearch_scalar_f32_k, NULL, 0); } +// clang-format on + +// if expected_dim is -1, then the dimension is not checked +// otherwise, an error is thrown if the dimension is not as expected +LDBVec *ldb_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_kind_t to) +{ + usearch_error_t error = NULL; + + LDBVec *result; + int16 typlen; + bool typbyval; + char typalign; + Datum *elemsp; + bool *nullsp; + int nelemsp; + + if(ARR_NDIM(array) > 1) { + elog(ERROR, "array must be 1-D"); + } + + if(ARR_HASNULL(array) && array_contains_nulls(array)) { + elog(ERROR, "array must not contain nulls"); + } + + get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign); + deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, &nullsp, &nelemsp); + if(nelemsp == 0) { + elog(ERROR, "array must not be empty"); + } + + if(nelemsp > LDB_VEC_MAX_DIM) { + elog(ERROR, "array too large. max vec dimension is %d", LDB_VEC_MAX_DIM); + } + + if(expected_dim != -1 && nelemsp != expected_dim) { + elog(ERROR, "array has wrong dimension %d, expected %d", nelemsp, expected_dim); + } + + result = NewLDBVec(nelemsp, scalar_size(to)); + // rsult->data is usually opeque to us and handled by usearch + // since it contains types like float16, float8, ufloat8, which are not available in C. + // here, we init it with 32-bit floats which are the same here and in usearch + // so this is fine + float *vec_floats = (float *)palloc0(nelemsp * sizeof(float)); + + switch(ARR_ELEMTYPE(array)) { + case INT4OID: + for(int i = 0; i < nelemsp; i++) { + vec_floats[ i ] = DatumGetInt32(elemsp[ i ]); + } + break; + case FLOAT4OID: + for(int i = 0; i < nelemsp; i++) { + vec_floats[ i ] = DatumGetFloat4(elemsp[ i ]); + } + break; + case FLOAT8OID: + case NUMERICOID: + // todo:: x, FLOAT8, NUMERIC + elog(ERROR, "unimplemented cast to vec type"); + break; + default: + elog(ERROR, "unknown array elem type %d", ARR_ELEMTYPE(array)); + } + + if (usearch_scalar_f8_k == to) { + // sanity check and throw an error if uvec8 is used with floats outside of [-1, 1] + for(int i = 0; i < nelemsp; i++) { + if(vec_floats[ i ] < -1 || vec_floats[ i ] > 1) { + elog(ERROR, "uvec8 must be in range [-1, 1]"); + } + } + } + + usearch_cast( + usearch_scalar_f32_k, vec_floats, to, LDBVEC_DATA_PTR(result), result->dim, LDBVEC_DATA_SIZE(result), &error); + + return result; +} \ No newline at end of file diff --git a/src/vector_casts.h b/src/vector_casts.h new file mode 100644 index 000000000..ee53b3665 --- /dev/null +++ b/src/vector_casts.h @@ -0,0 +1,25 @@ +#ifndef LDB_VECTOR_CASTS_H +#define LDB_VECTOR_CASTS_H + +#include +#include + +#include "usearch.h" +#include "vec_type.h" + +static inline int scalar_size(usearch_scalar_kind_t s) +{ + switch(s) { + // clang-format off + case usearch_scalar_f64_k: return 8; + case usearch_scalar_f32_k: return 4; + case usearch_scalar_f16_k: return 2; + case usearch_scalar_f8_k: return 1; + case usearch_scalar_b1_k: return 1; + // clang-format on + } + assert(false); +} + +LDBVec *ldb_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_kind_t to); +#endif // LDB_VECTOR_CASTS_H \ No newline at end of file From 8a417b3cd5c34c7306a527c86a40c4e7fa93775d Mon Sep 17 00:00:00 2001 From: Narek Galstyan Date: Sun, 6 Aug 2023 08:17:59 +0000 Subject: [PATCH 3/6] Add more casts --- sql/lanterndb.sql | 31 +++++++++++- src/vec_type.h | 24 +++++++-- src/vector_casts.c | 124 +++++++++++++++++++++++++++++++-------------- src/vector_casts.h | 20 +++----- 4 files changed, 142 insertions(+), 57 deletions(-) diff --git a/sql/lanterndb.sql b/sql/lanterndb.sql index f74ca4ed6..5828e6209 100644 --- a/sql/lanterndb.sql +++ b/sql/lanterndb.sql @@ -85,8 +85,37 @@ CREATE TYPE uvec8 ( -- -- Unit ector which has 1 byte elements in range [-1, 1] CREATE FUNCTION ldb_cast_uvec8_uvec8(uvec8, integer, boolean) RETURNS uvec8 AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_cast_array_uvec8(integer[], integer, boolean) RETURNS uvec8 + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_cast_array_uvec8(real[], integer, boolean) RETURNS uvec8 + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_cast_array_uvec8(double precision[], integer, boolean) RETURNS uvec8 + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_cast_array_uvec8(numeric[], integer, boolean) RETURNS uvec8 + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +CREATE FUNCTION ldb_cast_vec_real(uvec8, integer, boolean) RETURNS real[] + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- casts CREATE CAST (uvec8 AS uvec8) - WITH FUNCTION ldb_cast_uvec8_uvec8(uvec8, integer, boolean) AS IMPLICIT; \ No newline at end of file + WITH FUNCTION ldb_cast_uvec8_uvec8(uvec8, integer, boolean) AS IMPLICIT; + +CREATE CAST (integer[] AS uvec8) + WITH FUNCTION ldb_cast_array_uvec8(integer[], integer, boolean) AS ASSIGNMENT; + +CREATE CAST (real[] AS uvec8) + WITH FUNCTION ldb_cast_array_uvec8(real[], integer, boolean) AS ASSIGNMENT; + +CREATE CAST (double precision[] AS uvec8) + WITH FUNCTION ldb_cast_array_uvec8(double precision[], integer, boolean) AS ASSIGNMENT; + +CREATE CAST (numeric[] AS uvec8) + WITH FUNCTION ldb_cast_array_uvec8(numeric[], integer, boolean) AS ASSIGNMENT; + +CREATE CAST (uvec8 AS real[]) + WITH FUNCTION ldb_cast_vec_real(uvec8, integer, boolean) AS ASSIGNMENT; + + + diff --git a/src/vec_type.h b/src/vec_type.h index 4714f5451..06d4d1853 100644 --- a/src/vec_type.h +++ b/src/vec_type.h @@ -6,20 +6,34 @@ typedef struct { int32 vl_len_; /* varlena header (do not touch directly!) */ uint16 dim; /* number of dimensions */ - uint16 elem_size; + uint16 elem_type; char data[ FLEXIBLE_ARRAY_MEMBER ]; } LDBVec; -static inline LDBVec *NewLDBVec(int dim, int elem_size) +static inline int VecScalarSize(usearch_scalar_kind_t s) +{ + switch(s) { + // clang-format off + case usearch_scalar_f64_k: return 8; + case usearch_scalar_f32_k: return 4; + case usearch_scalar_f16_k: return 2; + case usearch_scalar_f8_k: return 1; + case usearch_scalar_b1_k: return 1; + // clang-format on + } + assert(false); +} + +static inline LDBVec *NewLDBVec(int dim, int elem_type) { LDBVec *result; int size; - size = sizeof(LDBVec) + dim * elem_size; + size = sizeof(LDBVec) + dim * VecScalarSize(elem_type); result = (LDBVec *)palloc0(size); SET_VARSIZE(result, size); result->dim = dim; - result->elem_size = elem_size; + result->elem_type = elem_type; return result; } @@ -29,7 +43,7 @@ static inline LDBVec *NewLDBVec(int dim, int elem_size) /* * Returns a pointer to the actual array data. */ -#define LDBVEC_DATA_SIZE(a) (((a->dim)) * (a->elem_size)) +#define LDBVEC_DATA_SIZE(a) (((a->dim)) * (VecScalarSize(a->elem_type))) /* * Returns a pointer to the actual array data. diff --git a/src/vector_casts.c b/src/vector_casts.c index 4397436e8..346cb21c1 100644 --- a/src/vector_casts.c +++ b/src/vector_casts.c @@ -7,6 +7,7 @@ #include #include #include +#include #include // for get_typlenbyvalalign #include "usearch.h" @@ -105,11 +106,11 @@ Datum ldb_generic_vec_recv(PG_FUNCTION_ARGS) int32 typmod = PG_GETARG_INT32(2); LDBVec *result; uint16 dim; - uint16 elem_size; + uint16 elem_type; elog(ERROR, "vec recv called"); dim = pq_getmsgint(buf, sizeof(uint16)); - elem_size = pq_getmsgint(buf, sizeof(uint16)); + elem_type = pq_getmsgint(buf, sizeof(uint16)); if(dim < 1 || dim > LDB_VEC_MAX_DIM) { elog(ERROR, "received binary vec with invalid invalid dimension %d", dim); @@ -119,8 +120,8 @@ Datum ldb_generic_vec_recv(PG_FUNCTION_ARGS) elog(ERROR, "received binary vec with wrong dimension %d, expected %d", dim, typmod); } - result = NewLDBVec(dim, elem_size); - pq_copymsgbytes(buf, result->data, elem_size * dim); + result = NewLDBVec(dim, elem_type); + pq_copymsgbytes(buf, result->data, elem_type * dim); // todo:: validate that the copy succeeded and result is not corrupted PG_RETURN_POINTER(result); } @@ -136,8 +137,8 @@ Datum ldb_generic_vec_send(PG_FUNCTION_ARGS) elog(ERROR, "vec send called"); pq_begintypsend(&buf); pq_sendint(&buf, vec->dim, sizeof(uint16)); - pq_sendint(&buf, vec->elem_size, sizeof(uint16)); - pq_sendbytes(&buf, vec->data, vec->elem_size * vec->dim); + pq_sendint(&buf, vec->elem_type, sizeof(uint16)); + pq_sendbytes(&buf, vec->data, vec->elem_type * vec->dim); PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } @@ -153,18 +154,10 @@ Datum ldb_uvec8_in(PG_FUNCTION_ARGS) // to the generic vec reader ArrayType *array = ldb_generic_vec_in(fcinfo); assert(array != NULL); - // if the element type ever changes (e.g. to FLOAT8OID), - // the usearch_cast call must be reworked. - // on 64bit systems FLOAT8OID fits inline in a Datum BUT on 32 bit systems it - // does not so when casting from FLOAT8 array, I can no longer assume - // ARR_DATA_PTR(array) is an array of element values (there will be another - // indirection via a pointer) + assert(array->elemtype == FLOAT4OID); int ndims = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); - res = ldb_cast_array_vec(array, ndims, usearch_scalar_f8_k); - // in vec8, each vector element is one byte, hense the 1 - // res = NewLDBVec(ndims, 1); - // float *array_elems = (float *)ARR_DATA_PTR(array); + res = ldb_generic_cast_array_vec(array, ndims, usearch_scalar_f8_k); if(error) { elog(ERROR, "error in float downcasting: %s", error); @@ -184,25 +177,16 @@ Datum ldb_uvec8_out(PG_FUNCTION_ARGS) usearch_error_t error = NULL; LDBVec *vec = DatumGetLDBVec(PG_GETARG_DATUM(0)); assert(vec->dim != 0); - assert(vec->elem_size == 1); + assert(vec->elem_type == usearch_scalar_f8_k); array_elems_size = sizeof(float) * vec->dim; array_elems = palloc0(array_elems_size); - array_elems_datum = palloc0(sizeof(Datum) * vec->dim); + usearch_cast(usearch_scalar_f8_k, vec->data, usearch_scalar_f32_k, array_elems, array_elems_size, vec->dim, &error); if(error != NULL) { elog(ERROR, "error casting: %s", error); } - for(int i = 0; i < vec->dim; i++) { - array_elems_datum[ i ] = Float4GetDatum(array_elems[ i ]); - } - - res = construct_array(array_elems_datum, vec->dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT); - assert(res != NULL); - - assert(res->elemtype == FLOAT4OID); - int ndims = ArrayGetNItems(ARR_NDIM(res), ARR_DIMS(res)); - assert(ndims = vec->dim); + res = ldb_generic_cast_vec_array(array_elems, vec->dim); return ldb_generic_vec_out(res); } @@ -235,9 +219,7 @@ Datum ldb_cast_vec_vec(FunctionCallInfo fcinfo, int from, int to, LDBVec *dest, usearch_error_t error = NULL; assert((from == to) ^ (dest != NULL)); - if(typmod != -1 && src->dim != typmod) { - elog(ERROR, "vector cast wrong dimension %d, expected %d", src->dim, typmod); - } + CheckVecDimConstraint(src->dim, typmod); if(from != to) { assert(src->dim == dest->dim); @@ -259,9 +241,47 @@ Datum ldb_cast_uvec8_uvec8(PG_FUNCTION_ARGS) { return ldb_cast_vec_vec(fcinfo, u Datum ldb_cast_vec32_vec32(PG_FUNCTION_ARGS) { return ldb_cast_vec_vec(fcinfo, usearch_scalar_f32_k, usearch_scalar_f32_k, NULL, 0); } // clang-format on +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_cast_array_uvec8); + +Datum ldb_cast_array_uvec8(PG_FUNCTION_ARGS) +{ + ArrayType *array = PG_GETARG_ARRAYTYPE_P(0); + int32 typmod = PG_GETARG_INT32(1); + LDBVec *res = ldb_generic_cast_array_vec(array, -1, usearch_scalar_f8_k); + CheckVecDimConstraint(res->dim, typmod); + return PointerGetDatum(ldb_generic_cast_array_vec(array, -1, usearch_scalar_f8_k)); +} + +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_cast_vec_real); + +Datum ldb_cast_vec_real(PG_FUNCTION_ARGS) +{ + LDBVec *vec = DatumGetLDBVec(PG_GETARG_DATUM(0)); + int32 typmod = PG_GETARG_INT32(1); + float *array_elems; + CheckVecDimConstraint(vec->dim, typmod); + if(vec->elem_type == usearch_scalar_f32_k) { + array_elems = (float *)vec->data; + } else { + usearch_error_t error = NULL; + LDBVec *new_vec = NewLDBVec(vec->dim, usearch_scalar_f32_k); + usearch_cast(vec->elem_type, + vec->data, + usearch_scalar_f32_k, + new_vec->data, + vec->dim * VecScalarSize(usearch_scalar_f32_k), + vec->dim, + &error); + assert(!error); + array_elems = (float *)new_vec->data; + } + ArrayType *res = ldb_generic_cast_vec_array(array_elems, vec->dim); + return PointerGetDatum(res); +} + // if expected_dim is -1, then the dimension is not checked // otherwise, an error is thrown if the dimension is not as expected -LDBVec *ldb_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_kind_t to) +LDBVec *ldb_generic_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_kind_t to) { usearch_error_t error = NULL; @@ -295,7 +315,14 @@ LDBVec *ldb_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_ki elog(ERROR, "array has wrong dimension %d, expected %d", nelemsp, expected_dim); } - result = NewLDBVec(nelemsp, scalar_size(to)); + if(VecScalarSize(to) > 4) { + // would anyone need this? + // I can just allocate float8 for vec_floats and support this + // but am not sure it is necessary + elog(ERROR, "larger than 4byte element sizes not supported"); + } + + result = NewLDBVec(nelemsp, to); // rsult->data is usually opeque to us and handled by usearch // since it contains types like float16, float8, ufloat8, which are not available in C. // here, we init it with 32-bit floats which are the same here and in usearch @@ -314,15 +341,19 @@ LDBVec *ldb_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_ki } break; case FLOAT8OID: - case NUMERICOID: - // todo:: x, FLOAT8, NUMERIC - elog(ERROR, "unimplemented cast to vec type"); + for(int i = 0; i < nelemsp; i++) { + vec_floats[ i ] = (float)DatumGetFloat8(elemsp[ i ]); + } break; + case NUMERICOID: + for(int i = 0; i < nelemsp; i++) { + vec_floats[ i ] = DatumGetFloat4(DirectFunctionCall1(numeric_float4, elemsp[ i ])); + } default: elog(ERROR, "unknown array elem type %d", ARR_ELEMTYPE(array)); } - if (usearch_scalar_f8_k == to) { + if(usearch_scalar_f8_k == to) { // sanity check and throw an error if uvec8 is used with floats outside of [-1, 1] for(int i = 0; i < nelemsp; i++) { if(vec_floats[ i ] < -1 || vec_floats[ i ] > 1) { @@ -335,4 +366,21 @@ LDBVec *ldb_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_ki usearch_scalar_f32_k, vec_floats, to, LDBVEC_DATA_PTR(result), result->dim, LDBVEC_DATA_SIZE(result), &error); return result; -} \ No newline at end of file +} + +ArrayType *ldb_generic_cast_vec_array(float *array_elems, int dim) +{ + ArrayType *res; + Datum *array_elems_datum = palloc0(sizeof(Datum) * dim); + + for(int i = 0; i < dim; i++) { + array_elems_datum[ i ] = Float4GetDatum(array_elems[ i ]); + } + res = construct_array(array_elems_datum, dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT); + assert(res != NULL); + + assert(res->elemtype == FLOAT4OID); + int ndims = ArrayGetNItems(ARR_NDIM(res), ARR_DIMS(res)); + assert(ndims = dim); + return res; +} diff --git a/src/vector_casts.h b/src/vector_casts.h index ee53b3665..568ad9f75 100644 --- a/src/vector_casts.h +++ b/src/vector_casts.h @@ -1,25 +1,19 @@ #ifndef LDB_VECTOR_CASTS_H #define LDB_VECTOR_CASTS_H -#include #include +#include #include "usearch.h" #include "vec_type.h" -static inline int scalar_size(usearch_scalar_kind_t s) +static inline void CheckVecDimConstraint(int dim, int cast) { - switch(s) { - // clang-format off - case usearch_scalar_f64_k: return 8; - case usearch_scalar_f32_k: return 4; - case usearch_scalar_f16_k: return 2; - case usearch_scalar_f8_k: return 1; - case usearch_scalar_b1_k: return 1; - // clang-format on + if(cast != -1 && dim != cast) { + elog(ERROR, "invalid cast. vector dim: %d, cast dim:%d", dim, cast); } - assert(false); } -LDBVec *ldb_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_kind_t to); -#endif // LDB_VECTOR_CASTS_H \ No newline at end of file +ArrayType *ldb_generic_cast_vec_array(float *array_elems, int dim); +LDBVec *ldb_generic_cast_array_vec(ArrayType *array, int expected_dim, usearch_scalar_kind_t to); +#endif // LDB_VECTOR_CASTS_H From a656f8fe20536aa2a2196535515ef30108e91351 Mon Sep 17 00:00:00 2001 From: Narek Galstyan Date: Sun, 6 Aug 2023 08:41:26 +0000 Subject: [PATCH 4/6] Add tests --- test/expected/casts.out | 85 +++++++++++++++++++++++++++++++++++++++++ test/sql/casts.sql | 31 +++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 test/expected/casts.out create mode 100644 test/sql/casts.sql diff --git a/test/expected/casts.out b/test/expected/casts.out new file mode 100644 index 000000000..15a781592 --- /dev/null +++ b/test/expected/casts.out @@ -0,0 +1,85 @@ +CREATE EXTENSION IF NOT EXISTS lanterndb; +CREATE EXTENSION + +SELECT '{1,1,1}'::uvec8, '{0,0,0}'::uvec8, '{-1, 0, 1, -1, 000}'::uvec8; + uvec8 | uvec8 | uvec8 +---------+---------+--------------- + {1,1,1} | {0,0,0} | {-1,0,1,-1,0} +(1 row) + +SELECT '{0.111,0.222222,-0.33333333, -0.42424242424242}'::float4[]; + float4 +------------------------------------------ + {0.111,0.222222,-0.33333334,-0.42424244} +(1 row) + +SELECT '{0.111,0.222222,-0.33333333, -0.42424242424242}'::uvec8; + uvec8 +------------------------- + {0.11,0.22,-0.33,-0.42} +(1 row) + +SELECT '{0.111,0.22,0.33}'::uvec8(3); + uvec8 +------------------ + {0.11,0.22,0.33} +(1 row) + +SELECT '{-0.42,0.0000001,0.00002}'::float[3]::uvec8(3); + uvec8 +------------- + {-0.42,0,0} +(1 row) + +SELECT '{-0.42,0.0000001,0.00002}'::float[3]::uvec8(3)::float4[3]; + float4 +------------- + {-0.42,0,0} +(1 row) + +SELECT '{1,2,3}'::uvec8; +psql:test/sql/casts.sql:16: ERROR: uvec8 must be in range [-1, 1] +LINE 1: SELECT '{1,2,3}'::uvec8; + ^ +SELECT '{}'::uvec8; +psql:test/sql/casts.sql:17: ERROR: array must not be empty +LINE 1: SELECT '{}'::uvec8; + ^ +SELECT 'abra'::uvec8; +psql:test/sql/casts.sql:18: ERROR: malformed array literal: "abra" +LINE 1: SELECT 'abra'::uvec8; + ^ +DETAIL: Array value must start with "{" or dimension information. +SELECT '{"haha"}'::uvec8; +psql:test/sql/casts.sql:19: ERROR: invalid input syntax for type real: "haha" +LINE 1: SELECT '{"haha"}'::uvec8; + ^ +SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8; +psql:test/sql/casts.sql:20: ERROR: array must be 1-D +LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8; + ^ +SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8(4); +psql:test/sql/casts.sql:21: ERROR: array must be 1-D +LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8(4); + ^ +SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8[4]; +psql:test/sql/casts.sql:23: ERROR: malformed array literal: ".1" +LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8[4]; + ^ +DETAIL: Array value must start with "{" or dimension information. +SELECT '{0.111,NULL}'::uvec8(2); +psql:test/sql/casts.sql:25: ERROR: array must not contain nulls +LINE 1: SELECT '{0.111,NULL}'::uvec8(2); + ^ +SELECT '{0.1,0.2,0.3}'::uvec8(3,3); +psql:test/sql/casts.sql:26: ERROR: wrong number of modifiers +LINE 1: SELECT '{0.1,0.2,0.3}'::uvec8(3,3); + ^ +SELECT '{0.1,0.2,0.3}'::uvec8(2); +psql:test/sql/casts.sql:27: ERROR: invalid cast. vector dim: 3, cast dim:2 +SELECT '{0.1,0.2,0.3}'::uvec8(3)::uvec8(2); +psql:test/sql/casts.sql:28: ERROR: invalid cast. vector dim: 3, cast dim:2 +SELECT '{1,1,1}'::int[3]::uvec8(4); +psql:test/sql/casts.sql:29: ERROR: invalid cast. vector dim: 3, cast dim:4 +SELECT '{1,1,1}'::int[3]::uvec8(3)::uvec8(4); +psql:test/sql/casts.sql:30: ERROR: invalid cast. vector dim: 3, cast dim:4 diff --git a/test/sql/casts.sql b/test/sql/casts.sql new file mode 100644 index 000000000..cdec5aeaf --- /dev/null +++ b/test/sql/casts.sql @@ -0,0 +1,31 @@ +CREATE EXTENSION IF NOT EXISTS lanterndb; + +\qecho +\set ON_ERROR_STOP on + +SELECT '{1,1,1}'::uvec8, '{0,0,0}'::uvec8, '{-1, 0, 1, -1, 000}'::uvec8; + +SELECT '{0.111,0.222222,-0.33333333, -0.42424242424242}'::float4[]; +SELECT '{0.111,0.222222,-0.33333333, -0.42424242424242}'::uvec8; +SELECT '{0.111,0.22,0.33}'::uvec8(3); +SELECT '{-0.42,0.0000001,0.00002}'::float[3]::uvec8(3); +SELECT '{-0.42,0.0000001,0.00002}'::float[3]::uvec8(3)::float4[3]; + + +\set ON_ERROR_STOP off +SELECT '{1,2,3}'::uvec8; +SELECT '{}'::uvec8; +SELECT 'abra'::uvec8; +SELECT '{"haha"}'::uvec8; +SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8; +SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8(4); +-- todo:: the next one gives a funky error message. Make them more informative +SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8[4]; + +SELECT '{0.111,NULL}'::uvec8(2); +SELECT '{0.1,0.2,0.3}'::uvec8(3,3); +SELECT '{0.1,0.2,0.3}'::uvec8(2); +SELECT '{0.1,0.2,0.3}'::uvec8(3)::uvec8(2); +SELECT '{1,1,1}'::int[3]::uvec8(4); +SELECT '{1,1,1}'::int[3]::uvec8(3)::uvec8(4); +\set ON_ERROR_STOP on From 0262149b52f82e81b8dc1a7679227dfa83459056 Mon Sep 17 00:00:00 2001 From: Narek Galstyan Date: Sun, 6 Aug 2023 09:12:52 +0000 Subject: [PATCH 5/6] Add vec8 type --- sql/lanterndb.sql | 35 ++++++++++------ src/vector_casts.c | 89 ++++++++++++++++++++--------------------- test/expected/casts.out | 32 +++++++++------ test/sql/casts.sql | 4 ++ 4 files changed, 88 insertions(+), 72 deletions(-) diff --git a/sql/lanterndb.sql b/sql/lanterndb.sql index 5828e6209..f396ef7cf 100644 --- a/sql/lanterndb.sql +++ b/sql/lanterndb.sql @@ -60,18 +60,10 @@ CREATE FUNCTION ldb_generic_vec_typmod_in(cstring[]) RETURNS integer -- 8-byte unit-vector (i.e. vector with elements in range [-1, 1]) CREATE TYPE uvec8; -CREATE FUNCTION ldb_uvec8_in(cstring, oid, integer) RETURNS uvec8 - AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; - -CREATE FUNCTION ldb_uvec8_out(uvec8) RETURNS cstring - AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; - -CREATE FUNCTION ldb_uvec8_recv(internal, oid, integer) RETURNS uvec8 - AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; - -CREATE FUNCTION ldb_uvec8_send(uvec8) RETURNS bytea - AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; - +CREATE FUNCTION ldb_uvec8_in(cstring, oid, integer) RETURNS uvec8 AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_uvec8_out(uvec8) RETURNS cstring AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_uvec8_recv(internal, oid, integer) RETURNS uvec8 AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_uvec8_send(uvec8) RETURNS bytea AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE uvec8 ( INPUT = ldb_uvec8_in, @@ -82,7 +74,24 @@ CREATE TYPE uvec8 ( STORAGE = extended ); --- -- Unit ector which has 1 byte elements in range [-1, 1] +CREATE TYPE vec8; + +CREATE FUNCTION ldb_vec8_in(cstring, oid, integer) RETURNS vec8 AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_vec8_out(vec8) RETURNS cstring AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_vec8_recv(internal, oid, integer) RETURNS vec8 AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION ldb_vec8_send(vec8) RETURNS bytea AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +CREATE TYPE vec8 ( + INPUT = ldb_vec8_in, + OUTPUT = ldb_vec8_out, + RECEIVE = ldb_vec8_recv, + SEND = ldb_vec8_send, + TYPMOD_IN = ldb_generic_vec_typmod_in, + STORAGE = extended +); + +-- cast functions + CREATE FUNCTION ldb_cast_uvec8_uvec8(uvec8, integer, boolean) RETURNS uvec8 AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION ldb_cast_array_uvec8(integer[], integer, boolean) RETURNS uvec8 diff --git a/src/vector_casts.c b/src/vector_casts.c index 346cb21c1..a080b596d 100644 --- a/src/vector_casts.c +++ b/src/vector_casts.c @@ -15,10 +15,9 @@ /*** Functions generic over all vec types (uvec8, vec8, vec16, vec32) */ -ArrayType *ldb_generic_vec_in(PG_FUNCTION_ARGS) +LDBVec *ldb_generic_vec_in(FunctionCallInfo fcinfo, usearch_scalar_kind_t to) { char *str = PG_GETARG_CSTRING(0); - // The second argument is the type OID of our newly created type // we do not need that. For arrays, it would be the element type // and not the array type. It would be good to have this be the @@ -28,13 +27,12 @@ ArrayType *ldb_generic_vec_in(PG_FUNCTION_ARGS) // In the future, we may want to have vec64 (but unlikely, since // postgres's own double precision works just fine) int32 typioparam = FLOAT4OID; - // represents the number of dimensions in the vec type: '{1,1,1}'::vec(3) // ....^... - int32 typmod = PG_GETARG_INT32(2); - + int32 typmod = PG_GETARG_INT32(2); Oid oid = fmgr_internal_function("array_in"); ArrayType *array; + LDBVec *res; array = (ArrayType *)DatumGetPointer(OidInputFunctionCall(oid, str, typioparam, typmod)); // postgres does not enforce the array bounds(cite?), but we want to enforce them @@ -53,7 +51,13 @@ ArrayType *ldb_generic_vec_in(PG_FUNCTION_ARGS) if(typmod != -1 && ARR_NDIM(array) != typmod) { elog(ERROR, "vec type expected %d dimensions but provided array has %d dimensions", typmod, ARR_NDIM(array)); } - return array; + assert(array != NULL); + assert(array->elemtype == FLOAT4OID); + + int ndims = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); + res = ldb_generic_cast_array_vec(array, ndims, to); + + return res; } PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_generic_vec_typmod_in); @@ -90,8 +94,24 @@ Datum ldb_generic_vec_typmod_in(PG_FUNCTION_ARGS) return Int32GetDatum(*tl); } -Datum ldb_generic_vec_out(ArrayType *arr) +Datum ldb_generic_vec_out(FunctionCallInfo fcinfo, usearch_scalar_kind_t from) { + ArrayType *arr; + float *array_elems; + int array_elems_size; + usearch_error_t error = NULL; + LDBVec *vec = DatumGetLDBVec(PG_GETARG_DATUM(0)); + assert(vec->dim != 0); + array_elems_size = sizeof(float) * vec->dim; + array_elems = palloc0(array_elems_size); + + usearch_cast(from, vec->data, usearch_scalar_f32_k, array_elems, array_elems_size, vec->dim, &error); + if(error != NULL) { + elog(ERROR, "error casting: %s", error); + } + + arr = ldb_generic_cast_vec_array(array_elems, vec->dim); + Oid oid = fmgr_internal_function("array_out"); char *res = OidOutputFunctionCall(oid, PointerGetDatum(arr)); return CStringGetDatum(res); @@ -146,56 +166,29 @@ Datum ldb_generic_vec_send(PG_FUNCTION_ARGS) PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_uvec8_in); -Datum ldb_uvec8_in(PG_FUNCTION_ARGS) -{ - LDBVec *res; - usearch_error_t error = NULL; - // fcinfo below is from the macro PG_FUNCTION_ARGS and passes SQL-arguments - // to the generic vec reader - ArrayType *array = ldb_generic_vec_in(fcinfo); - assert(array != NULL); +// fcinfo below is from the macro PG_FUNCTION_ARGS and passes SQL-arguments +// to the generic vec reader +Datum ldb_uvec8_in(PG_FUNCTION_ARGS) { return PointerGetDatum(ldb_generic_vec_in(fcinfo, usearch_scalar_f8_k)); } - assert(array->elemtype == FLOAT4OID); - int ndims = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); - res = ldb_generic_cast_array_vec(array, ndims, usearch_scalar_f8_k); +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_vec8_in); - if(error) { - elog(ERROR, "error in float downcasting: %s", error); - } - - return PointerGetDatum(res); -} +Datum ldb_vec8_in(PG_FUNCTION_ARGS) { return PointerGetDatum(ldb_generic_vec_in(fcinfo, usearch_scalar_b1_k)); } PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_uvec8_out); +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_vec8_out); -Datum ldb_uvec8_out(PG_FUNCTION_ARGS) -{ - ArrayType *res; - float *array_elems; - Datum *array_elems_datum; - int array_elems_size; - usearch_error_t error = NULL; - LDBVec *vec = DatumGetLDBVec(PG_GETARG_DATUM(0)); - assert(vec->dim != 0); - assert(vec->elem_type == usearch_scalar_f8_k); - array_elems_size = sizeof(float) * vec->dim; - array_elems = palloc0(array_elems_size); - - usearch_cast(usearch_scalar_f8_k, vec->data, usearch_scalar_f32_k, array_elems, array_elems_size, vec->dim, &error); - if(error != NULL) { - elog(ERROR, "error casting: %s", error); - } - - res = ldb_generic_cast_vec_array(array_elems, vec->dim); - - return ldb_generic_vec_out(res); -} +Datum ldb_uvec8_out(PG_FUNCTION_ARGS) { return ldb_generic_vec_out(fcinfo, usearch_scalar_f8_k); } +Datum ldb_vec8_out(PG_FUNCTION_ARGS) { return ldb_generic_vec_out(fcinfo, usearch_scalar_b1_k); } PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_uvec8_recv); +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_vec8_recv); Datum ldb_uvec8_recv(PG_FUNCTION_ARGS) { return ldb_generic_vec_recv(fcinfo); } +Datum ldb_vec8_recv(PG_FUNCTION_ARGS) { return ldb_generic_vec_recv(fcinfo); } PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_uvec8_send); +PGDLLEXPORT PG_FUNCTION_INFO_V1(ldb_vec8_send); Datum ldb_uvec8_send(PG_FUNCTION_ARGS) { return ldb_generic_vec_send(fcinfo); } +Datum ldb_vec8_send(PG_FUNCTION_ARGS) { return ldb_generic_vec_send(fcinfo); } // PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector); // Datum @@ -365,6 +358,10 @@ LDBVec *ldb_generic_cast_array_vec(ArrayType *array, int expected_dim, usearch_s usearch_cast( usearch_scalar_f32_k, vec_floats, to, LDBVEC_DATA_PTR(result), result->dim, LDBVEC_DATA_SIZE(result), &error); + if(error) { + elog(ERROR, "error in float downcasting: %s", error); + } + return result; } diff --git a/test/expected/casts.out b/test/expected/casts.out index 15a781592..a6c477433 100644 --- a/test/expected/casts.out +++ b/test/expected/casts.out @@ -37,49 +37,55 @@ SELECT '{-0.42,0.0000001,0.00002}'::float[3]::uvec8(3)::float4[3]; {-0.42,0,0} (1 row) +SELECT '{.1, 0.33, .42, .55, -.42, -0.00001, -.1234567}'::vec8, '{.1, 0.33, .42, .55, -.42, -0.00001, -.1234567}'::uvec8; + vec8 | uvec8 +-----------------+------------------------------------ + {1,1,1,1,1,1,1} | {0.1,0.33,0.42,0.55,-0.42,0,-0.12} +(1 row) + SELECT '{1,2,3}'::uvec8; -psql:test/sql/casts.sql:16: ERROR: uvec8 must be in range [-1, 1] +psql:test/sql/casts.sql:20: ERROR: uvec8 must be in range [-1, 1] LINE 1: SELECT '{1,2,3}'::uvec8; ^ SELECT '{}'::uvec8; -psql:test/sql/casts.sql:17: ERROR: array must not be empty +psql:test/sql/casts.sql:21: ERROR: array must not be empty LINE 1: SELECT '{}'::uvec8; ^ SELECT 'abra'::uvec8; -psql:test/sql/casts.sql:18: ERROR: malformed array literal: "abra" +psql:test/sql/casts.sql:22: ERROR: malformed array literal: "abra" LINE 1: SELECT 'abra'::uvec8; ^ DETAIL: Array value must start with "{" or dimension information. SELECT '{"haha"}'::uvec8; -psql:test/sql/casts.sql:19: ERROR: invalid input syntax for type real: "haha" +psql:test/sql/casts.sql:23: ERROR: invalid input syntax for type real: "haha" LINE 1: SELECT '{"haha"}'::uvec8; ^ SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8; -psql:test/sql/casts.sql:20: ERROR: array must be 1-D +psql:test/sql/casts.sql:24: ERROR: array must be 1-D LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8; ^ SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8(4); -psql:test/sql/casts.sql:21: ERROR: array must be 1-D +psql:test/sql/casts.sql:25: ERROR: array must be 1-D LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8(4); ^ SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8[4]; -psql:test/sql/casts.sql:23: ERROR: malformed array literal: ".1" +psql:test/sql/casts.sql:27: ERROR: malformed array literal: ".1" LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8[4]; ^ DETAIL: Array value must start with "{" or dimension information. SELECT '{0.111,NULL}'::uvec8(2); -psql:test/sql/casts.sql:25: ERROR: array must not contain nulls +psql:test/sql/casts.sql:29: ERROR: array must not contain nulls LINE 1: SELECT '{0.111,NULL}'::uvec8(2); ^ SELECT '{0.1,0.2,0.3}'::uvec8(3,3); -psql:test/sql/casts.sql:26: ERROR: wrong number of modifiers +psql:test/sql/casts.sql:30: ERROR: wrong number of modifiers LINE 1: SELECT '{0.1,0.2,0.3}'::uvec8(3,3); ^ SELECT '{0.1,0.2,0.3}'::uvec8(2); -psql:test/sql/casts.sql:27: ERROR: invalid cast. vector dim: 3, cast dim:2 +psql:test/sql/casts.sql:31: ERROR: invalid cast. vector dim: 3, cast dim:2 SELECT '{0.1,0.2,0.3}'::uvec8(3)::uvec8(2); -psql:test/sql/casts.sql:28: ERROR: invalid cast. vector dim: 3, cast dim:2 +psql:test/sql/casts.sql:32: ERROR: invalid cast. vector dim: 3, cast dim:2 SELECT '{1,1,1}'::int[3]::uvec8(4); -psql:test/sql/casts.sql:29: ERROR: invalid cast. vector dim: 3, cast dim:4 +psql:test/sql/casts.sql:33: ERROR: invalid cast. vector dim: 3, cast dim:4 SELECT '{1,1,1}'::int[3]::uvec8(3)::uvec8(4); -psql:test/sql/casts.sql:30: ERROR: invalid cast. vector dim: 3, cast dim:4 +psql:test/sql/casts.sql:34: ERROR: invalid cast. vector dim: 3, cast dim:4 diff --git a/test/sql/casts.sql b/test/sql/casts.sql index cdec5aeaf..cfaa822c9 100644 --- a/test/sql/casts.sql +++ b/test/sql/casts.sql @@ -3,6 +3,7 @@ CREATE EXTENSION IF NOT EXISTS lanterndb; \qecho \set ON_ERROR_STOP on +-- uvec8 SELECT '{1,1,1}'::uvec8, '{0,0,0}'::uvec8, '{-1, 0, 1, -1, 000}'::uvec8; SELECT '{0.111,0.222222,-0.33333333, -0.42424242424242}'::float4[]; @@ -11,6 +12,9 @@ SELECT '{0.111,0.22,0.33}'::uvec8(3); SELECT '{-0.42,0.0000001,0.00002}'::float[3]::uvec8(3); SELECT '{-0.42,0.0000001,0.00002}'::float[3]::uvec8(3)::float4[3]; +-- vec8 +SELECT '{.1, 0.33, .42, .55, -.42, -0.00001, -.1234567}'::vec8, '{.1, 0.33, .42, .55, -.42, -0.00001, -.1234567}'::uvec8; + \set ON_ERROR_STOP off SELECT '{1,2,3}'::uvec8; From 27b253f51fc63390111fdb4b24f58acb1783d812 Mon Sep 17 00:00:00 2001 From: Narek Galstyan Date: Sun, 6 Aug 2023 10:35:47 +0000 Subject: [PATCH 6/6] Add more tests and a bugfix --- src/vec_type.h | 2 +- src/vector_casts.c | 5 +++-- test/expected/casts.out | 39 ++++++++++++++++++++++++++------------- test/sql/casts.sql | 5 +++++ 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/vec_type.h b/src/vec_type.h index 06d4d1853..e667b45e0 100644 --- a/src/vec_type.h +++ b/src/vec_type.h @@ -39,7 +39,7 @@ static inline LDBVec *NewLDBVec(int dim, int elem_type) } /* Confined by uint16 in LDBVec structure */ -#define LDB_VEC_MAX_DIM 1 << 16 +#define LDB_VEC_MAX_DIM ((1 << 16) - 1) /* * Returns a pointer to the actual array data. */ diff --git a/src/vector_casts.c b/src/vector_casts.c index a080b596d..c52eb3081 100644 --- a/src/vector_casts.c +++ b/src/vector_casts.c @@ -84,7 +84,7 @@ Datum ldb_generic_vec_typmod_in(PG_FUNCTION_ARGS) } if(*tl < 1) { - elog(ERROR, "vector dimensions must be at least 1"); + elog(ERROR, "vector dimension must be >= 1"); } if(*tl > LDB_VEC_MAX_DIM) { @@ -342,8 +342,9 @@ LDBVec *ldb_generic_cast_array_vec(ArrayType *array, int expected_dim, usearch_s for(int i = 0; i < nelemsp; i++) { vec_floats[ i ] = DatumGetFloat4(DirectFunctionCall1(numeric_float4, elemsp[ i ])); } + break; default: - elog(ERROR, "unknown array elem type %d", ARR_ELEMTYPE(array)); + elog(ERROR, "unknown array element type %d", ARR_ELEMTYPE(array)); } if(usearch_scalar_f8_k == to) { diff --git a/test/expected/casts.out b/test/expected/casts.out index a6c477433..66e0eecb1 100644 --- a/test/expected/casts.out +++ b/test/expected/casts.out @@ -43,49 +43,62 @@ SELECT '{.1, 0.33, .42, .55, -.42, -0.00001, -.1234567}'::vec8, '{.1, 0.33, .42, {1,1,1,1,1,1,1} | {0.1,0.33,0.42,0.55,-0.42,0,-0.12} (1 row) +SELECT ARRAY[1,.2,.3]::uvec8; + array +------------- + {1,0.2,0.3} +(1 row) + +SELECT ARRAY(SELECT ROUND(RANDOM()) FROM generate_series(1,65534))::uvec8 SELECT '{1,2,3}'::uvec8; -psql:test/sql/casts.sql:20: ERROR: uvec8 must be in range [-1, 1] +psql:test/sql/casts.sql:22: ERROR: uvec8 must be in range [-1, 1] LINE 1: SELECT '{1,2,3}'::uvec8; ^ SELECT '{}'::uvec8; -psql:test/sql/casts.sql:21: ERROR: array must not be empty +psql:test/sql/casts.sql:23: ERROR: array must not be empty LINE 1: SELECT '{}'::uvec8; ^ SELECT 'abra'::uvec8; -psql:test/sql/casts.sql:22: ERROR: malformed array literal: "abra" +psql:test/sql/casts.sql:24: ERROR: malformed array literal: "abra" LINE 1: SELECT 'abra'::uvec8; ^ DETAIL: Array value must start with "{" or dimension information. SELECT '{"haha"}'::uvec8; -psql:test/sql/casts.sql:23: ERROR: invalid input syntax for type real: "haha" +psql:test/sql/casts.sql:25: ERROR: invalid input syntax for type real: "haha" LINE 1: SELECT '{"haha"}'::uvec8; ^ SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8; -psql:test/sql/casts.sql:24: ERROR: array must be 1-D +psql:test/sql/casts.sql:26: ERROR: array must be 1-D LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8; ^ SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8(4); -psql:test/sql/casts.sql:25: ERROR: array must be 1-D +psql:test/sql/casts.sql:27: ERROR: array must be 1-D LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8(4); ^ SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8[4]; -psql:test/sql/casts.sql:27: ERROR: malformed array literal: ".1" +psql:test/sql/casts.sql:29: ERROR: malformed array literal: ".1" LINE 1: SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8[4]; ^ DETAIL: Array value must start with "{" or dimension information. SELECT '{0.111,NULL}'::uvec8(2); -psql:test/sql/casts.sql:29: ERROR: array must not contain nulls +psql:test/sql/casts.sql:31: ERROR: array must not contain nulls LINE 1: SELECT '{0.111,NULL}'::uvec8(2); ^ +SELECT '{.1, .2}'::uvec8(-2); +psql:test/sql/casts.sql:32: ERROR: vector dimension must be >= 1 +LINE 1: SELECT '{.1, .2}'::uvec8(-2); + ^ +SELECT ARRAY(SELECT ROUND(RANDOM()) FROM generate_series(1,65536))::uvec8; +psql:test/sql/casts.sql:33: ERROR: array too large. max vec dimension is 65535 SELECT '{0.1,0.2,0.3}'::uvec8(3,3); -psql:test/sql/casts.sql:30: ERROR: wrong number of modifiers +psql:test/sql/casts.sql:35: ERROR: wrong number of modifiers LINE 1: SELECT '{0.1,0.2,0.3}'::uvec8(3,3); ^ SELECT '{0.1,0.2,0.3}'::uvec8(2); -psql:test/sql/casts.sql:31: ERROR: invalid cast. vector dim: 3, cast dim:2 +psql:test/sql/casts.sql:36: ERROR: invalid cast. vector dim: 3, cast dim:2 SELECT '{0.1,0.2,0.3}'::uvec8(3)::uvec8(2); -psql:test/sql/casts.sql:32: ERROR: invalid cast. vector dim: 3, cast dim:2 +psql:test/sql/casts.sql:37: ERROR: invalid cast. vector dim: 3, cast dim:2 SELECT '{1,1,1}'::int[3]::uvec8(4); -psql:test/sql/casts.sql:33: ERROR: invalid cast. vector dim: 3, cast dim:4 +psql:test/sql/casts.sql:38: ERROR: invalid cast. vector dim: 3, cast dim:4 SELECT '{1,1,1}'::int[3]::uvec8(3)::uvec8(4); -psql:test/sql/casts.sql:34: ERROR: invalid cast. vector dim: 3, cast dim:4 +psql:test/sql/casts.sql:39: ERROR: invalid cast. vector dim: 3, cast dim:4 diff --git a/test/sql/casts.sql b/test/sql/casts.sql index cfaa822c9..6e1239b9c 100644 --- a/test/sql/casts.sql +++ b/test/sql/casts.sql @@ -15,6 +15,8 @@ SELECT '{-0.42,0.0000001,0.00002}'::float[3]::uvec8(3)::float4[3]; -- vec8 SELECT '{.1, 0.33, .42, .55, -.42, -0.00001, -.1234567}'::vec8, '{.1, 0.33, .42, .55, -.42, -0.00001, -.1234567}'::uvec8; +SELECT ARRAY[1,.2,.3]::uvec8; +SELECT ARRAY(SELECT ROUND(RANDOM()) FROM generate_series(1,65534))::uvec8 \gset \set ON_ERROR_STOP off SELECT '{1,2,3}'::uvec8; @@ -27,6 +29,9 @@ SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8(4); SELECT '{{.1,.2,.3},{.4,.5,.6}}'::uvec8[4]; SELECT '{0.111,NULL}'::uvec8(2); +SELECT '{.1, .2}'::uvec8(-2); +SELECT ARRAY(SELECT ROUND(RANDOM()) FROM generate_series(1,65536))::uvec8; + SELECT '{0.1,0.2,0.3}'::uvec8(3,3); SELECT '{0.1,0.2,0.3}'::uvec8(2); SELECT '{0.1,0.2,0.3}'::uvec8(3)::uvec8(2);