From f97ef49e85342068e8026e57abe9d292db319864 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Sun, 8 Mar 2026 20:18:15 +0800 Subject: [PATCH 1/9] feat: implement advanced state API for Go and Python - Go SDK: add state package with codec, keyed state, and structures - codec: bool/bytes/float/int/string/uint/json and ordered types - keyed: Value, List, Map, Aggregating, Reducing, PriorityQueue keyed states - structures: underlying implementations for each state type - Python API: refactor store into codec, keyed, structures submodules - keyed: KeyedValueState, KeyedListState, KeyedMapState, etc. - structures: state structures aligned with Go - Runtime: update fs_context to work with new state API Made-with: Cursor --- go-sdk/state/codec/bool_codec.go | 30 ++ go-sdk/state/codec/bytes_codec.go | 13 + go-sdk/state/codec/default_codec.go | 39 +++ go-sdk/state/codec/float32_codec.go | 25 ++ go-sdk/state/codec/float64_codec.go | 25 ++ go-sdk/state/codec/int32_codec.go | 24 ++ go-sdk/state/codec/int64_codec.go | 24 ++ go-sdk/state/codec/interface.go | 16 + go-sdk/state/codec/json_codec.go | 18 ++ go-sdk/state/codec/ordered_float32_codec.go | 38 +++ go-sdk/state/codec/ordered_float64_codec.go | 38 +++ go-sdk/state/codec/ordered_int32_codec.go | 25 ++ go-sdk/state/codec/ordered_int64_codec.go | 15 + go-sdk/state/codec/ordered_int_codec.go | 25 ++ go-sdk/state/codec/ordered_uint32_codec.go | 25 ++ go-sdk/state/codec/ordered_uint64_codec.go | 25 ++ go-sdk/state/codec/ordered_uint_codec.go | 25 ++ go-sdk/state/codec/string_codec.go | 11 + go-sdk/state/codec/uint32_codec.go | 24 ++ go-sdk/state/codec/uint64_codec.go | 24 ++ go-sdk/state/common/common.go | 66 +++++ go-sdk/state/context.go | 137 +++++++++ go-sdk/state/keyed/keyed_aggregating_state.go | 113 ++++++++ go-sdk/state/keyed/keyed_list_state.go | 257 +++++++++++++++++ go-sdk/state/keyed/keyed_map_state.go | 153 ++++++++++ .../state/keyed/keyed_priority_queue_state.go | 162 +++++++++++ go-sdk/state/keyed/keyed_reducing_state.go | 116 ++++++++ go-sdk/state/keyed/keyed_state_factory.go | 39 +++ go-sdk/state/keyed/keyed_value_state.go | 89 ++++++ go-sdk/state/state.go | 51 ++++ go-sdk/state/structures/aggregating.go | 108 +++++++ go-sdk/state/structures/list.go | 209 ++++++++++++++ go-sdk/state/structures/map.go | 253 ++++++++++++++++ go-sdk/state/structures/priority_queue.go | 126 ++++++++ go-sdk/state/structures/reducing.go | 100 +++++++ go-sdk/state/structures/value.go | 65 +++++ .../functionstream-api/src/fs_api/__init__.py | 72 +++++ .../functionstream-api/src/fs_api/context.py | 50 +++- .../src/fs_api/store/__init__.py | 99 ++++++- .../src/fs_api/store/codec/__init__.py | 273 ++++++++++++++++++ .../src/fs_api/store/common/__init__.py | 68 +++++ .../src/fs_api/store/keyed/__init__.py | 32 ++ .../src/fs_api/store/keyed/_keyed_common.py | 38 +++ .../store/keyed/keyed_aggregating_state.py | 80 +++++ .../fs_api/store/keyed/keyed_list_state.py | 94 ++++++ .../src/fs_api/store/keyed/keyed_map_state.py | 149 ++++++++++ .../store/keyed/keyed_priority_queue_state.py | 92 ++++++ .../store/keyed/keyed_reducing_state.py | 70 +++++ .../fs_api/store/keyed/keyed_state_factory.py | 94 ++++++ .../fs_api/store/keyed/keyed_value_state.py | 56 ++++ .../src/fs_api/store/structures/__init__.py | 37 +++ .../store/structures/aggregating_state.py | 86 ++++++ .../src/fs_api/store/structures/list_state.py | 95 ++++++ .../src/fs_api/store/structures/map_state.py | 168 +++++++++++ .../store/structures/priority_queue_state.py | 84 ++++++ .../fs_api/store/structures/reducing_state.py | 63 ++++ .../fs_api/store/structures/value_state.py | 54 ++++ .../src/fs_runtime/store/fs_context.py | 50 +++- 58 files changed, 4425 insertions(+), 12 deletions(-) create mode 100644 go-sdk/state/codec/bool_codec.go create mode 100644 go-sdk/state/codec/bytes_codec.go create mode 100644 go-sdk/state/codec/default_codec.go create mode 100644 go-sdk/state/codec/float32_codec.go create mode 100644 go-sdk/state/codec/float64_codec.go create mode 100644 go-sdk/state/codec/int32_codec.go create mode 100644 go-sdk/state/codec/int64_codec.go create mode 100644 go-sdk/state/codec/interface.go create mode 100644 go-sdk/state/codec/json_codec.go create mode 100644 go-sdk/state/codec/ordered_float32_codec.go create mode 100644 go-sdk/state/codec/ordered_float64_codec.go create mode 100644 go-sdk/state/codec/ordered_int32_codec.go create mode 100644 go-sdk/state/codec/ordered_int64_codec.go create mode 100644 go-sdk/state/codec/ordered_int_codec.go create mode 100644 go-sdk/state/codec/ordered_uint32_codec.go create mode 100644 go-sdk/state/codec/ordered_uint64_codec.go create mode 100644 go-sdk/state/codec/ordered_uint_codec.go create mode 100644 go-sdk/state/codec/string_codec.go create mode 100644 go-sdk/state/codec/uint32_codec.go create mode 100644 go-sdk/state/codec/uint64_codec.go create mode 100644 go-sdk/state/common/common.go create mode 100644 go-sdk/state/context.go create mode 100644 go-sdk/state/keyed/keyed_aggregating_state.go create mode 100644 go-sdk/state/keyed/keyed_list_state.go create mode 100644 go-sdk/state/keyed/keyed_map_state.go create mode 100644 go-sdk/state/keyed/keyed_priority_queue_state.go create mode 100644 go-sdk/state/keyed/keyed_reducing_state.go create mode 100644 go-sdk/state/keyed/keyed_state_factory.go create mode 100644 go-sdk/state/keyed/keyed_value_state.go create mode 100644 go-sdk/state/state.go create mode 100644 go-sdk/state/structures/aggregating.go create mode 100644 go-sdk/state/structures/list.go create mode 100644 go-sdk/state/structures/map.go create mode 100644 go-sdk/state/structures/priority_queue.go create mode 100644 go-sdk/state/structures/reducing.go create mode 100644 go-sdk/state/structures/value.go create mode 100644 python/functionstream-api/src/fs_api/store/codec/__init__.py create mode 100644 python/functionstream-api/src/fs_api/store/common/__init__.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/__init__.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py create mode 100644 python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py create mode 100644 python/functionstream-api/src/fs_api/store/structures/__init__.py create mode 100644 python/functionstream-api/src/fs_api/store/structures/aggregating_state.py create mode 100644 python/functionstream-api/src/fs_api/store/structures/list_state.py create mode 100644 python/functionstream-api/src/fs_api/store/structures/map_state.py create mode 100644 python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py create mode 100644 python/functionstream-api/src/fs_api/store/structures/reducing_state.py create mode 100644 python/functionstream-api/src/fs_api/store/structures/value_state.py diff --git a/go-sdk/state/codec/bool_codec.go b/go-sdk/state/codec/bool_codec.go new file mode 100644 index 00000000..6be75688 --- /dev/null +++ b/go-sdk/state/codec/bool_codec.go @@ -0,0 +1,30 @@ +package codec + +import "fmt" + +type BoolCodec struct{} + +func (c BoolCodec) Encode(value bool) ([]byte, error) { + if value { + return []byte{1}, nil + } + return []byte{0}, nil +} + +func (c BoolCodec) Decode(data []byte) (bool, error) { + if len(data) != 1 { + return false, fmt.Errorf("invalid bool payload length: %d", len(data)) + } + switch data[0] { + case 0: + return false, nil + case 1: + return true, nil + default: + return false, fmt.Errorf("invalid bool payload byte: %d", data[0]) + } +} + +func (c BoolCodec) EncodedSize() int { return 1 } + +func (c BoolCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/bytes_codec.go b/go-sdk/state/codec/bytes_codec.go new file mode 100644 index 00000000..d276fa69 --- /dev/null +++ b/go-sdk/state/codec/bytes_codec.go @@ -0,0 +1,13 @@ +package codec + +import "github.com/functionstream/function-stream/go-sdk/state/common" + +type BytesCodec struct{} + +func (c BytesCodec) Encode(value []byte) ([]byte, error) { return common.DupBytes(value), nil } + +func (c BytesCodec) Decode(data []byte) ([]byte, error) { return common.DupBytes(data), nil } + +func (c BytesCodec) EncodedSize() int { return -1 } + +func (c BytesCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/default_codec.go b/go-sdk/state/codec/default_codec.go new file mode 100644 index 00000000..c8dc85d8 --- /dev/null +++ b/go-sdk/state/codec/default_codec.go @@ -0,0 +1,39 @@ +package codec + +import ( + "fmt" + "reflect" +) + +func DefaultCodecFor[V any]() (Codec[V], error) { + var zero V + t := reflect.TypeOf(zero) + if t == nil { + return nil, fmt.Errorf("default codec: type parameter V must not be interface without type constraint") + } + k := t.Kind() + switch k { + case reflect.Bool: + return any(BoolCodec{}).(Codec[V]), nil + case reflect.Int32: + return any(OrderedInt32Codec{}).(Codec[V]), nil + case reflect.Int64: + return any(OrderedInt64Codec{}).(Codec[V]), nil + case reflect.Uint32: + return any(OrderedUint32Codec{}).(Codec[V]), nil + case reflect.Uint64: + return any(OrderedUint64Codec{}).(Codec[V]), nil + case reflect.Float32: + return any(OrderedFloat32Codec{}).(Codec[V]), nil + case reflect.Float64: + return any(OrderedFloat64Codec{}).(Codec[V]), nil + case reflect.String: + return any(StringCodec{}).(Codec[V]), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Uint, reflect.Uint8, reflect.Uint16: + return any(JSONCodec[V]{}).(Codec[V]), nil + case reflect.Struct, reflect.Map, reflect.Slice, reflect.Array, reflect.Interface: + return any(JSONCodec[V]{}).(Codec[V]), nil + default: + return any(JSONCodec[V]{}).(Codec[V]), nil + } +} diff --git a/go-sdk/state/codec/float32_codec.go b/go-sdk/state/codec/float32_codec.go new file mode 100644 index 00000000..719cdcaa --- /dev/null +++ b/go-sdk/state/codec/float32_codec.go @@ -0,0 +1,25 @@ +package codec + +import ( + "encoding/binary" + "fmt" + "math" +) + +type Float32Codec struct{} + +func (c Float32Codec) Encode(value float32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, math.Float32bits(value)) + return out, nil +} + +func (c Float32Codec) Decode(data []byte) (float32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid float32 payload length: %d", len(data)) + } + return math.Float32frombits(binary.BigEndian.Uint32(data)), nil +} + +func (c Float32Codec) EncodedSize() int { return 4 } +func (c Float32Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/float64_codec.go b/go-sdk/state/codec/float64_codec.go new file mode 100644 index 00000000..cff6b2c0 --- /dev/null +++ b/go-sdk/state/codec/float64_codec.go @@ -0,0 +1,25 @@ +package codec + +import ( + "encoding/binary" + "fmt" + "math" +) + +type Float64Codec struct{} + +func (c Float64Codec) Encode(value float64) ([]byte, error) { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, math.Float64bits(value)) + return out, nil +} + +func (c Float64Codec) Decode(data []byte) (float64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid float64 payload length: %d", len(data)) + } + return math.Float64frombits(binary.BigEndian.Uint64(data)), nil +} + +func (c Float64Codec) EncodedSize() int { return 8 } +func (c Float64Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/int32_codec.go b/go-sdk/state/codec/int32_codec.go new file mode 100644 index 00000000..25bbf245 --- /dev/null +++ b/go-sdk/state/codec/int32_codec.go @@ -0,0 +1,24 @@ +package codec + +import ( + "encoding/binary" + "fmt" +) + +type Int32Codec struct{} + +func (c Int32Codec) Encode(value int32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, uint32(value)) + return out, nil +} + +func (c Int32Codec) Decode(data []byte) (int32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid int32 payload length: %d", len(data)) + } + return int32(binary.BigEndian.Uint32(data)), nil +} + +func (c Int32Codec) EncodedSize() int { return 4 } +func (c Int32Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/int64_codec.go b/go-sdk/state/codec/int64_codec.go new file mode 100644 index 00000000..8db9a7a4 --- /dev/null +++ b/go-sdk/state/codec/int64_codec.go @@ -0,0 +1,24 @@ +package codec + +import ( + "encoding/binary" + "fmt" +) + +type Int64Codec struct{} + +func (c Int64Codec) Encode(value int64) ([]byte, error) { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, uint64(value)) + return out, nil +} + +func (c Int64Codec) Decode(data []byte) (int64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid int64 payload length: %d", len(data)) + } + return int64(binary.BigEndian.Uint64(data)), nil +} + +func (c Int64Codec) EncodedSize() int { return 8 } +func (c Int64Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/interface.go b/go-sdk/state/codec/interface.go new file mode 100644 index 00000000..b5a958b1 --- /dev/null +++ b/go-sdk/state/codec/interface.go @@ -0,0 +1,16 @@ +package codec + +// Codec is the core encoding interface. EncodedSize reports encoded byte length: +// >0 for fixed-size, <=0 for variable-size. +// IsOrderedKeyCodec reports whether the encoding is byte-orderable (for use as map/keyed state key). +type Codec[T any] interface { + Encode(value T) ([]byte, error) + Decode(data []byte) (T, error) + EncodedSize() int + IsOrderedKeyCodec() bool +} + +func FixedEncodedSize[T any](c Codec[T]) (int, bool) { + n := c.EncodedSize() + return n, n > 0 +} diff --git a/go-sdk/state/codec/json_codec.go b/go-sdk/state/codec/json_codec.go new file mode 100644 index 00000000..9b45bb87 --- /dev/null +++ b/go-sdk/state/codec/json_codec.go @@ -0,0 +1,18 @@ +package codec + +import "encoding/json" + +type JSONCodec[T any] struct{} + +func (c JSONCodec[T]) Encode(value T) ([]byte, error) { return json.Marshal(value) } + +func (c JSONCodec[T]) Decode(data []byte) (T, error) { + var out T + if err := json.Unmarshal(data, &out); err != nil { + return out, err + } + return out, nil +} + +func (c JSONCodec[T]) EncodedSize() int { return -1 } +func (c JSONCodec[T]) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/ordered_float32_codec.go b/go-sdk/state/codec/ordered_float32_codec.go new file mode 100644 index 00000000..a8860613 --- /dev/null +++ b/go-sdk/state/codec/ordered_float32_codec.go @@ -0,0 +1,38 @@ +package codec + +import ( + "encoding/binary" + "fmt" + "math" +) + +type OrderedFloat32Codec struct{} + +func (c OrderedFloat32Codec) Encode(value float32) ([]byte, error) { + bits := math.Float32bits(value) + if (bits & (uint32(1) << 31)) != 0 { + bits = ^bits + } else { + bits ^= uint32(1) << 31 + } + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, bits) + return out, nil +} + +func (c OrderedFloat32Codec) Decode(data []byte) (float32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid ordered float32 payload length: %d", len(data)) + } + encoded := binary.BigEndian.Uint32(data) + if (encoded & (uint32(1) << 31)) != 0 { + encoded ^= uint32(1) << 31 + } else { + encoded = ^encoded + } + return math.Float32frombits(encoded), nil +} + +func (c OrderedFloat32Codec) EncodedSize() int { return 4 } + +func (c OrderedFloat32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_float64_codec.go b/go-sdk/state/codec/ordered_float64_codec.go new file mode 100644 index 00000000..f481c7d4 --- /dev/null +++ b/go-sdk/state/codec/ordered_float64_codec.go @@ -0,0 +1,38 @@ +package codec + +import ( + "encoding/binary" + "fmt" + "math" +) + +type OrderedFloat64Codec struct{} + +func (c OrderedFloat64Codec) Encode(value float64) ([]byte, error) { + bits := math.Float64bits(value) + if (bits & (uint64(1) << 63)) != 0 { + bits = ^bits + } else { + bits ^= uint64(1) << 63 + } + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, bits) + return out, nil +} + +func (c OrderedFloat64Codec) Decode(data []byte) (float64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid ordered float64 payload length: %d", len(data)) + } + encoded := binary.BigEndian.Uint64(data) + if (encoded & (uint64(1) << 63)) != 0 { + encoded ^= uint64(1) << 63 + } else { + encoded = ^encoded + } + return math.Float64frombits(encoded), nil +} + +func (c OrderedFloat64Codec) EncodedSize() int { return 8 } + +func (c OrderedFloat64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int32_codec.go b/go-sdk/state/codec/ordered_int32_codec.go new file mode 100644 index 00000000..258f8bd4 --- /dev/null +++ b/go-sdk/state/codec/ordered_int32_codec.go @@ -0,0 +1,25 @@ +package codec + +import ( + "encoding/binary" + "fmt" +) + +type OrderedInt32Codec struct{} + +func (c OrderedInt32Codec) Encode(value int32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, uint32(value)^(uint32(1)<<31)) + return out, nil +} + +func (c OrderedInt32Codec) Decode(data []byte) (int32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid ordered int32 payload length: %d", len(data)) + } + return int32(binary.BigEndian.Uint32(data) ^ (uint32(1) << 31)), nil +} + +func (c OrderedInt32Codec) EncodedSize() int { return 4 } + +func (c OrderedInt32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int64_codec.go b/go-sdk/state/codec/ordered_int64_codec.go new file mode 100644 index 00000000..e228dbfc --- /dev/null +++ b/go-sdk/state/codec/ordered_int64_codec.go @@ -0,0 +1,15 @@ +package codec + +import "github.com/functionstream/function-stream/go-sdk/state/common" + +type OrderedInt64Codec struct{} + +func (c OrderedInt64Codec) Encode(value int64) ([]byte, error) { + return common.EncodeInt64Lex(value), nil +} + +func (c OrderedInt64Codec) Decode(data []byte) (int64, error) { return common.DecodeInt64Lex(data) } + +func (c OrderedInt64Codec) EncodedSize() int { return 8 } + +func (c OrderedInt64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int_codec.go b/go-sdk/state/codec/ordered_int_codec.go new file mode 100644 index 00000000..35899ee7 --- /dev/null +++ b/go-sdk/state/codec/ordered_int_codec.go @@ -0,0 +1,25 @@ +package codec + +import "strconv" + +type OrderedIntCodec struct{} + +func (c OrderedIntCodec) Encode(value int) ([]byte, error) { + if strconv.IntSize == 32 { + return OrderedInt32Codec{}.Encode(int32(value)) + } + return OrderedInt64Codec{}.Encode(int64(value)) +} + +func (c OrderedIntCodec) Decode(data []byte) (int, error) { + if strconv.IntSize == 32 { + v, err := OrderedInt32Codec{}.Decode(data) + return int(v), err + } + v, err := OrderedInt64Codec{}.Decode(data) + return int(v), err +} + +func (c OrderedIntCodec) EncodedSize() int { return strconv.IntSize / 8 } + +func (c OrderedIntCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint32_codec.go b/go-sdk/state/codec/ordered_uint32_codec.go new file mode 100644 index 00000000..f115e039 --- /dev/null +++ b/go-sdk/state/codec/ordered_uint32_codec.go @@ -0,0 +1,25 @@ +package codec + +import ( + "encoding/binary" + "fmt" +) + +type OrderedUint32Codec struct{} + +func (c OrderedUint32Codec) Encode(value uint32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, value) + return out, nil +} + +func (c OrderedUint32Codec) Decode(data []byte) (uint32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid ordered uint32 payload length: %d", len(data)) + } + return binary.BigEndian.Uint32(data), nil +} + +func (c OrderedUint32Codec) EncodedSize() int { return 4 } + +func (c OrderedUint32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint64_codec.go b/go-sdk/state/codec/ordered_uint64_codec.go new file mode 100644 index 00000000..f3fda43d --- /dev/null +++ b/go-sdk/state/codec/ordered_uint64_codec.go @@ -0,0 +1,25 @@ +package codec + +import ( + "encoding/binary" + "fmt" +) + +type OrderedUint64Codec struct{} + +func (c OrderedUint64Codec) Encode(value uint64) ([]byte, error) { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, value) + return out, nil +} + +func (c OrderedUint64Codec) Decode(data []byte) (uint64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid ordered uint64 payload length: %d", len(data)) + } + return binary.BigEndian.Uint64(data), nil +} + +func (c OrderedUint64Codec) EncodedSize() int { return 8 } + +func (c OrderedUint64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint_codec.go b/go-sdk/state/codec/ordered_uint_codec.go new file mode 100644 index 00000000..507ccb68 --- /dev/null +++ b/go-sdk/state/codec/ordered_uint_codec.go @@ -0,0 +1,25 @@ +package codec + +import "strconv" + +type OrderedUintCodec struct{} + +func (c OrderedUintCodec) Encode(value uint) ([]byte, error) { + if strconv.IntSize == 32 { + return OrderedUint32Codec{}.Encode(uint32(value)) + } + return OrderedUint64Codec{}.Encode(uint64(value)) +} + +func (c OrderedUintCodec) Decode(data []byte) (uint, error) { + if strconv.IntSize == 32 { + v, err := OrderedUint32Codec{}.Decode(data) + return uint(v), err + } + v, err := OrderedUint64Codec{}.Decode(data) + return uint(v), err +} + +func (c OrderedUintCodec) EncodedSize() int { return strconv.IntSize / 8 } + +func (c OrderedUintCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/string_codec.go b/go-sdk/state/codec/string_codec.go new file mode 100644 index 00000000..d530e668 --- /dev/null +++ b/go-sdk/state/codec/string_codec.go @@ -0,0 +1,11 @@ +package codec + +type StringCodec struct{} + +func (c StringCodec) Encode(value string) ([]byte, error) { return []byte(value), nil } + +func (c StringCodec) Decode(data []byte) (string, error) { return string(data), nil } + +func (c StringCodec) EncodedSize() int { return -1 } + +func (c StringCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/uint32_codec.go b/go-sdk/state/codec/uint32_codec.go new file mode 100644 index 00000000..5e6d3aaa --- /dev/null +++ b/go-sdk/state/codec/uint32_codec.go @@ -0,0 +1,24 @@ +package codec + +import ( + "encoding/binary" + "fmt" +) + +type Uint32Codec struct{} + +func (c Uint32Codec) Encode(value uint32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, value) + return out, nil +} + +func (c Uint32Codec) Decode(data []byte) (uint32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid uint32 payload length: %d", len(data)) + } + return binary.BigEndian.Uint32(data), nil +} + +func (c Uint32Codec) EncodedSize() int { return 4 } +func (c Uint32Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/uint64_codec.go b/go-sdk/state/codec/uint64_codec.go new file mode 100644 index 00000000..c7aed98a --- /dev/null +++ b/go-sdk/state/codec/uint64_codec.go @@ -0,0 +1,24 @@ +package codec + +import ( + "encoding/binary" + "fmt" +) + +type Uint64Codec struct{} + +func (c Uint64Codec) Encode(value uint64) ([]byte, error) { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, value) + return out, nil +} + +func (c Uint64Codec) Decode(data []byte) (uint64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid uint64 payload length: %d", len(data)) + } + return binary.BigEndian.Uint64(data), nil +} + +func (c Uint64Codec) EncodedSize() int { return 8 } +func (c Uint64Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/common/common.go b/go-sdk/state/common/common.go new file mode 100644 index 00000000..de4a23b0 --- /dev/null +++ b/go-sdk/state/common/common.go @@ -0,0 +1,66 @@ +package common + +import ( + "encoding/binary" + "fmt" + "strings" + + "github.com/functionstream/function-stream/go-sdk/api" +) + +type Store = api.Store + +const ( + StateValuePrefix = "__fssdk__/value/" + StateListPrefix = "__fssdk__/list/" + StatePQPrefix = "__fssdk__/priority_queue/" + StateMapGroup = "__fssdk__/map" + StateListGroup = "__fssdk__/list" + StatePQGroup = "__fssdk__/priority_queue" + StateAggregatingPrefix = "__fssdk__/aggregating/" + StateReducingPrefix = "__fssdk__/reducing/" +) + +func ValidateStateName(name string) (string, error) { + stateName := strings.TrimSpace(name) + if stateName == "" { + return "", api.NewError(api.ErrStoreInvalidName, "state name must not be empty") + } + return stateName, nil +} + +func EncodeInt64Lex(v int64) []byte { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, uint64(v)^(uint64(1)<<63)) + return out +} + +func DecodeInt64Lex(data []byte) (int64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid int64 lex key length: %d", len(data)) + } + return int64(binary.BigEndian.Uint64(data) ^ (uint64(1) << 63)), nil +} + +func EncodePriorityUserKey(priority int64, seq uint64) []byte { + out := make([]byte, 16) + copy(out[:8], EncodeInt64Lex(priority)) + binary.BigEndian.PutUint64(out[8:], seq) + return out +} + +func DecodePriorityUserKey(data []byte) (int64, error) { + if len(data) != 16 { + return 0, fmt.Errorf("invalid priority key length: %d", len(data)) + } + return DecodeInt64Lex(data[:8]) +} + +func DupBytes(input []byte) []byte { + if input == nil { + return nil + } + out := make([]byte, len(input)) + copy(out, input) + return out +} diff --git a/go-sdk/state/context.go b/go-sdk/state/context.go new file mode 100644 index 00000000..9ef76d3c --- /dev/null +++ b/go-sdk/state/context.go @@ -0,0 +1,137 @@ +package state + +import "fmt" + +const DefaultStateStoreName = "__fssdk_structured_state__" + +func NewValueStateFromContext[T any](ctx Context, stateName string, codec Codec[T]) (*ValueState[T], error) { + return NewValueStateFromContextWithStore[T](ctx, DefaultStateStoreName, stateName, codec) +} + +func NewValueStateFromContextWithStore[T any](ctx Context, storeName string, stateName string, codec Codec[T]) (*ValueState[T], error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewValueState[T](store, stateName, codec) +} + +func NewMapStateFromContext[K any, V any](ctx Context, stateName string, keyCodec Codec[K], valueCodec Codec[V]) (*MapState[K, V], error) { + return NewMapStateFromContextWithStore[K, V](ctx, DefaultStateStoreName, stateName, keyCodec, valueCodec) +} + +func NewMapStateFromContextWithStore[K any, V any](ctx Context, storeName string, stateName string, keyCodec Codec[K], valueCodec Codec[V]) (*MapState[K, V], error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewMapState[K, V](store, stateName, keyCodec, valueCodec) +} + +func NewListStateFromContext[T any](ctx Context, stateName string, codec Codec[T]) (*ListState[T], error) { + return NewListStateFromContextWithStore[T](ctx, DefaultStateStoreName, stateName, codec) +} + +func NewListStateFromContextWithStore[T any](ctx Context, storeName string, stateName string, codec Codec[T]) (*ListState[T], error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewListState[T](store, stateName, codec) +} + +func NewPriorityQueueStateFromContext[T any](ctx Context, stateName string, codec Codec[T]) (*PriorityQueueState[T], error) { + return NewPriorityQueueStateFromContextWithStore[T](ctx, DefaultStateStoreName, stateName, codec) +} + +func NewPriorityQueueStateFromContextWithStore[T any](ctx Context, storeName string, stateName string, codec Codec[T]) (*PriorityQueueState[T], error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewPriorityQueueState[T](store, stateName, codec) +} + +func NewKeyedStateFactoryFromContext(ctx Context, stateName string) (*KeyedStateFactory, error) { + return NewKeyedStateFactoryFromContextWithStore(ctx, DefaultStateStoreName, stateName) +} + +func NewKeyedStateFactoryFromContextWithStore(ctx Context, storeName string, stateName string) (*KeyedStateFactory, error) { + return NewKeyedValueStateFactoryFromContextWithStore(ctx, storeName, stateName) +} + +func NewKeyedValueStateFactoryFromContext(ctx Context, stateName string) (*KeyedValueStateFactory, error) { + return NewKeyedValueStateFactoryFromContextWithStore(ctx, DefaultStateStoreName, stateName) +} + +func NewKeyedValueStateFactoryFromContextWithStore(ctx Context, storeName string, stateName string) (*KeyedValueStateFactory, error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewKeyedValueStateFactory(store, stateName) +} + +func NewKeyedMapStateFactoryFromContext[MK any, MV any](ctx Context, stateName string, mapKeyCodec Codec[MK], mapValueCodec Codec[MV]) (*KeyedMapStateFactory[MK, MV], error) { + return NewKeyedMapStateFactoryFromContextWithStore[MK, MV](ctx, DefaultStateStoreName, stateName, mapKeyCodec, mapValueCodec) +} + +func NewKeyedMapStateFactoryFromContextWithStore[MK any, MV any](ctx Context, storeName string, stateName string, mapKeyCodec Codec[MK], mapValueCodec Codec[MV]) (*KeyedMapStateFactory[MK, MV], error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewKeyedMapStateFactory[MK, MV](store, stateName, mapKeyCodec, mapValueCodec) +} + +func NewKeyedListStateFactoryFromContext[V any](ctx Context, stateName string, keyGroup []byte, valueCodec Codec[V]) (*KeyedListStateFactory[V], error) { + return NewKeyedListStateFactoryFromContextWithStore[V](ctx, DefaultStateStoreName, stateName, keyGroup, valueCodec) +} + +func NewKeyedListStateFactoryFromContextWithStore[V any](ctx Context, storeName string, stateName string, keyGroup []byte, valueCodec Codec[V]) (*KeyedListStateFactory[V], error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewKeyedListStateFactory[V](store, stateName, keyGroup, valueCodec) +} + +// NewKeyedListStateFactoryAutoCodecFromContext 从 Context 创建 KeyedListStateFactory，不显式传入 valueCodec， +// 由工厂根据类型 V 自动选择 codec（基础类型 / string / struct 用 JSON）。 +func NewKeyedListStateFactoryAutoCodecFromContext[V any](ctx Context, stateName string, keyGroup []byte) (*KeyedListStateFactory[V], error) { + return NewKeyedListStateFactoryAutoCodecFromContextWithStore[V](ctx, DefaultStateStoreName, stateName, keyGroup) +} + +func NewKeyedListStateFactoryAutoCodecFromContextWithStore[V any](ctx Context, storeName string, stateName string, keyGroup []byte) (*KeyedListStateFactory[V], error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewKeyedListStateFactoryAutoCodec[V](store, stateName, keyGroup) +} + +func NewKeyedPriorityQueueStateFactoryFromContext(ctx Context, stateName string) (*KeyedPriorityQueueStateFactory, error) { + return NewKeyedPriorityQueueStateFactoryFromContextWithStore(ctx, DefaultStateStoreName, stateName) +} + +func NewKeyedPriorityQueueStateFactoryFromContextWithStore(ctx Context, storeName string, stateName string) (*KeyedPriorityQueueStateFactory, error) { + store, err := requireStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return NewKeyedPriorityQueueStateFactory(store, stateName) +} + +func requireStoreFromContext(ctx Context, storeName string) (Store, error) { + if ctx == nil { + return nil, fmt.Errorf("context must not be nil") + } + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + if store == nil { + return nil, fmt.Errorf("context returned nil store for %q", storeName) + } + return store, nil +} diff --git a/go-sdk/state/keyed/keyed_aggregating_state.go b/go-sdk/state/keyed/keyed_aggregating_state.go new file mode 100644 index 00000000..930840bc --- /dev/null +++ b/go-sdk/state/keyed/keyed_aggregating_state.go @@ -0,0 +1,113 @@ +package keyed + +import ( + "fmt" + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type AggregateFunc[T any, ACC any, R any] interface { + CreateAccumulator() ACC + Add(value T, accumulator ACC) ACC + GetResult(accumulator ACC) R + Merge(a ACC, b ACC) ACC +} + +type KeyedAggregatingStateFactory[T any, ACC any, R any] struct { + inner *keyedStateFactory + groupKey []byte + accCodec codec.Codec[ACC] + aggFunc AggregateFunc[T, ACC, R] +} + +func NewKeyedAggregatingStateFactory[T any, ACC any, R any]( + store common.Store, + keyGroup []byte, + accCodec codec.Codec[ACC], + aggFunc AggregateFunc[T, ACC, R], +) (*KeyedAggregatingStateFactory[T, ACC, R], error) { + + inner, err := newKeyedStateFactory(store, "", "aggregating") + if err != nil { + return nil, err + } + + return &KeyedAggregatingStateFactory[T, ACC, R]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + accCodec: accCodec, + aggFunc: aggFunc, + }, nil +} + +func (f *KeyedAggregatingStateFactory[T, ACC, R]) NewAggregatingState(primaryKey []byte, stateName string) (*KeyedAggregatingState[T, ACC, R], error) { + return &KeyedAggregatingState[T, ACC, R]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: []byte(stateName), + }, nil +} + +type KeyedAggregatingState[T any, ACC any, R any] struct { + factory *KeyedAggregatingStateFactory[T, ACC, R] + primaryKey []byte + namespace []byte +} + +func (s *KeyedAggregatingState[T, ACC, R]) buildCK() api.ComplexKey { + return api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + } +} + +func (s *KeyedAggregatingState[T, ACC, R]) Add(value T) error { + ck := s.buildCK() + + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil { + return fmt.Errorf("failed to get accumulator: %w", err) + } + + var acc ACC + if !found { + acc = s.factory.aggFunc.CreateAccumulator() + } else { + var err error + acc, err = s.factory.accCodec.Decode(raw) + if err != nil { + return fmt.Errorf("failed to decode accumulator: %w", err) + } + } + + newAcc := s.factory.aggFunc.Add(value, acc) + + encoded, err := s.factory.accCodec.Encode(newAcc) + if err != nil { + return fmt.Errorf("failed to encode new accumulator: %w", err) + } + return s.factory.inner.store.Put(ck, encoded) +} + +func (s *KeyedAggregatingState[T, ACC, R]) Get() (R, bool, error) { + var zero R + ck := s.buildCK() + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + + acc, err := s.factory.accCodec.Decode(raw) + if err != nil { + return zero, false, err + } + + return s.factory.aggFunc.GetResult(acc), true, nil +} + +func (s *KeyedAggregatingState[T, ACC, R]) Clear() error { + return s.factory.inner.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/keyed/keyed_list_state.go b/go-sdk/state/keyed/keyed_list_state.go new file mode 100644 index 00000000..b52dc6d4 --- /dev/null +++ b/go-sdk/state/keyed/keyed_list_state.go @@ -0,0 +1,257 @@ +package keyed + +import ( + "encoding/binary" + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type KeyedListStateFactory[V any] struct { + inner *keyedStateFactory + keyGroup []byte + fixedSize int + valueCodec codec.Codec[V] + isFixed bool +} + +func NewKeyedListStateFactory[V any](store common.Store, name string, keyGroup []byte, valueCodec codec.Codec[V]) (*KeyedListStateFactory[V], error) { + inner, err := newKeyedStateFactory(store, name, "list") + if err != nil { + return nil, err + } + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list state factory %q key group must not be nil", inner.name) + } + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list value codec must not be nil") + } + fixedSize, isFixed := codec.FixedEncodedSize[V](valueCodec) + return &KeyedListStateFactory[V]{ + inner: inner, + keyGroup: common.DupBytes(keyGroup), + fixedSize: fixedSize, + valueCodec: valueCodec, + isFixed: isFixed, + }, nil +} + +func NewKeyedListStateFactoryAutoCodec[V any](store common.Store, name string, keyGroup []byte) (*KeyedListStateFactory[V], error) { + valueCodec, err := codec.DefaultCodecFor[V]() + if err != nil { + return nil, err + } + return NewKeyedListStateFactory[V](store, name, keyGroup, valueCodec) +} + +type KeyedListState[V any] struct { + factory *KeyedListStateFactory[V] + complexKey api.ComplexKey + fixedSize int + valueCodec codec.Codec[V] + serialize func(V) ([]byte, error) + serializeBatch func([]V) ([]byte, error) + decode func([]byte) ([]V, error) +} + +func newKeyedListFromFactory[V any](f *KeyedListStateFactory[V], key []byte, namespace []byte) (*KeyedListState[V], error) { + if f == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list factory must not be nil") + } + if key == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list key must not be nil") + } + if namespace == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list namespace must not be nil") + } + s := &KeyedListState[V]{ + factory: f, + valueCodec: f.valueCodec, + complexKey: api.ComplexKey{ + KeyGroup: f.keyGroup, + Key: key, + Namespace: namespace, + UserKey: []byte{}, + }, + fixedSize: f.fixedSize, + } + if f.isFixed { + s.serialize = s.serializeValueFixed + s.serializeBatch = s.serializeValuesFixedBatch + s.decode = s.deserializeValuesFixed + } else { + s.serialize = s.serializeValueVarLen + s.serializeBatch = s.serializeValuesVarLenBatch + s.decode = s.deserializeValuesVarLen + } + return s, nil +} + +func NewKeyedListFromFactory[V any](f *KeyedListStateFactory[V], key []byte, namespace []byte) (*KeyedListState[V], error) { + return newKeyedListFromFactory[V](f, key, namespace) +} + +func (s *KeyedListState[V]) Add(value V) error { + payload, err := s.serialize(value) + if err != nil { + return err + } + return s.factory.inner.store.Merge(s.complexKey, payload) +} + +func (s *KeyedListState[V]) AddAll(values []V) error { + payload, err := s.serializeBatch(values) + if err != nil { + return err + } + if err := s.factory.inner.store.Merge(s.complexKey, payload); err != nil { + return err + } + return nil +} + +func (s *KeyedListState[V]) Get() ([]V, error) { + raw, found, err := s.factory.inner.store.Get(s.complexKey) + if err != nil { + return nil, err + } + if !found { + return []V{}, nil + } + return s.decode(raw) +} + +// Update replaces the list with the given values (one Put with batch payload). +func (s *KeyedListState[V]) Update(values []V) error { + if len(values) == 0 { + return s.Clear() + } + payload, err := s.serializeBatch(values) + if err != nil { + return err + } + return s.factory.inner.store.Put(s.complexKey, payload) +} + +func (s *KeyedListState[V]) Clear() error { + return s.factory.inner.store.Delete(s.complexKey) +} + +func (s *KeyedListState[V]) serializeValueVarLen(value V) ([]byte, error) { + encoded, err := s.valueCodec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode keyed list value failed: %w", err) + } + out := make([]byte, 4, 4+len(encoded)) + binary.BigEndian.PutUint32(out, uint32(len(encoded))) + out = append(out, encoded...) + return out, nil +} + +func (s *KeyedListState[V]) serializeValuesVarLenBatch(values []V) ([]byte, error) { + total := 0 + encodedValues := make([][]byte, 0, len(values)) + for _, value := range values { + encoded, err := s.valueCodec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode keyed list value failed: %w", err) + } + encodedValues = append(encodedValues, encoded) + total += 4 + len(encoded) + } + out := make([]byte, 0, total) + for _, encoded := range encodedValues { + var lenBuf [4]byte + binary.BigEndian.PutUint32(lenBuf[:], uint32(len(encoded))) + out = append(out, lenBuf[:]...) + out = append(out, encoded...) + } + return out, nil +} + +func (s *KeyedListState[V]) deserializeValuesVarLen(raw []byte) ([]V, error) { + out := make([]V, 0, 16) + idx := 0 + for idx < len(raw) { + if len(raw)-idx < 4 { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted keyed list payload: truncated length") + } + + itemLen := int(binary.BigEndian.Uint32(raw[idx : idx+4])) + idx += 4 + + if itemLen < 0 || len(raw)-idx < itemLen { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted keyed list payload: invalid element length") + } + + itemRaw := raw[idx : idx+itemLen] + idx += itemLen + + value, err := s.valueCodec.Decode(itemRaw) + if err != nil { + return nil, fmt.Errorf("decode keyed list value failed: %w", err) + } + out = append(out, value) + } + return out, nil +} + +func (s *KeyedListState[V]) serializeValueFixed(value V) ([]byte, error) { + if s.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + encoded, err := s.valueCodec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode keyed list value failed: %w", err) + } + if len(encoded) != s.fixedSize { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec encoded unexpected length: got %d, want %d", len(encoded), s.fixedSize) + } + out := make([]byte, 0, s.fixedSize) + out = append(out, encoded...) + return out, nil +} + +func (s *KeyedListState[V]) serializeValuesFixedBatch(values []V) ([]byte, error) { + if s.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + total := s.fixedSize * len(values) + out := make([]byte, 0, total) + for _, value := range values { + encoded, err := s.valueCodec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode keyed list value failed: %w", err) + } + if len(encoded) != s.fixedSize { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec encoded unexpected length: got %d, want %d", len(encoded), s.fixedSize) + } + out = append(out, encoded...) + } + return out, nil +} + +func (s *KeyedListState[V]) deserializeValuesFixed(raw []byte) ([]V, error) { + if s.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + + if len(raw)%s.fixedSize != 0 { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted keyed list payload: fixed-size data length mismatch") + } + + count := len(raw) / s.fixedSize + out := make([]V, 0, count) + + for idx := 0; idx < len(raw); idx += s.fixedSize { + itemRaw := raw[idx : idx+s.fixedSize] + value, err := s.valueCodec.Decode(itemRaw) + if err != nil { + return nil, fmt.Errorf("decode keyed list value failed: %w", err) + } + out = append(out, value) + } + return out, nil +} diff --git a/go-sdk/state/keyed/keyed_map_state.go b/go-sdk/state/keyed/keyed_map_state.go new file mode 100644 index 00000000..ffc6e1c4 --- /dev/null +++ b/go-sdk/state/keyed/keyed_map_state.go @@ -0,0 +1,153 @@ +package keyed + +import ( + "fmt" + "iter" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type KeyedMapStateFactory[MK any, MV any] struct { + inner *keyedStateFactory + groupKey []byte + mapKeyCodec codec.Codec[MK] + mapValueCodec codec.Codec[MV] +} + +func NewKeyedMapStateFactory[MK any, MV any]( + store common.Store, + keyGroup []byte, + mapKeyCodec codec.Codec[MK], + mapValueCodec codec.Codec[MV], +) (*KeyedMapStateFactory[MK, MV], error) { + + inner, err := newKeyedStateFactory(store, "", "map") + if err != nil { + return nil, err + } + + if mapKeyCodec == nil || mapValueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "codec must not be nil") + } + + if !mapKeyCodec.IsOrderedKeyCodec() { + return nil, api.NewError(api.ErrStoreInternal, "map key codec must be ordered") + } + + return &KeyedMapStateFactory[MK, MV]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + mapKeyCodec: mapKeyCodec, + mapValueCodec: mapValueCodec, + }, nil +} + +type KeyedMapState[MK any, MV any] struct { + factory *KeyedMapStateFactory[MK, MV] + primaryKey []byte + namespace []byte +} + +func (f *KeyedMapStateFactory[MK, MV]) NewKeyedMap(primaryKey []byte, mapName string) (*KeyedMapState[MK, MV], error) { + if primaryKey == nil || mapName == "" { + return nil, api.NewError(api.ErrStoreInternal, "primary key and map name are required") + } + return &KeyedMapState[MK, MV]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: []byte(mapName), + }, nil +} + +func (s *KeyedMapState[MK, MV]) buildCK(mapKey MK) (api.ComplexKey, error) { + encodedMapKey, err := s.factory.mapKeyCodec.Encode(mapKey) + if err != nil { + return api.ComplexKey{}, fmt.Errorf("encode map userKey failed: %w", err) + } + return api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: encodedMapKey, + }, nil +} + +func (s *KeyedMapState[MK, MV]) Put(mapKey MK, value MV) error { + ck, err := s.buildCK(mapKey) + if err != nil { + return err + } + encodedValue, err := s.factory.mapValueCodec.Encode(value) + if err != nil { + return err + } + return s.factory.inner.store.Put(ck, encodedValue) +} + +func (s *KeyedMapState[MK, MV]) Get(mapKey MK) (MV, bool, error) { + var zero MV + ck, err := s.buildCK(mapKey) + if err != nil { + return zero, false, err + } + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + decoded, err := s.factory.mapValueCodec.Decode(raw) + if err != nil { + return zero, false, err + } + return decoded, true, nil +} + +func (s *KeyedMapState[MK, MV]) Delete(mapKey MK) error { + ck, err := s.buildCK(mapKey) + if err != nil { + return err + } + return s.factory.inner.store.Delete(ck) +} + +func (s *KeyedMapState[MK, MV]) Clear() error { + return s.factory.inner.store.DeletePrefix(api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + }) +} + +func (s *KeyedMapState[MK, MV]) All() iter.Seq2[MK, MV] { + return func(yield func(MK, MV) bool) { + iter, err := s.factory.inner.store.ScanComplex( + s.factory.groupKey, + s.primaryKey, + s.namespace, + ) + if err != nil { + return + } + defer iter.Close() + + for { + has, err := iter.HasNext() + if err != nil || !has { + return + } + keyRaw, valRaw, ok, err := iter.Next() + if err != nil || !ok { + return + } + + k, _ := s.factory.mapKeyCodec.Decode(keyRaw) + v, _ := s.factory.mapValueCodec.Decode(valRaw) + + if !yield(k, v) { + return + } + } + } +} diff --git a/go-sdk/state/keyed/keyed_priority_queue_state.go b/go-sdk/state/keyed/keyed_priority_queue_state.go new file mode 100644 index 00000000..94790bdb --- /dev/null +++ b/go-sdk/state/keyed/keyed_priority_queue_state.go @@ -0,0 +1,162 @@ +package keyed + +import ( + "fmt" + "iter" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type KeyedPriorityQueueStateFactory[V any] struct { + inner *keyedStateFactory + groupKey []byte + valueCodec codec.Codec[V] +} + +func NewKeyedPriorityQueueStateFactory[V any]( + store common.Store, + keyGroup []byte, + valueCodec codec.Codec[V], +) (*KeyedPriorityQueueStateFactory[V], error) { + + inner, err := newKeyedStateFactory(store, "", "pq") + if err != nil { + return nil, err + } + + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "value codec must not be nil") + } + + if !valueCodec.IsOrderedKeyCodec() { + return nil, api.NewError(api.ErrStoreInternal, "priority queue value codec must be ordered") + } + + return &KeyedPriorityQueueStateFactory[V]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + valueCodec: valueCodec, + }, nil +} + +type KeyedPriorityQueueState[V any] struct { + factory *KeyedPriorityQueueStateFactory[V] + primaryKey []byte + namespace []byte +} + +func (f *KeyedPriorityQueueStateFactory[V]) NewKeyedPriorityQueue(primaryKey []byte, namespace []byte) (*KeyedPriorityQueueState[V], error) { + if primaryKey == nil || namespace == nil { + return nil, api.NewError(api.ErrStoreInternal, "primary key and queue name are required") + } + return &KeyedPriorityQueueState[V]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: common.DupBytes(namespace), + }, nil +} + +func (s *KeyedPriorityQueueState[V]) Add(value V) error { + userKey, err := s.factory.valueCodec.Encode(value) + if err != nil { + return fmt.Errorf("encode pq element failed: %w", err) + } + + ck := api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: userKey, + } + + return s.factory.inner.store.Put(ck, []byte{}) +} + +func (s *KeyedPriorityQueueState[V]) Peek() (V, bool, error) { + var zero V + + iter, err := s.factory.inner.store.ScanComplex( + s.factory.groupKey, + s.primaryKey, + s.namespace, + ) + if err != nil { + return zero, false, err + } + defer iter.Close() + + has, err := iter.HasNext() + if err != nil || !has { + return zero, false, err + } + + userKey, _, ok, err := iter.Next() + if err != nil || !ok { + return zero, false, err + } + + val, err := s.factory.valueCodec.Decode(userKey) + if err != nil { + return zero, false, err + } + return val, true, nil +} + +func (s *KeyedPriorityQueueState[V]) Poll() (V, bool, error) { + val, found, err := s.Peek() + if err != nil || !found { + return val, found, err + } + + userKey, _ := s.factory.valueCodec.Encode(val) + ck := api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: userKey, + } + + err = s.factory.inner.store.Delete(ck) + return val, true, err +} + +func (s *KeyedPriorityQueueState[V]) Clear() error { + return s.factory.inner.store.DeletePrefix(api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + }) +} + +func (s *KeyedPriorityQueueState[V]) All() iter.Seq[V] { + return func(yield func(V) bool) { + iter, err := s.factory.inner.store.ScanComplex( + s.factory.groupKey, + s.primaryKey, + s.namespace, + ) + if err != nil { + return + } + defer iter.Close() + + for { + has, err := iter.HasNext() + if err != nil || !has { + return + } + userKey, _, ok, err := iter.Next() + if err != nil || !ok { + return + } + + v, _ := s.factory.valueCodec.Decode(userKey) + if !yield(v) { + return + } + } + } +} \ No newline at end of file diff --git a/go-sdk/state/keyed/keyed_reducing_state.go b/go-sdk/state/keyed/keyed_reducing_state.go new file mode 100644 index 00000000..3911ec32 --- /dev/null +++ b/go-sdk/state/keyed/keyed_reducing_state.go @@ -0,0 +1,116 @@ +package keyed + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type ReduceFunc[V any] func(value1 V, value2 V) (V, error) + +type KeyedReducingStateFactory[V any] struct { + inner *keyedStateFactory + groupKey []byte + valueCodec codec.Codec[V] + reduceFunc ReduceFunc[V] +} + +func NewKeyedReducingStateFactory[V any]( + store common.Store, + keyGroup []byte, + valueCodec codec.Codec[V], + reduceFunc ReduceFunc[V], +) (*KeyedReducingStateFactory[V], error) { + + inner, err := newKeyedStateFactory(store, "", "reducing") + if err != nil { + return nil, err + } + + if valueCodec == nil || reduceFunc == nil { + return nil, api.NewError(api.ErrStoreInternal, "value codec and reduce function are required") + } + + return &KeyedReducingStateFactory[V]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + valueCodec: valueCodec, + reduceFunc: reduceFunc, + }, nil +} + +func (f *KeyedReducingStateFactory[V]) NewReducingState(primaryKey []byte, namespace []byte) (*KeyedReducingState[V], error) { + if primaryKey == nil || namespace == nil { + return nil, api.NewError(api.ErrStoreInternal, "primary key and state name are required") + } + return &KeyedReducingState[V]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: common.DupBytes(namespace), + }, nil +} + +type KeyedReducingState[V any] struct { + factory *KeyedReducingStateFactory[V] + primaryKey []byte + namespace []byte +} + +func (s *KeyedReducingState[V]) buildCK() api.ComplexKey { + return api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + } +} + +func (s *KeyedReducingState[V]) Add(value V) error { + ck := s.buildCK() + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil { + return fmt.Errorf("failed to get old value for reducing state: %w", err) + } + + var result V + if !found { + result = value + } else { + oldValue, err := s.factory.valueCodec.Decode(raw) + if err != nil { + return fmt.Errorf("failed to decode old value: %w", err) + } + + result, err = s.factory.reduceFunc(oldValue, value) + if err != nil { + return fmt.Errorf("error in user reduce function: %w", err) + } + } + + encoded, err := s.factory.valueCodec.Encode(result) + if err != nil { + return fmt.Errorf("failed to encode reduced value: %w", err) + } + + return s.factory.inner.store.Put(ck, encoded) +} + +func (s *KeyedReducingState[V]) Get() (V, bool, error) { + var zero V + ck := s.buildCK() + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + val, err := s.factory.valueCodec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("failed to decode value: %w", err) + } + return val, true, nil +} + +func (s *KeyedReducingState[V]) Clear() error { + return s.factory.inner.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/keyed/keyed_state_factory.go b/go-sdk/state/keyed/keyed_state_factory.go new file mode 100644 index 00000000..914e2418 --- /dev/null +++ b/go-sdk/state/keyed/keyed_state_factory.go @@ -0,0 +1,39 @@ +package keyed + +import ( + "encoding/base64" + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type keyedStateFactory struct { + store common.Store + name string +} + +func newKeyedStateFactory(store common.Store, name string, kind string) (*keyedStateFactory, error) { + stateName, err := common.ValidateStateName(name) + if err != nil { + return nil, err + } + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed %s state factory %q store must not be nil", kind, stateName) + } + return &keyedStateFactory{store: store, name: stateName}, nil +} + + + +func keyedSubStateName[K any](base string, keyCodec codec.Codec[K], key K, kind string) (string, error) { + if keyCodec == nil { + return "", api.NewError(api.ErrStoreInternal, "key codec must not be nil") + } + encoded, err := keyCodec.Encode(key) + if err != nil { + return "", fmt.Errorf("encode keyed state key failed: %w", err) + } + return fmt.Sprintf("%s/%s/%s", base, kind, base64.RawURLEncoding.EncodeToString(encoded)), nil +} diff --git a/go-sdk/state/keyed/keyed_value_state.go b/go-sdk/state/keyed/keyed_value_state.go new file mode 100644 index 00000000..5e6c2187 --- /dev/null +++ b/go-sdk/state/keyed/keyed_value_state.go @@ -0,0 +1,89 @@ +package keyed + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) +type KeyedValueStateFactory[V any] struct { + inner *keyedStateFactory + groupKey []byte + valueCodec codec.Codec[V] +} + +func NewKeyedValueStateFactory[V any]( + store common.Store, + keyGroup []byte, + valueCodec codec.Codec[V], +) (*KeyedValueStateFactory[V], error) { + + inner, err := newKeyedStateFactory(store, "", "value") + if err != nil { + return nil, err + } + + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "value codec must not be nil") + } + + return &KeyedValueStateFactory[V]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + valueCodec: valueCodec, + }, nil +} + +type KeyedValueState[V any] struct { + factory *KeyedValueStateFactory[V] + primaryKey []byte + namespace []byte +} + +func (f *KeyedValueStateFactory[V]) NewKeyedValue(primaryKey []byte, stateName string) (*KeyedValueState[V], error) { + if primaryKey == nil || stateName == "" { + return nil, api.NewError(api.ErrStoreInternal, "primary key and state name are required") + } + return &KeyedValueState[V]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: []byte(stateName), + }, nil +} + +func (s *KeyedValueState[V]) buildCK() api.ComplexKey { + return api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + } +} + +func (s *KeyedValueState[V]) Update(value V) error { + ck := s.buildCK() + encoded, err := s.factory.valueCodec.Encode(value) + if err != nil { + return fmt.Errorf("encode value state failed: %w", err) + } + return s.factory.inner.store.Put(ck, encoded) +} + +func (s *KeyedValueState[V]) Value() (V, bool, error) { + var zero V + ck := s.buildCK() + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + decoded, err := s.factory.valueCodec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("decode value state failed: %w", err) + } + return decoded, true, nil +} + +func (s *KeyedValueState[V]) Clear() error { + return s.factory.inner.store.Delete(s.buildCK()) +} \ No newline at end of file diff --git a/go-sdk/state/state.go b/go-sdk/state/state.go new file mode 100644 index 00000000..2bfad332 --- /dev/null +++ b/go-sdk/state/state.go @@ -0,0 +1,51 @@ +package state + +import ( + "github.com/functionstream/function-stream/go-sdk/api" + statecodec "github.com/functionstream/function-stream/go-sdk/state/codec" +) + +type Store = api.Store +type Context = api.Context +// +//type Codec[T any] = statecodec.Codec[T] +// +//type JSONCodec[T any] = statecodec.JSONCodec[T] +type BytesCodec = statecodec.BytesCodec +type StringCodec = statecodec.StringCodec +type BoolCodec = statecodec.BoolCodec + +type Int64Codec = statecodec.Int64Codec +type Uint64Codec = statecodec.Uint64Codec +type Int32Codec = statecodec.Int32Codec +type Uint32Codec = statecodec.Uint32Codec +type Float64Codec = statecodec.Float64Codec +type Float32Codec = statecodec.Float32Codec + +type OrderedInt64Codec = statecodec.OrderedInt64Codec +type OrderedIntCodec = statecodec.OrderedIntCodec +type OrderedUint64Codec = statecodec.OrderedUint64Codec +type OrderedUintCodec = statecodec.OrderedUintCodec +type OrderedInt32Codec = statecodec.OrderedInt32Codec +type OrderedUint32Codec = statecodec.OrderedUint32Codec +type OrderedFloat64Codec = statecodec.OrderedFloat64Codec +type OrderedFloat32Codec = statecodec.OrderedFloat32Codec +// +//type ValueState[T any] = structuresstate.ValueState[T] +//type MapEntry[K any, V any] = structuresstate.MapEntry[K, V] +//type MapState[K any, V any] = structuresstate.MapState[K, V] +//type ListState[T any] = structuresstate.ListState[T] +//type PriorityQueueState[T any] = structuresstate.PriorityQueueState[T] +// +//type KeyedStateFactory = keyedstate.KeyedValueStateFactory +//type KeyedValueStateFactory = keyedstate.KeyedValueStateFactory +//type KeyedMapStateFactory[MK any, MV any] = keyedstate.KeyedMapStateFactory[MK, MV] +//type KeyedListStateFactory[V any] = keyedstate.KeyedListStateFactory[V] +//type KeyedPriorityQueueStateFactory = keyedstate.KeyedPriorityQueueStateFactory +//type KeyedValueState[K any, V any] = keyedstate.KeyedValueState[K, V] +//type KeyedMapEntry[K any, V any] = keyedstate.KeyedMapEntry[K, V] +//type KeyedMapState[MK any, MV any] = keyedstate.KeyedMapState[MK, MV] +//type KeyedListState[V any] = keyedstate.KeyedListState[V] +//type KeyedPriorityItem[T any] = keyedstate.KeyedPriorityItem[T] +//type KeyedPriorityQueueState[K any, V any] = keyedstate.KeyedPriorityQueueState[K, V] +// diff --git a/go-sdk/state/structures/aggregating.go b/go-sdk/state/structures/aggregating.go new file mode 100644 index 00000000..6e11c68c --- /dev/null +++ b/go-sdk/state/structures/aggregating.go @@ -0,0 +1,108 @@ +package structures + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type AggregateFunc[T any, ACC any, R any] interface { + CreateAccumulator() ACC + Add(value T, accumulator ACC) ACC + GetResult(accumulator ACC) R + Merge(a ACC, b ACC) ACC +} + +type AggregatingState[T any, ACC any, R any] struct { + store common.Store + complexKey api.ComplexKey + accCodec codec.Codec[ACC] + aggFunc AggregateFunc[T, ACC, R] +} + +func NewAggregatingState[T any, ACC any, R any]( + store common.Store, + name string, + accCodec codec.Codec[ACC], + aggFunc AggregateFunc[T, ACC, R], +) (*AggregatingState[T, ACC, R], error) { + stateName, err := common.ValidateStateName(name) + if err != nil { + return nil, err + } + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "aggregating state %q store must not be nil", stateName) + } + if accCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "aggregating state %q acc codec must not be nil", stateName) + } + if aggFunc == nil { + return nil, api.NewError(api.ErrStoreInternal, "aggregating state %q agg func must not be nil", stateName) + } + ck := api.ComplexKey{ + KeyGroup: []byte(common.StateAggregatingPrefix), + Key: []byte(stateName), + Namespace: []byte("data"), + UserKey: []byte{}, + } + return &AggregatingState[T, ACC, R]{ + store: store, + complexKey: ck, + accCodec: accCodec, + aggFunc: aggFunc, + }, nil +} + +func (s *AggregatingState[T, ACC, R]) buildCK() api.ComplexKey { + return s.complexKey +} + +func (s *AggregatingState[T, ACC, R]) Add(value T) error { + ck := s.buildCK() + + raw, found, err := s.store.Get(ck) + if err != nil { + return fmt.Errorf("failed to get accumulator: %w", err) + } + + var acc ACC + if !found { + acc = s.aggFunc.CreateAccumulator() + } else { + var err error + acc, err = s.accCodec.Decode(raw) + if err != nil { + return fmt.Errorf("failed to decode accumulator: %w", err) + } + } + + newAcc := s.aggFunc.Add(value, acc) + + encoded, err := s.accCodec.Encode(newAcc) + if err != nil { + return fmt.Errorf("failed to encode new accumulator: %w", err) + } + return s.store.Put(ck, encoded) +} + +func (s *AggregatingState[T, ACC, R]) Get() (R, bool, error) { + var zero R + ck := s.buildCK() + raw, found, err := s.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + + acc, err := s.accCodec.Decode(raw) + if err != nil { + return zero, false, err + } + + return s.aggFunc.GetResult(acc), true, nil +} + +func (s *AggregatingState[T, ACC, R]) Clear() error { + return s.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/structures/list.go b/go-sdk/state/structures/list.go new file mode 100644 index 00000000..2d5bbe55 --- /dev/null +++ b/go-sdk/state/structures/list.go @@ -0,0 +1,209 @@ +package structures + +import ( + "encoding/binary" + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type ListState[T any] struct { + store common.Store + complexKey api.ComplexKey + codec codec.Codec[T] + fixedSize int + serialize func(T) ([]byte, error) + decode func([]byte) ([]T, error) + serializeBatch func([]T) ([]byte, error) +} + +func NewListState[T any](store common.Store, name string, itemCodec codec.Codec[T]) (*ListState[T], error) { + stateName, err := common.ValidateStateName(name) + if err != nil { + return nil, err + } + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "list state %q store must not be nil", stateName) + } + if itemCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "list state %q codec must not be nil", stateName) + } + fixedSize, isFixed := codec.FixedEncodedSize[T](itemCodec) + l := &ListState[T]{ + store: store, + complexKey: api.ComplexKey{ + KeyGroup: []byte(common.StateListGroup), + Key: []byte(stateName), + Namespace: []byte(""), + UserKey: []byte{}, + }, + codec: itemCodec, + fixedSize: fixedSize, + } + if isFixed { + l.serialize = l.serializeValueFixed + l.serializeBatch = l.serializeValuesFixedBatch + l.decode = l.deserializeValuesFixed + } else { + l.serialize = l.serializeValueVarLen + l.serializeBatch = l.serializeValuesVarLenBatch + l.decode = l.deserializeValuesVarLen + } + return l, nil +} + +func (l *ListState[T]) Add(value T) error { + payload, err := l.serialize(value) + if err != nil { + return err + } + return l.store.Merge(l.complexKey, payload) +} + +func (l *ListState[T]) AddAll(values []T) error { + payload, err := l.serializeBatch(values) + if err != nil { + return err + } + if err := l.store.Merge(l.complexKey, payload); err != nil { + return err + } + return nil +} + +func (l *ListState[T]) Get() ([]T, error) { + raw, found, err := l.store.Get(l.complexKey) + if err != nil { + return nil, err + } + if !found { + return []T{}, nil + } + return l.decode(raw) +} + +func (l *ListState[T]) Update(values []T) error { + if len(values) == 0 { + return l.Clear() + } + payload, err := l.serializeBatch(values) + if err != nil { + return err + } + return l.store.Put(l.complexKey, payload) +} + +func (l *ListState[T]) Clear() error { + return l.store.Delete(l.complexKey) +} + +func (l *ListState[T]) serializeValueVarLen(value T) ([]byte, error) { + encoded, err := l.codec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode list value failed: %w", err) + } + out := make([]byte, 4, 4+len(encoded)) + binary.BigEndian.PutUint32(out, uint32(len(encoded))) + out = append(out, encoded...) + return out, nil +} + +func (l *ListState[T]) serializeValuesVarLenBatch(values []T) ([]byte, error) { + total := 0 + encodedValues := make([][]byte, 0, len(values)) + for _, value := range values { + encoded, err := l.codec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode list value failed: %w", err) + } + encodedValues = append(encodedValues, encoded) + total += 4 + len(encoded) + } + out := make([]byte, 0, total) + for _, encoded := range encodedValues { + var lenBuf [4]byte + binary.BigEndian.PutUint32(lenBuf[:], uint32(len(encoded))) + out = append(out, lenBuf[:]...) + out = append(out, encoded...) + } + return out, nil +} + +func (l *ListState[T]) deserializeValuesVarLen(raw []byte) ([]T, error) { + out := make([]T, 0, 16) + idx := 0 + for idx < len(raw) { + if len(raw)-idx < 4 { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted list payload: truncated length") + } + itemLen := int(binary.BigEndian.Uint32(raw[idx : idx+4])) + idx += 4 + if itemLen < 0 || len(raw)-idx < itemLen { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted list payload: invalid element length") + } + itemRaw := raw[idx : idx+itemLen] + idx += itemLen + value, err := l.codec.Decode(itemRaw) + if err != nil { + return nil, fmt.Errorf("decode list value failed: %w", err) + } + out = append(out, value) + } + return out, nil +} + +func (l *ListState[T]) serializeValueFixed(value T) ([]byte, error) { + if l.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + encoded, err := l.codec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode list value failed: %w", err) + } + if len(encoded) != l.fixedSize { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec encoded unexpected length: got %d, want %d", len(encoded), l.fixedSize) + } + out := make([]byte, 0, l.fixedSize) + out = append(out, encoded...) + return out, nil +} + +func (l *ListState[T]) serializeValuesFixedBatch(values []T) ([]byte, error) { + if l.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + total := l.fixedSize * len(values) + out := make([]byte, 0, total) + for _, value := range values { + encoded, err := l.codec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode list value failed: %w", err) + } + if len(encoded) != l.fixedSize { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec encoded unexpected length: got %d, want %d", len(encoded), l.fixedSize) + } + out = append(out, encoded...) + } + return out, nil +} + +func (l *ListState[T]) deserializeValuesFixed(raw []byte) ([]T, error) { + if l.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + if len(raw)%l.fixedSize != 0 { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted list payload: fixed-size data length mismatch") + } + out := make([]T, 0, len(raw)/l.fixedSize) + for idx := 0; idx < len(raw); idx += l.fixedSize { + itemRaw := raw[idx : idx+l.fixedSize] + value, err := l.codec.Decode(itemRaw) + if err != nil { + return nil, fmt.Errorf("decode list value failed: %w", err) + } + out = append(out, value) + } + return out, nil +} diff --git a/go-sdk/state/structures/map.go b/go-sdk/state/structures/map.go new file mode 100644 index 00000000..b14d1bcd --- /dev/null +++ b/go-sdk/state/structures/map.go @@ -0,0 +1,253 @@ +package structures + +import ( + "fmt" + "iter" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type MapEntry[K any, V any] struct { + Key K + Value V +} + +type MapState[K any, V any] struct { + store common.Store + keyGroup []byte + key []byte + namespace []byte + keyCodec codec.Codec[K] + valueCodec codec.Codec[V] +} + +func NewMapState[K any, V any](store common.Store, name string, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*MapState[K, V], error) { + stateName, err := common.ValidateStateName(name) + if err != nil { + return nil, err + } + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "map state %q store must not be nil", stateName) + } + if keyCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "map state %q key codec must not be nil", stateName) + } + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "map state %q value codec must not be nil", stateName) + } + if !keyCodec.IsOrderedKeyCodec() { + return nil, api.NewError(api.ErrStoreInternal, "map state %q key codec must be ordered (IsOrderedKeyCodec)", stateName) + } + return &MapState[K, V]{store: store, keyGroup: []byte(common.StateMapGroup), key: []byte(stateName), namespace: []byte("entries"), keyCodec: keyCodec, valueCodec: valueCodec}, nil +} + +func NewMapStateAutoKeyCodec[K any, V any](store common.Store, name string, valueCodec codec.Codec[V]) (*MapState[K, V], error) { + autoKeyCodec, err := inferOrderedKeyCodec[K]() + if err != nil { + return nil, err + } + return NewMapState[K, V](store, name, autoKeyCodec, valueCodec) +} + +func (m *MapState[K, V]) Put(key K, value V) error { + encodedKey, err := m.keyCodec.Encode(key) + if err != nil { + return fmt.Errorf("encode map key failed: %w", err) + } + encodedValue, err := m.valueCodec.Encode(value) + if err != nil { + return fmt.Errorf("encode map value failed: %w", err) + } + return m.store.Put(m.ck(encodedKey), encodedValue) +} + +func (m *MapState[K, V]) Get(key K) (V, bool, error) { + var zero V + encodedKey, err := m.keyCodec.Encode(key) + if err != nil { + return zero, false, fmt.Errorf("encode map key failed: %w", err) + } + raw, found, err := m.store.Get(m.ck(encodedKey)) + if err != nil { + return zero, false, err + } + if !found { + return zero, false, nil + } + decoded, err := m.valueCodec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("decode map value failed: %w", err) + } + return decoded, true, nil +} + +func (m *MapState[K, V]) Delete(key K) error { + encodedKey, err := m.keyCodec.Encode(key) + if err != nil { + return fmt.Errorf("encode map key failed: %w", err) + } + return m.store.Delete(m.ck(encodedKey)) +} + +func (m *MapState[K, V]) Len() (uint64, error) { + iter, err := m.store.ScanComplex(m.keyGroup, m.key, m.namespace) + if err != nil { + return 0, err + } + defer iter.Close() + var count uint64 + for { + has, err := iter.HasNext() + if err != nil { + return 0, err + } + if !has { + return count, nil + } + _, _, ok, err := iter.Next() + if err != nil { + return 0, err + } + if !ok { + return count, nil + } + count++ + } +} + +func (m *MapState[K, V]) Entries() ([]MapEntry[K, V], error) { + iter, err := m.store.ScanComplex(m.keyGroup, m.key, m.namespace) + if err != nil { + return nil, err + } + defer iter.Close() + out := make([]MapEntry[K, V], 0, 16) + for { + has, err := iter.HasNext() + if err != nil { + return nil, err + } + if !has { + return out, nil + } + keyBytes, valueBytes, ok, err := iter.Next() + if err != nil { + return nil, err + } + if !ok { + return out, nil + } + decodedKey, err := m.keyCodec.Decode(keyBytes) + if err != nil { + return nil, fmt.Errorf("decode map key failed: %w", err) + } + decodedValue, err := m.valueCodec.Decode(valueBytes) + if err != nil { + return nil, fmt.Errorf("decode map value failed: %w", err) + } + out = append(out, MapEntry[K, V]{Key: decodedKey, Value: decodedValue}) + } +} + +func (m *MapState[K, V]) Range(startInclusive K, endExclusive K) ([]MapEntry[K, V], error) { + startBytes, err := m.keyCodec.Encode(startInclusive) + if err != nil { + return nil, fmt.Errorf("encode map start key failed: %w", err) + } + endBytes, err := m.keyCodec.Encode(endExclusive) + if err != nil { + return nil, fmt.Errorf("encode map end key failed: %w", err) + } + userKeys, err := m.store.ListComplex(m.keyGroup, m.key, m.namespace, startBytes, endBytes) + if err != nil { + return nil, err + } + out := make([]MapEntry[K, V], 0, len(userKeys)) + for _, userKey := range userKeys { + raw, found, err := m.store.Get(m.ck(userKey)) + if err != nil { + return nil, err + } + if !found { + return nil, api.NewError(api.ErrResultUnexpected, "map range key disappeared during scan") + } + decodedKey, err := m.keyCodec.Decode(userKey) + if err != nil { + return nil, fmt.Errorf("decode map key failed: %w", err) + } + decodedValue, err := m.valueCodec.Decode(raw) + if err != nil { + return nil, fmt.Errorf("decode map value failed: %w", err) + } + out = append(out, MapEntry[K, V]{Key: decodedKey, Value: decodedValue}) + } + return out, nil +} + +func (m *MapState[K, V]) Clear() error { + return m.store.DeletePrefix(api.ComplexKey{KeyGroup: m.keyGroup, Key: m.key, Namespace: m.namespace, UserKey: nil}) +} + +func (m *MapState[K, V]) All() iter.Seq2[K, V] { + return func(yield func(K, V) bool) { + it, err := m.store.ScanComplex(m.keyGroup, m.key, m.namespace) + if err != nil { + return + } + defer it.Close() + + for { + has, err := it.HasNext() + if err != nil || !has { + return + } + keyRaw, valRaw, ok, err := it.Next() + if err != nil || !ok { + return + } + + k, _ := m.keyCodec.Decode(keyRaw) + v, _ := m.valueCodec.Decode(valRaw) + + if !yield(k, v) { + return + } + } + } +} + +func (m *MapState[K, V]) ck(userKey []byte) api.ComplexKey { + return api.ComplexKey{KeyGroup: m.keyGroup, Key: m.key, Namespace: m.namespace, UserKey: userKey} +} + +func inferOrderedKeyCodec[K any]() (codec.Codec[K], error) { + var zero K + switch any(zero).(type) { + case string: + return any(codec.StringCodec{}).(codec.Codec[K]), nil + case []byte: + return any(codec.BytesCodec{}).(codec.Codec[K]), nil + case bool: + return any(codec.BoolCodec{}).(codec.Codec[K]), nil + case int: + return any(codec.OrderedIntCodec{}).(codec.Codec[K]), nil + case uint: + return any(codec.OrderedUintCodec{}).(codec.Codec[K]), nil + case int32: + return any(codec.OrderedInt32Codec{}).(codec.Codec[K]), nil + case uint32: + return any(codec.OrderedUint32Codec{}).(codec.Codec[K]), nil + case int64: + return any(codec.OrderedInt64Codec{}).(codec.Codec[K]), nil + case uint64: + return any(codec.OrderedUint64Codec{}).(codec.Codec[K]), nil + case float32: + return any(codec.OrderedFloat32Codec{}).(codec.Codec[K]), nil + case float64: + return any(codec.OrderedFloat64Codec{}).(codec.Codec[K]), nil + default: + return nil, api.NewError(api.ErrStoreInternal, "unsupported map key type for auto codec") + } +} diff --git a/go-sdk/state/structures/priority_queue.go b/go-sdk/state/structures/priority_queue.go new file mode 100644 index 00000000..798833bb --- /dev/null +++ b/go-sdk/state/structures/priority_queue.go @@ -0,0 +1,126 @@ +package structures + +import ( + "fmt" + "iter" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type PriorityQueueState[T any] struct { + store common.Store + keyGroup []byte + key []byte + namespace []byte + valueCodec codec.Codec[T] +} + +func NewPriorityQueueState[T any](store common.Store, name string, itemCodec codec.Codec[T]) (*PriorityQueueState[T], error) { + stateName, err := common.ValidateStateName(name) + if err != nil { + return nil, err + } + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "priority queue state %q store must not be nil", stateName) + } + if itemCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "priority queue state %q codec must not be nil", stateName) + } + if !itemCodec.IsOrderedKeyCodec() { + return nil, api.NewError(api.ErrStoreInternal, "priority queue value codec must be ordered") + } + return &PriorityQueueState[T]{ + store: store, + keyGroup: []byte(common.StatePQGroup), + key: []byte(stateName), + namespace: []byte("items"), + valueCodec: itemCodec, + }, nil +} + +func (q *PriorityQueueState[T]) ck(userKey []byte) api.ComplexKey { + return api.ComplexKey{KeyGroup: q.keyGroup, Key: q.key, Namespace: q.namespace, UserKey: userKey} +} + +func (q *PriorityQueueState[T]) Add(value T) error { + userKey, err := q.valueCodec.Encode(value) + if err != nil { + return fmt.Errorf("encode pq element failed: %w", err) + } + return q.store.Put(q.ck(userKey), []byte{}) +} + +func (q *PriorityQueueState[T]) Peek() (T, bool, error) { + var zero T + it, err := q.store.ScanComplex(q.keyGroup, q.key, q.namespace) + if err != nil { + return zero, false, err + } + defer it.Close() + + has, err := it.HasNext() + if err != nil || !has { + return zero, false, err + } + + userKey, _, ok, err := it.Next() + if err != nil || !ok { + return zero, false, err + } + + val, err := q.valueCodec.Decode(userKey) + if err != nil { + return zero, false, err + } + return val, true, nil +} + +func (q *PriorityQueueState[T]) Poll() (T, bool, error) { + val, found, err := q.Peek() + if err != nil || !found { + return val, found, err + } + + userKey, _ := q.valueCodec.Encode(val) + if err = q.store.Delete(q.ck(userKey)); err != nil { + return val, true, err + } + return val, true, nil +} + +func (q *PriorityQueueState[T]) Clear() error { + return q.store.DeletePrefix(api.ComplexKey{ + KeyGroup: q.keyGroup, + Key: q.key, + Namespace: q.namespace, + UserKey: nil, + }) +} + +func (q *PriorityQueueState[T]) All() iter.Seq[T] { + return func(yield func(T) bool) { + it, err := q.store.ScanComplex(q.keyGroup, q.key, q.namespace) + if err != nil { + return + } + defer it.Close() + + for { + has, err := it.HasNext() + if err != nil || !has { + return + } + userKey, _, ok, err := it.Next() + if err != nil || !ok { + return + } + + v, _ := q.valueCodec.Decode(userKey) + if !yield(v) { + return + } + } + } +} diff --git a/go-sdk/state/structures/reducing.go b/go-sdk/state/structures/reducing.go new file mode 100644 index 00000000..99f4b6a9 --- /dev/null +++ b/go-sdk/state/structures/reducing.go @@ -0,0 +1,100 @@ +package structures + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type ReduceFunc[V any] func(value1 V, value2 V) (V, error) + +type ReducingState[V any] struct { + store common.Store + complexKey api.ComplexKey + valueCodec codec.Codec[V] + reduceFunc ReduceFunc[V] +} + +func NewReducingState[V any]( + store common.Store, + name string, + valueCodec codec.Codec[V], + reduceFunc ReduceFunc[V], +) (*ReducingState[V], error) { + stateName, err := common.ValidateStateName(name) + if err != nil { + return nil, err + } + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "reducing state %q store must not be nil", stateName) + } + if valueCodec == nil || reduceFunc == nil { + return nil, api.NewError(api.ErrStoreInternal, "reducing state %q value codec and reduce function are required", stateName) + } + ck := api.ComplexKey{ + KeyGroup: []byte(common.StateReducingPrefix), + Key: []byte(stateName), + Namespace: []byte("data"), + UserKey: []byte{}, + } + return &ReducingState[V]{ + store: store, + complexKey: ck, + valueCodec: valueCodec, + reduceFunc: reduceFunc, + }, nil +} + +func (s *ReducingState[V]) buildCK() api.ComplexKey { + return s.complexKey +} + +func (s *ReducingState[V]) Add(value V) error { + ck := s.buildCK() + raw, found, err := s.store.Get(ck) + if err != nil { + return fmt.Errorf("failed to get old value for reducing state: %w", err) + } + + var result V + if !found { + result = value + } else { + oldValue, err := s.valueCodec.Decode(raw) + if err != nil { + return fmt.Errorf("failed to decode old value: %w", err) + } + + result, err = s.reduceFunc(oldValue, value) + if err != nil { + return fmt.Errorf("error in user reduce function: %w", err) + } + } + + encoded, err := s.valueCodec.Encode(result) + if err != nil { + return fmt.Errorf("failed to encode reduced value: %w", err) + } + + return s.store.Put(ck, encoded) +} + +func (s *ReducingState[V]) Get() (V, bool, error) { + var zero V + ck := s.buildCK() + raw, found, err := s.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + val, err := s.valueCodec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("failed to decode value: %w", err) + } + return val, true, nil +} + +func (s *ReducingState[V]) Clear() error { + return s.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/structures/value.go b/go-sdk/state/structures/value.go new file mode 100644 index 00000000..0950def1 --- /dev/null +++ b/go-sdk/state/structures/value.go @@ -0,0 +1,65 @@ +package structures + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type ValueState[T any] struct { + store common.Store + complexKey api.ComplexKey + codec codec.Codec[T] +} + +func NewValueState[T any](store common.Store, name string, valueCodec codec.Codec[T]) (*ValueState[T], error) { + stateName, err := common.ValidateStateName(name) + if err != nil { + return nil, err + } + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "value state %q store must not be nil", stateName) + } + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "value state %q codec must not be nil", stateName) + } + ck := api.ComplexKey{ + KeyGroup: []byte(common.StateValuePrefix), + Key: []byte(stateName), + Namespace: []byte("data"), + UserKey: []byte{}, + } + return &ValueState[T]{store: store, complexKey: ck, codec: valueCodec}, nil +} + +func (v *ValueState[T]) buildCK() api.ComplexKey { + return v.complexKey +} + +func (v *ValueState[T]) Update(value T) error { + encoded, err := v.codec.Encode(value) + if err != nil { + return fmt.Errorf("encode value state failed: %w", err) + } + return v.store.Put(v.buildCK(), encoded) +} + +func (v *ValueState[T]) Value() (T, bool, error) { + var zero T + ck := v.buildCK() + raw, found, err := v.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + decoded, err := v.codec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("decode value state failed: %w", err) + } + return decoded, true, nil +} + +func (v *ValueState[T]) Clear() error { + return v.store.Delete(v.buildCK()) +} diff --git a/python/functionstream-api/src/fs_api/__init__.py b/python/functionstream-api/src/fs_api/__init__.py index 1ecdd42d..7c185282 100644 --- a/python/functionstream-api/src/fs_api/__init__.py +++ b/python/functionstream-api/src/fs_api/__init__.py @@ -23,6 +23,42 @@ ComplexKey, KvIterator, KvStore, + Codec, + JsonCodec, + PickleCodec, + BytesCodec, + StringCodec, + BoolCodec, + Int64Codec, + Uint64Codec, + Int32Codec, + Uint32Codec, + Float64Codec, + Float32Codec, + OrderedInt64Codec, + OrderedUint64Codec, + OrderedInt32Codec, + OrderedUint32Codec, + OrderedFloat64Codec, + OrderedFloat32Codec, + ValueState, + MapEntry, + MapState, + infer_ordered_key_codec, + create_map_state_auto_key_codec, + ListState, + PriorityQueueState, + AggregateFunc, + AggregatingState, + ReduceFunc, + ReducingState, + KeyedStateFactory, + KeyedValueState, + KeyedMapState, + KeyedListState, + KeyedPriorityQueueState, + KeyedAggregatingState, + KeyedReducingState, ) __all__ = [ @@ -35,5 +71,41 @@ "ComplexKey", "KvIterator", "KvStore", + "Codec", + "JsonCodec", + "PickleCodec", + "BytesCodec", + "StringCodec", + "BoolCodec", + "Int64Codec", + "Uint64Codec", + "Int32Codec", + "Uint32Codec", + "Float64Codec", + "Float32Codec", + "OrderedInt64Codec", + "OrderedUint64Codec", + "OrderedInt32Codec", + "OrderedUint32Codec", + "OrderedFloat64Codec", + "OrderedFloat32Codec", + "ValueState", + "MapEntry", + "MapState", + "infer_ordered_key_codec", + "create_map_state_auto_key_codec", + "ListState", + "PriorityQueueState", + "AggregateFunc", + "AggregatingState", + "ReduceFunc", + "ReducingState", + "KeyedStateFactory", + "KeyedValueState", + "KeyedMapState", + "KeyedListState", + "KeyedPriorityQueueState", + "KeyedAggregatingState", + "KeyedReducingState", ] diff --git a/python/functionstream-api/src/fs_api/context.py b/python/functionstream-api/src/fs_api/context.py index 52cbd8a5..c9578cbe 100644 --- a/python/functionstream-api/src/fs_api/context.py +++ b/python/functionstream-api/src/fs_api/context.py @@ -17,7 +17,19 @@ """ import abc from typing import Dict -from .store import KvStore +from .store import ( + Codec, + KvStore, + ValueState, + MapState, + ListState, + PriorityQueueState, + KeyedStateFactory, + KeyedValueState, + KeyedMapState, + KeyedListState, + KeyedPriorityQueueState, +) class Context(abc.ABC): @@ -44,4 +56,40 @@ def getConfig(self) -> Dict[str, str]: Dict[str, str]: Configuration dictionary """ + @abc.abstractmethod + def getOrCreateValueState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> ValueState: + pass + + @abc.abstractmethod + def getOrCreateMapState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> MapState: + pass + + @abc.abstractmethod + def getOrCreateListState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> ListState: + pass + + @abc.abstractmethod + def getOrCreatePriorityQueueState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> PriorityQueueState: + pass + + @abc.abstractmethod + def getOrCreateKeyedStateFactory(self, state_name: str, store_name: str = "__fssdk_structured_state__") -> KeyedStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedValueState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedValueState: + pass + + @abc.abstractmethod + def getOrCreateKeyedMapState(self, state_name: str, key_codec: Codec, map_key_codec: Codec, map_value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedMapState: + pass + + @abc.abstractmethod + def getOrCreateKeyedListState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedListState: + pass + + @abc.abstractmethod + def getOrCreateKeyedPriorityQueueState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedPriorityQueueState: + pass + __all__ = ['Context'] diff --git a/python/functionstream-api/src/fs_api/store/__init__.py b/python/functionstream-api/src/fs_api/store/__init__.py index 1cfdc8f8..454c99a2 100644 --- a/python/functionstream-api/src/fs_api/store/__init__.py +++ b/python/functionstream-api/src/fs_api/store/__init__.py @@ -5,24 +5,103 @@ # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on "AS IS" BASIS, +# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from .error import KvError, KvNotFoundError, KvIOError, KvOtherError from .complexkey import ComplexKey from .iterator import KvIterator from .store import KvStore +from .codec import ( + Codec, + JsonCodec, + PickleCodec, + BytesCodec, + StringCodec, + BoolCodec, + Int64Codec, + Uint64Codec, + Int32Codec, + Uint32Codec, + Float64Codec, + Float32Codec, + OrderedInt64Codec, + OrderedUint64Codec, + OrderedInt32Codec, + OrderedUint32Codec, + OrderedFloat64Codec, + OrderedFloat32Codec, +) + +from .structures import ( + ValueState, + ListState, + MapEntry, + MapState, + infer_ordered_key_codec, + create_map_state_auto_key_codec, + PriorityQueueState, + AggregateFunc, + AggregatingState, + ReduceFunc, + ReducingState, +) + +from .keyed import ( + KeyedStateFactory, + KeyedValueState, + KeyedMapState, + KeyedListState, + KeyedPriorityQueueState, + KeyedAggregatingState, + KeyedReducingState, +) + __all__ = [ - 'KvError', - 'KvNotFoundError', - 'KvIOError', - 'KvOtherError', - 'ComplexKey', - 'KvIterator', - 'KvStore', + "KvError", + "KvNotFoundError", + "KvIOError", + "KvOtherError", + "ComplexKey", + "KvIterator", + "KvStore", + "Codec", + "JsonCodec", + "PickleCodec", + "BytesCodec", + "StringCodec", + "BoolCodec", + "Int64Codec", + "Uint64Codec", + "Int32Codec", + "Uint32Codec", + "Float64Codec", + "Float32Codec", + "OrderedInt64Codec", + "OrderedUint64Codec", + "OrderedInt32Codec", + "OrderedUint32Codec", + "OrderedFloat64Codec", + "OrderedFloat32Codec", + "ValueState", + "MapEntry", + "MapState", + "infer_ordered_key_codec", + "create_map_state_auto_key_codec", + "ListState", + "PriorityQueueState", + "AggregateFunc", + "AggregatingState", + "ReduceFunc", + "ReducingState", + "KeyedStateFactory", + "KeyedValueState", + "KeyedMapState", + "KeyedListState", + "KeyedPriorityQueueState", + "KeyedAggregatingState", + "KeyedReducingState", ] - diff --git a/python/functionstream-api/src/fs_api/store/codec/__init__.py b/python/functionstream-api/src/fs_api/store/codec/__init__.py new file mode 100644 index 00000000..376cf8c7 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/__init__.py @@ -0,0 +1,273 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import struct +from typing import Generic, TypeVar + +import cloudpickle + +T = TypeVar("T") + + +class Codec(Generic[T]): + supports_ordered_keys: bool = False + + def encode(self, value: T) -> bytes: + raise NotImplementedError + + def decode(self, data: bytes) -> T: + raise NotImplementedError + + +class JsonCodec(Codec[T]): + def encode(self, value: T) -> bytes: + return json.dumps(value).encode("utf-8") + + def decode(self, data: bytes) -> T: + return json.loads(data.decode("utf-8")) + + +class PickleCodec(Codec[T]): + def encode(self, value: T) -> bytes: + return cloudpickle.dumps(value) + + def decode(self, data: bytes) -> T: + return cloudpickle.loads(data) + + +class BytesCodec(Codec[bytes]): + supports_ordered_keys = True + + def encode(self, value: bytes) -> bytes: + return bytes(value) + + def decode(self, data: bytes) -> bytes: + return bytes(data) + + +class StringCodec(Codec[str]): + supports_ordered_keys = True + + def encode(self, value: str) -> bytes: + return value.encode("utf-8") + + def decode(self, data: bytes) -> str: + return data.decode("utf-8") + + +class BoolCodec(Codec[bool]): + supports_ordered_keys = True + + def encode(self, value: bool) -> bytes: + return b"\x01" if value else b"\x00" + + def decode(self, data: bytes) -> bool: + if len(data) != 1: + raise ValueError(f"invalid bool payload length: {len(data)}") + if data == b"\x00": + return False + if data == b"\x01": + return True + raise ValueError(f"invalid bool payload byte: {data[0]}") + + +class Int64Codec(Codec[int]): + def encode(self, value: int) -> bytes: + return struct.pack(">q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid int64 payload length: {len(data)}") + return struct.unpack(">q", data)[0] + + +class Uint64Codec(Codec[int]): + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("uint64 value must be >= 0") + return struct.pack(">Q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid uint64 payload length: {len(data)}") + return struct.unpack(">Q", data)[0] + + +class Int32Codec(Codec[int]): + def encode(self, value: int) -> bytes: + return struct.pack(">i", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid int32 payload length: {len(data)}") + return struct.unpack(">i", data)[0] + + +class Uint32Codec(Codec[int]): + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("uint32 value must be >= 0") + return struct.pack(">I", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid uint32 payload length: {len(data)}") + return struct.unpack(">I", data)[0] + + +class Float64Codec(Codec[float]): + def encode(self, value: float) -> bytes: + return struct.pack(">d", value) + + def decode(self, data: bytes) -> float: + if len(data) != 8: + raise ValueError(f"invalid float64 payload length: {len(data)}") + return struct.unpack(">d", data)[0] + + +class Float32Codec(Codec[float]): + def encode(self, value: float) -> bytes: + return struct.pack(">f", value) + + def decode(self, data: bytes) -> float: + if len(data) != 4: + raise ValueError(f"invalid float32 payload length: {len(data)}") + return struct.unpack(">f", data)[0] + + +class OrderedInt64Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + mapped = (value & 0xFFFFFFFFFFFFFFFF) ^ (1 << 63) + return struct.pack(">Q", mapped) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid ordered int64 payload length: {len(data)}") + mapped = struct.unpack(">Q", data)[0] + raw = mapped ^ (1 << 63) + if raw >= (1 << 63): + return raw - (1 << 64) + return raw + + +class OrderedUint64Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("ordered uint64 value must be >= 0") + return struct.pack(">Q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid ordered uint64 payload length: {len(data)}") + return struct.unpack(">Q", data)[0] + + +class OrderedInt32Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + mapped = (value & 0xFFFFFFFF) ^ (1 << 31) + return struct.pack(">I", mapped) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid ordered int32 payload length: {len(data)}") + mapped = struct.unpack(">I", data)[0] + raw = mapped ^ (1 << 31) + if raw >= (1 << 31): + return raw - (1 << 32) + return raw + + +class OrderedUint32Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("ordered uint32 value must be >= 0") + return struct.pack(">I", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid ordered uint32 payload length: {len(data)}") + return struct.unpack(">I", data)[0] + + +class OrderedFloat64Codec(Codec[float]): + supports_ordered_keys = True + + def encode(self, value: float) -> bytes: + bits = struct.unpack(">Q", struct.pack(">d", value))[0] + if bits & (1 << 63): + mapped = (~bits) & 0xFFFFFFFFFFFFFFFF + else: + mapped = bits ^ (1 << 63) + return struct.pack(">Q", mapped) + + def decode(self, data: bytes) -> float: + if len(data) != 8: + raise ValueError(f"invalid ordered float64 payload length: {len(data)}") + mapped = struct.unpack(">Q", data)[0] + if mapped & (1 << 63): + bits = mapped ^ (1 << 63) + else: + bits = (~mapped) & 0xFFFFFFFFFFFFFFFF + return struct.unpack(">d", struct.pack(">Q", bits))[0] + + +class OrderedFloat32Codec(Codec[float]): + supports_ordered_keys = True + + def encode(self, value: float) -> bytes: + bits = struct.unpack(">I", struct.pack(">f", value))[0] + if bits & (1 << 31): + mapped = (~bits) & 0xFFFFFFFF + else: + mapped = bits ^ (1 << 31) + return struct.pack(">I", mapped) + + def decode(self, data: bytes) -> float: + if len(data) != 4: + raise ValueError(f"invalid ordered float32 payload length: {len(data)}") + mapped = struct.unpack(">I", data)[0] + if mapped & (1 << 31): + bits = mapped ^ (1 << 31) + else: + bits = (~mapped) & 0xFFFFFFFF + return struct.unpack(">f", struct.pack(">I", bits))[0] + + +__all__ = [ + "Codec", + "JsonCodec", + "PickleCodec", + "BytesCodec", + "StringCodec", + "BoolCodec", + "Int64Codec", + "Uint64Codec", + "Int32Codec", + "Uint32Codec", + "Float64Codec", + "Float32Codec", + "OrderedInt64Codec", + "OrderedUint64Codec", + "OrderedInt32Codec", + "OrderedUint32Codec", + "OrderedFloat64Codec", + "OrderedFloat32Codec", +] diff --git a/python/functionstream-api/src/fs_api/store/common/__init__.py b/python/functionstream-api/src/fs_api/store/common/__init__.py new file mode 100644 index 00000000..64b03dfb --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/common/__init__.py @@ -0,0 +1,68 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct +from typing import Tuple + +from ..error import KvError + +VALUE_PREFIX = b"__fssdk__/value/" +LIST_PREFIX = b"__fssdk__/list/" +PQ_PREFIX = b"__fssdk__/priority_queue/" +AGGREGATING_PREFIX = b"__fssdk__/aggregating/" +REDUCING_PREFIX = b"__fssdk__/reducing/" +MAP_GROUP = b"__fssdk__/map" +LIST_GROUP = b"__fssdk__/list" +PQ_GROUP = b"__fssdk__/priority_queue" + + +def validate_state_name(name: str) -> None: + if not isinstance(name, str) or not name.strip(): + raise KvError("state name must be a non-empty string") + + +def encode_int64_lex(value: int) -> bytes: + mapped = (value & 0xFFFFFFFFFFFFFFFF) ^ (1 << 63) + return struct.pack(">Q", mapped) + + +def encode_priority_key(priority: int, seq: int) -> bytes: + return encode_int64_lex(priority) + struct.pack(">Q", seq) + + +def decode_priority_key(data: bytes) -> Tuple[int, int]: + if len(data) != 16: + raise KvError("invalid priority queue key length") + mapped_priority = struct.unpack(">Q", data[:8])[0] + unsigned_priority = mapped_priority ^ (1 << 63) + if unsigned_priority >= (1 << 63): + priority = unsigned_priority - (1 << 64) + else: + priority = unsigned_priority + seq = struct.unpack(">Q", data[8:])[0] + return priority, seq + + +__all__ = [ + "VALUE_PREFIX", + "LIST_PREFIX", + "PQ_PREFIX", + "AGGREGATING_PREFIX", + "REDUCING_PREFIX", + "MAP_GROUP", + "LIST_GROUP", + "PQ_GROUP", + "validate_state_name", + "encode_int64_lex", + "encode_priority_key", + "decode_priority_key", +] diff --git a/python/functionstream-api/src/fs_api/store/keyed/__init__.py b/python/functionstream-api/src/fs_api/store/keyed/__init__.py new file mode 100644 index 00000000..802ca888 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/__init__.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .keyed_state_factory import KeyedStateFactory +from .keyed_value_state import KeyedValueState +from .keyed_list_state import KeyedListState +from .keyed_map_state import KeyedMapEntry, KeyedMapState +from .keyed_priority_queue_state import KeyedPriorityQueueState +from .keyed_aggregating_state import AggregateFunc, KeyedAggregatingState +from .keyed_reducing_state import KeyedReducingState, ReduceFunc + +__all__ = [ + "KeyedStateFactory", + "KeyedValueState", + "KeyedListState", + "KeyedMapEntry", + "KeyedMapState", + "KeyedPriorityQueueState", + "KeyedAggregatingState", + "KeyedReducingState", + "AggregateFunc", + "ReduceFunc", +] diff --git a/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py b/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py new file mode 100644 index 00000000..81dc850f --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from ..error import KvError + +KEYED_VALUE_GROUP = b"__fssdk__/keyed/value" +KEYED_LIST_GROUP = b"__fssdk__/keyed/list" +KEYED_MAP_GROUP = b"__fssdk__/keyed/map" +KEYED_PQ_GROUP = b"__fssdk__/keyed/pq" +KEYED_AGGREGATING_GROUP = b"__fssdk__/keyed/aggregating" +KEYED_REDUCING_GROUP = b"__fssdk__/keyed/reducing" + + +def ensure_ordered_key_codec(codec: Any, label: str) -> None: + if not getattr(codec, "supports_ordered_keys", False): + raise KvError(f"{label} key codec must support ordered key encoding") + + +__all__ = [ + "ensure_ordered_key_codec", + "KEYED_VALUE_GROUP", + "KEYED_LIST_GROUP", + "KEYED_MAP_GROUP", + "KEYED_PQ_GROUP", + "KEYED_AGGREGATING_GROUP", + "KEYED_REDUCING_GROUP", +] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py new file mode 100644 index 00000000..06a8aed0 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py @@ -0,0 +1,80 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Optional, Protocol, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..store import KvStore + +from ._keyed_common import KEYED_AGGREGATING_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +T_agg = TypeVar("T_agg") +ACC = TypeVar("ACC") +R = TypeVar("R") + + +class AggregateFunc(Protocol[T_agg, ACC, R]): + def create_accumulator(self) -> ACC: ... + def add(self, value: T_agg, accumulator: ACC) -> ACC: ... + def get_result(self, accumulator: ACC) -> R: ... + def merge(self, a: ACC, b: ACC) -> ACC: ... + + +class KeyedAggregatingState(Generic[K, T_agg, ACC, R]): + def __init__( + self, + store: KvStore, + name: str, + key_codec: Codec[K], + acc_codec: Codec[ACC], + agg_func: AggregateFunc[T_agg, ACC, R], + ): + self._store = store + self._name = name.strip() + self._key_codec = key_codec + self._acc_codec = acc_codec + self._agg_func = agg_func + ensure_ordered_key_codec(key_codec, "keyed aggregating") + + def _build_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=KEYED_AGGREGATING_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=b"", + ) + + def add(self, key: K, value: T_agg) -> None: + ck = self._build_ck(key) + raw = self._store.get(ck) + if raw is None: + acc = self._agg_func.create_accumulator() + else: + acc = self._acc_codec.decode(raw) + new_acc = self._agg_func.add(value, acc) + self._store.put(ck, self._acc_codec.encode(new_acc)) + + def get(self, key: K) -> Tuple[Optional[R], bool]: + ck = self._build_ck(key) + raw = self._store.get(ck) + if raw is None: + return (None, False) + acc = self._acc_codec.decode(raw) + return (self._agg_func.get_result(acc), True) + + def clear(self, key: K) -> None: + self._store.delete(self._build_ck(key)) + + +__all__ = ["AggregateFunc", "KeyedAggregatingState"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py new file mode 100644 index 00000000..8635217b --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py @@ -0,0 +1,94 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct +from typing import Generic, List, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +from ._keyed_common import KEYED_LIST_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +V = TypeVar("V") + + +class KeyedListState(Generic[K, V]): + def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: Codec[V]): + self._store = store + self._name = name.strip() + self._key_codec = key_codec + self._value_codec = value_codec + ensure_ordered_key_codec(key_codec, "keyed list") + + def _build_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=KEYED_LIST_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=b"", + ) + + def add(self, key: K, value: V) -> None: + payload = self._serialize_one(value) + self._store.merge(self._build_ck(key), payload) + + def add_all(self, key: K, values: List[V]) -> None: + if not values: + return + payload = self._serialize_batch(values) + self._store.merge(self._build_ck(key), payload) + + def get(self, key: K) -> List[V]: + raw = self._store.get(self._build_ck(key)) + if raw is None: + return [] + return self._deserialize(raw) + + def update(self, key: K, values: List[V]) -> None: + if len(values) == 0: + self.clear(key) + return + self._store.put(self._build_ck(key), self._serialize_batch(values)) + + def clear(self, key: K) -> None: + self._store.delete(self._build_ck(key)) + + def _serialize_one(self, value: V) -> bytes: + encoded = self._value_codec.encode(value) + return struct.pack(">I", len(encoded)) + encoded + + def _serialize_batch(self, values: List[V]) -> bytes: + parts = [] + for v in values: + encoded = self._value_codec.encode(v) + parts.append(struct.pack(">I", len(encoded)) + encoded) + return b"".join(parts) + + def _deserialize(self, raw: bytes) -> List[V]: + out = [] + idx = 0 + while idx < len(raw): + if len(raw) - idx < 4: + raise KvError("corrupted keyed list payload: truncated length") + (item_len,) = struct.unpack(">I", raw[idx : idx + 4]) + idx += 4 + if item_len < 0 or len(raw) - idx < item_len: + raise KvError("corrupted keyed list payload: invalid element length") + out.append(self._value_codec.decode(raw[idx : idx + item_len])) + idx += item_len + return out + + +__all__ = ["KeyedListState"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py new file mode 100644 index 00000000..79d01146 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py @@ -0,0 +1,149 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Generic, List, Optional, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +from ._keyed_common import KEYED_MAP_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +MK = TypeVar("MK") +MV = TypeVar("MV") + + +@dataclass +class KeyedMapEntry(Generic[MK, MV]): + key: MK + value: MV + + +class KeyedMapState(Generic[K, MK, MV]): + def __init__( + self, + store: KvStore, + name: str, + key_codec: Codec[K], + map_key_codec: Codec[MK], + map_value_codec: Codec[MV], + ): + self._store = store + self._name = name.strip() + self._key_codec = key_codec + self._map_key_codec = map_key_codec + self._map_value_codec = map_value_codec + ensure_ordered_key_codec(key_codec, "keyed map") + ensure_ordered_key_codec(map_key_codec, "keyed map inner") + + def put(self, key: K, map_key: MK, value: MV) -> None: + ck = ComplexKey( + key_group=KEYED_MAP_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=self._map_key_codec.encode(map_key), + ) + self._store.put(ck, self._map_value_codec.encode(value)) + + def get(self, key: K, map_key: MK) -> Optional[MV]: + ck = ComplexKey( + key_group=KEYED_MAP_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=self._map_key_codec.encode(map_key), + ) + raw = self._store.get(ck) + if raw is None: + return None + return self._map_value_codec.decode(raw) + + def delete(self, key: K, map_key: MK) -> None: + ck = ComplexKey( + key_group=KEYED_MAP_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=self._map_key_codec.encode(map_key), + ) + self._store.delete(ck) + + def clear(self, key: K) -> None: + prefix_ck = ComplexKey( + key_group=KEYED_MAP_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=b"", + ) + self._store.delete_prefix(prefix_ck) + + def all(self, key: K): + it = self._store.scan_complex( + KEYED_MAP_GROUP, + self._key_codec.encode(key), + self._name.encode("utf-8"), + ) + while it.has_next(): + item = it.next() + if item is None: + break + key_bytes, value_bytes = item + yield self._map_key_codec.decode(key_bytes), self._map_value_codec.decode(value_bytes) + + def len(self, key: K) -> int: + it = self._store.scan_complex( + KEYED_MAP_GROUP, + self._key_codec.encode(key), + self._name.encode("utf-8"), + ) + n = 0 + while it.has_next(): + it.next() + n += 1 + return n + + def entries(self, key: K) -> List[KeyedMapEntry[MK, MV]]: + return [ + KeyedMapEntry(key=k, value=v) + for k, v in self.all(key) + ] + + def range( + self, key: K, start_inclusive: MK, end_exclusive: MK + ) -> List[KeyedMapEntry[MK, MV]]: + key_bytes = self._key_codec.encode(key) + start_bytes = self._map_key_codec.encode(start_inclusive) + end_bytes = self._map_key_codec.encode(end_exclusive) + user_keys = self._store.list_complex( + KEYED_MAP_GROUP, key_bytes, self._name.encode("utf-8"), + start_bytes, end_bytes, + ) + out = [] + for uk in user_keys: + ck = ComplexKey( + key_group=KEYED_MAP_GROUP, + key=key_bytes, + namespace=self._name.encode("utf-8"), + user_key=uk, + ) + raw = self._store.get(ck) + if raw is None: + raise KvError("map range key disappeared during scan") + out.append(KeyedMapEntry( + key=self._map_key_codec.decode(uk), + value=self._map_value_codec.decode(raw), + )) + return out + + +__all__ = ["KeyedMapEntry", "KeyedMapState"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py new file mode 100644 index 00000000..58183f53 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Iterator, Optional, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..store import KvStore + +from ._keyed_common import KEYED_PQ_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +V = TypeVar("V") + + +class KeyedPriorityQueueState(Generic[K, V]): + def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: Codec[V]): + self._store = store + self._name = name.strip() + self._key_codec = key_codec + self._value_codec = value_codec + ensure_ordered_key_codec(key_codec, "keyed priority queue") + ensure_ordered_key_codec(value_codec, "keyed priority queue value") + + def _ck(self, key: K, user_key: bytes) -> ComplexKey: + return ComplexKey( + key_group=KEYED_PQ_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=user_key, + ) + + def _prefix_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=KEYED_PQ_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=b"", + ) + + def add(self, key: K, value: V) -> None: + user_key = self._value_codec.encode(value) + self._store.put(self._ck(key, user_key), b"") + + def peek(self, key: K) -> Tuple[Optional[V], bool]: + it = self._store.scan_complex( + KEYED_PQ_GROUP, + self._key_codec.encode(key), + self._name.encode("utf-8"), + ) + if not it.has_next(): + return (None, False) + item = it.next() + if item is None: + return (None, False) + user_key, _ = item + return (self._value_codec.decode(user_key), True) + + def poll(self, key: K) -> Tuple[Optional[V], bool]: + val, found = self.peek(key) + if not found or val is None: + return (val, found) + self._store.delete(self._ck(key, self._value_codec.encode(val))) + return (val, True) + + def clear(self, key: K) -> None: + self._store.delete_prefix(self._prefix_ck(key)) + + def all(self, key: K) -> Iterator[V]: + it = self._store.scan_complex( + KEYED_PQ_GROUP, + self._key_codec.encode(key), + self._name.encode("utf-8"), + ) + while it.has_next(): + item = it.next() + if item is None: + break + user_key, _ = item + yield self._value_codec.decode(user_key) + + +__all__ = ["KeyedPriorityQueueState"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py new file mode 100644 index 00000000..0656eb4c --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py @@ -0,0 +1,70 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Generic, Optional, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..store import KvStore + +from ._keyed_common import KEYED_REDUCING_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +V = TypeVar("V") + +ReduceFunc = Callable[[V, V], V] + + +class KeyedReducingState(Generic[K, V]): + def __init__( + self, + store: KvStore, + name: str, + key_codec: Codec[K], + value_codec: Codec[V], + reduce_func: ReduceFunc[V], + ): + self._store = store + self._name = name.strip() + self._key_codec = key_codec + self._value_codec = value_codec + self._reduce_func = reduce_func + ensure_ordered_key_codec(key_codec, "keyed reducing") + + def _build_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=KEYED_REDUCING_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=b"", + ) + + def add(self, key: K, value: V) -> None: + ck = self._build_ck(key) + raw = self._store.get(ck) + if raw is None: + result = value + else: + result = self._reduce_func(self._value_codec.decode(raw), value) + self._store.put(ck, self._value_codec.encode(result)) + + def get(self, key: K) -> Tuple[Optional[V], bool]: + raw = self._store.get(self._build_ck(key)) + if raw is None: + return (None, False) + return (self._value_codec.decode(raw), True) + + def clear(self, key: K) -> None: + self._store.delete(self._build_ck(key)) + + +__all__ = ["ReduceFunc", "KeyedReducingState"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py new file mode 100644 index 00000000..c2563bff --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py @@ -0,0 +1,94 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TypeVar + +from ..codec import Codec +from ..error import KvError +from ..store import KvStore + +from .keyed_aggregating_state import AggregateFunc, KeyedAggregatingState +from .keyed_list_state import KeyedListState +from .keyed_map_state import KeyedMapState +from .keyed_priority_queue_state import KeyedPriorityQueueState +from .keyed_reducing_state import KeyedReducingState, ReduceFunc +from .keyed_value_state import KeyedValueState + +K = TypeVar("K") +V = TypeVar("V") +MK = TypeVar("MK") +MV = TypeVar("MV") +T_agg = TypeVar("T_agg") +ACC = TypeVar("ACC") +R = TypeVar("R") + + +class KeyedStateFactory: + def __init__(self, store: KvStore, name: str): + if store is None: + raise KvError("keyed state factory store must not be None") + if not isinstance(name, str) or not name.strip(): + raise KvError("keyed state factory name must be non-empty") + self._store = store + self._name = name.strip() + self._kind = None + + def new_keyed_value(self, key_codec: Codec[K], value_codec: Codec[V]) -> KeyedValueState[K, V]: + self._claim_kind("value") + return KeyedValueState(self._store, self._name, key_codec, value_codec) + + def new_keyed_map( + self, key_codec: Codec[K], map_key_codec: Codec[MK], map_value_codec: Codec[MV] + ) -> KeyedMapState[K, MK, MV]: + self._claim_kind("map") + return KeyedMapState( + self._store, self._name, key_codec, map_key_codec, map_value_codec + ) + + def new_keyed_list(self, key_codec: Codec[K], value_codec: Codec[V]) -> KeyedListState[K, V]: + self._claim_kind("list") + return KeyedListState(self._store, self._name, key_codec, value_codec) + + def new_keyed_priority_queue( + self, key_codec: Codec[K], value_codec: Codec[V] + ) -> KeyedPriorityQueueState[K, V]: + self._claim_kind("priority_queue") + return KeyedPriorityQueueState(self._store, self._name, key_codec, value_codec) + + def new_keyed_aggregating( + self, + key_codec: Codec[K], + acc_codec: Codec[ACC], + agg_func: AggregateFunc[T_agg, ACC, R], + ) -> KeyedAggregatingState[K, T_agg, ACC, R]: + self._claim_kind("aggregating") + return KeyedAggregatingState( + self._store, self._name, key_codec, acc_codec, agg_func + ) + + def new_keyed_reducing( + self, key_codec: Codec[K], value_codec: Codec[V], reduce_func: ReduceFunc[V] + ) -> KeyedReducingState[K, V]: + self._claim_kind("reducing") + return KeyedReducingState(self._store, self._name, key_codec, value_codec, reduce_func) + + def _claim_kind(self, kind: str) -> None: + if self._kind is None: + self._kind = kind + return + if self._kind != kind: + raise KvError( + f"keyed state factory '{self._name}' already bound to '{self._kind}', cannot create '{kind}'" + ) + + +__all__ = ["KeyedStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py new file mode 100644 index 00000000..0455a739 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py @@ -0,0 +1,56 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Optional, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..store import KvStore + +from ._keyed_common import KEYED_VALUE_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +V = TypeVar("V") + + +class KeyedValueState(Generic[K, V]): + def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: Codec[V]): + self._store = store + self._name = name.strip() + self._key_codec = key_codec + self._value_codec = value_codec + ensure_ordered_key_codec(key_codec, "keyed value") + + def _build_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=KEYED_VALUE_GROUP, + key=self._key_codec.encode(key), + namespace=self._name.encode("utf-8"), + user_key=b"", + ) + + def set(self, key: K, value: V) -> None: + ck = self._build_ck(key) + self._store.put(ck, self._value_codec.encode(value)) + + def get(self, key: K) -> Optional[V]: + ck = self._build_ck(key) + raw = self._store.get(ck) + if raw is None: + return None + return self._value_codec.decode(raw) + + def delete(self, key: K) -> None: + self._store.delete(self._build_ck(key)) + + +__all__ = ["KeyedValueState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/__init__.py b/python/functionstream-api/src/fs_api/store/structures/__init__.py new file mode 100644 index 00000000..9dce8ee3 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/__init__.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .value_state import ValueState +from .list_state import ListState +from .map_state import ( + MapEntry, + MapState, + infer_ordered_key_codec, + create_map_state_auto_key_codec, +) +from .priority_queue_state import PriorityQueueState +from .aggregating_state import AggregateFunc, AggregatingState +from .reducing_state import ReduceFunc, ReducingState + +__all__ = [ + "ValueState", + "ListState", + "MapEntry", + "MapState", + "infer_ordered_key_codec", + "create_map_state_auto_key_codec", + "PriorityQueueState", + "AggregateFunc", + "AggregatingState", + "ReduceFunc", + "ReducingState", +] diff --git a/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py new file mode 100644 index 00000000..236952aa --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Optional, Protocol, Tuple, TypeVar + +from ..common import AGGREGATING_PREFIX, validate_state_name +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +T = TypeVar("T") +ACC = TypeVar("ACC") +R = TypeVar("R") + + +class AggregateFunc(Protocol[T, ACC, R]): + def create_accumulator(self) -> ACC: + ... + + def add(self, value: T, accumulator: ACC) -> ACC: + ... + + def get_result(self, accumulator: ACC) -> R: + ... + + def merge(self, a: ACC, b: ACC) -> ACC: + ... + + +class AggregatingState(Generic[T, ACC, R]): + def __init__( + self, + store: KvStore, + name: str, + acc_codec: Codec[ACC], + agg_func: AggregateFunc[T, ACC, R], + ): + validate_state_name(name) + if store is None: + raise KvError("aggregating state store must not be None") + if acc_codec is None: + raise KvError("aggregating state acc codec must not be None") + if agg_func is None: + raise KvError("aggregating state agg func must not be None") + self._store = store + self._acc_codec = acc_codec + self._agg_func = agg_func + state_name = name.strip() + self._ck = ComplexKey( + key_group=AGGREGATING_PREFIX, + key=state_name.encode("utf-8"), + namespace=b"data", + user_key=b"", + ) + + def add(self, value: T) -> None: + raw = self._store.get(self._ck) + if raw is None: + acc = self._agg_func.create_accumulator() + else: + acc = self._acc_codec.decode(raw) + new_acc = self._agg_func.add(value, acc) + self._store.put(self._ck, self._acc_codec.encode(new_acc)) + + def get(self) -> Tuple[Optional[R], bool]: + raw = self._store.get(self._ck) + if raw is None: + return (None, False) + acc = self._acc_codec.decode(raw) + return (self._agg_func.get_result(acc), True) + + def clear(self) -> None: + self._store.delete(self._ck) + + +__all__ = ["AggregateFunc", "AggregatingState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/list_state.py b/python/functionstream-api/src/fs_api/store/structures/list_state.py new file mode 100644 index 00000000..452cf588 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/list_state.py @@ -0,0 +1,95 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct +from typing import Generic, List, TypeVar + +from ..common import LIST_GROUP, validate_state_name +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +T = TypeVar("T") + + +class ListState(Generic[T]): + def __init__(self, store: KvStore, name: str, codec: Codec[T]): + validate_state_name(name) + if store is None: + raise KvError("list state store must not be None") + if codec is None: + raise KvError("list state codec must not be None") + self._store = store + self._codec = codec + state_name = name.strip() + self._ck = ComplexKey( + key_group=LIST_GROUP, + key=state_name.encode("utf-8"), + namespace=b"", + user_key=b"", + ) + + def add(self, value: T) -> None: + payload = self._serialize_one(value) + self._store.merge(self._ck, payload) + + def add_all(self, values: List[T]) -> None: + if not values: + return + payload = self._serialize_batch(values) + self._store.merge(self._ck, payload) + + def get(self) -> List[T]: + raw = self._store.get(self._ck) + if raw is None: + return [] + return self._deserialize(raw) + + def update(self, values: List[T]) -> None: + if len(values) == 0: + self.clear() + return + payload = self._serialize_batch(values) + self._store.put(self._ck, payload) + + def clear(self) -> None: + self._store.delete(self._ck) + + def _serialize_one(self, value: T) -> bytes: + encoded = self._codec.encode(value) + return struct.pack(">I", len(encoded)) + encoded + + def _serialize_batch(self, values: List[T]) -> bytes: + parts: List[bytes] = [] + for v in values: + encoded = self._codec.encode(v) + parts.append(struct.pack(">I", len(encoded)) + encoded) + return b"".join(parts) + + def _deserialize(self, raw: bytes) -> List[T]: + out: List[T] = [] + idx = 0 + while idx < len(raw): + if len(raw) - idx < 4: + raise KvError("corrupted list payload: truncated length") + (item_len,) = struct.unpack(">I", raw[idx : idx + 4]) + idx += 4 + if item_len < 0 or len(raw) - idx < item_len: + raise KvError("corrupted list payload: invalid element length") + item_raw = raw[idx : idx + item_len] + idx += item_len + out.append(self._codec.decode(item_raw)) + return out + + +__all__ = ["ListState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/map_state.py b/python/functionstream-api/src/fs_api/store/structures/map_state.py new file mode 100644 index 00000000..6d1bec6b --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/map_state.py @@ -0,0 +1,168 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Generator, Generic, List, Optional, Tuple, Type, TypeVar + +from ..common import MAP_GROUP, validate_state_name +from ..codec import ( + BoolCodec, + BytesCodec, + Codec, + OrderedFloat64Codec, + OrderedInt64Codec, + StringCodec, +) +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +K = TypeVar("K") +V = TypeVar("V") + + +@dataclass +class MapEntry(Generic[K, V]): + key: K + value: V + + +class MapState(Generic[K, V]): + def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: Codec[V]): + validate_state_name(name) + if store is None: + raise KvError("map state store must not be None") + if key_codec is None or value_codec is None: + raise KvError("map state codecs must not be None") + if not getattr(key_codec, "supports_ordered_keys", False): + raise KvError("map state key codec must support ordered key encoding for range scans") + self._store = store + self._key_codec = key_codec + self._value_codec = value_codec + self._key = name.encode("utf-8") + self._namespace = b"entries" + + @classmethod + def with_auto_key_codec( + cls, + store: KvStore, + name: str, + key_type: Type[K], + value_codec: Codec[V], + ) -> "MapState[K, V]": + key_codec = infer_ordered_key_codec(key_type) + return cls(store, name, key_codec, value_codec) + + def put(self, key: K, value: V) -> None: + encoded_key = self._key_codec.encode(key) + encoded_value = self._value_codec.encode(value) + self._store.put(self._ck(encoded_key), encoded_value) + + def get(self, key: K) -> Optional[V]: + encoded_key = self._key_codec.encode(key) + raw = self._store.get(self._ck(encoded_key)) + if raw is None: + return None + return self._value_codec.decode(raw) + + def delete(self, key: K) -> None: + encoded_key = self._key_codec.encode(key) + self._store.delete(self._ck(encoded_key)) + + def clear(self) -> None: + self._store.delete_prefix(self._ck(b"")) + + def all(self) -> Generator[Tuple[K, V], None, None]: + it = self._store.scan_complex(MAP_GROUP, self._key, self._namespace) + while it.has_next(): + item = it.next() + if item is None: + break + key_bytes, value_bytes = item + yield self._key_codec.decode(key_bytes), self._value_codec.decode(value_bytes) + + def len(self) -> int: + it = self._store.scan_complex(MAP_GROUP, self._key, self._namespace) + size = 0 + while it.has_next(): + item = it.next() + if item is None: + break + size += 1 + return size + + def entries(self) -> List[MapEntry[K, V]]: + it = self._store.scan_complex(MAP_GROUP, self._key, self._namespace) + out: List[MapEntry[K, V]] = [] + while it.has_next(): + item = it.next() + if item is None: + break + key_bytes, value_bytes = item + out.append( + MapEntry( + key=self._key_codec.decode(key_bytes), + value=self._value_codec.decode(value_bytes), + ) + ) + return out + + def range(self, start_inclusive: K, end_exclusive: K) -> List[MapEntry[K, V]]: + start_bytes = self._key_codec.encode(start_inclusive) + end_bytes = self._key_codec.encode(end_exclusive) + user_keys = self._store.list_complex(MAP_GROUP, self._key, self._namespace, start_bytes, end_bytes) + out: List[MapEntry[K, V]] = [] + for user_key in user_keys: + raw = self._store.get(self._ck(user_key)) + if raw is None: + raise KvError("map range key disappeared during scan") + out.append( + MapEntry( + key=self._key_codec.decode(user_key), + value=self._value_codec.decode(raw), + ) + ) + return out + + def _ck(self, user_key: bytes) -> ComplexKey: + return ComplexKey( + key_group=MAP_GROUP, + key=self._key, + namespace=self._namespace, + user_key=user_key, + ) + + +def infer_ordered_key_codec(key_type: Type[Any]) -> Codec[Any]: + if key_type is str: + return StringCodec() + if key_type is bytes: + return BytesCodec() + if key_type is bool: + return BoolCodec() + if key_type is int: + return OrderedInt64Codec() + if key_type is float: + return OrderedFloat64Codec() + raise KvError("unsupported map key type for auto codec") + + +def create_map_state_auto_key_codec( + store: KvStore, + name: str, + key_type: Type[K], + value_codec: Codec[V], +) -> MapState[K, V]: + return MapState.with_auto_key_codec(store, name, key_type, value_codec) + + +__all__ = ["MapEntry", "MapState", "infer_ordered_key_codec", "create_map_state_auto_key_codec"] diff --git a/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py new file mode 100644 index 00000000..5f118304 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py @@ -0,0 +1,84 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Iterator, Optional, Tuple, TypeVar + +from ..common import PQ_GROUP, validate_state_name +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +T = TypeVar("T") + + +class PriorityQueueState(Generic[T]): + def __init__(self, store: KvStore, name: str, codec: Codec[T]): + validate_state_name(name) + if store is None: + raise KvError("priority queue store must not be None") + if codec is None: + raise KvError("priority queue codec must not be None") + if not getattr(codec, "supports_ordered_keys", False): + raise KvError("priority queue value codec must be ordered") + self._store = store + self._codec = codec + state_name = name.strip() + self._key = state_name.encode("utf-8") + self._namespace = b"items" + + def _ck(self, user_key: bytes) -> ComplexKey: + return ComplexKey( + key_group=PQ_GROUP, + key=self._key, + namespace=self._namespace, + user_key=user_key, + ) + + def add(self, value: T) -> None: + user_key = self._codec.encode(value) + self._store.put(self._ck(user_key), b"") + + def peek(self) -> Tuple[Optional[T], bool]: + it = self._store.scan_complex(PQ_GROUP, self._key, self._namespace) + if not it.has_next(): + return (None, False) + item = it.next() + if item is None: + return (None, False) + user_key, _ = item + return (self._codec.decode(user_key), True) + + def poll(self) -> Tuple[Optional[T], bool]: + val, found = self.peek() + if not found or val is None: + return (val, found) + user_key = self._codec.encode(val) + self._store.delete(self._ck(user_key)) + return (val, True) + + def clear(self) -> None: + self._store.delete_prefix( + ComplexKey(key_group=PQ_GROUP, key=self._key, namespace=self._namespace, user_key=b"") + ) + + def all(self) -> Iterator[T]: + it = self._store.scan_complex(PQ_GROUP, self._key, self._namespace) + while it.has_next(): + item = it.next() + if item is None: + break + user_key, _ = item + yield self._codec.decode(user_key) + + +__all__ = ["PriorityQueueState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/reducing_state.py b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py new file mode 100644 index 00000000..e201a11c --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py @@ -0,0 +1,63 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Generic, Optional, Tuple, TypeVar + +from ..common import REDUCING_PREFIX, validate_state_name +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +V = TypeVar("V") + +ReduceFunc = Callable[[V, V], V] + + +class ReducingState(Generic[V]): + def __init__(self, store: KvStore, name: str, value_codec: Codec[V], reduce_func: ReduceFunc[V]): + validate_state_name(name) + if store is None: + raise KvError("reducing state store must not be None") + if value_codec is None or reduce_func is None: + raise KvError("reducing state value codec and reduce function are required") + self._store = store + self._value_codec = value_codec + self._reduce_func = reduce_func + state_name = name.strip() + self._ck = ComplexKey( + key_group=REDUCING_PREFIX, + key=state_name.encode("utf-8"), + namespace=b"data", + user_key=b"", + ) + + def add(self, value: V) -> None: + raw = self._store.get(self._ck) + if raw is None: + result = value + else: + old_value = self._value_codec.decode(raw) + result = self._reduce_func(old_value, value) + self._store.put(self._ck, self._value_codec.encode(result)) + + def get(self) -> Tuple[Optional[V], bool]: + raw = self._store.get(self._ck) + if raw is None: + return (None, False) + return (self._value_codec.decode(raw), True) + + def clear(self) -> None: + self._store.delete(self._ck) + + +__all__ = ["ReduceFunc", "ReducingState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/value_state.py b/python/functionstream-api/src/fs_api/store/structures/value_state.py new file mode 100644 index 00000000..ad8dd4ec --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/value_state.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Optional, Tuple, TypeVar + +from ..common import VALUE_PREFIX, validate_state_name +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +T = TypeVar("T") + + +class ValueState(Generic[T]): + def __init__(self, store: KvStore, name: str, codec: Codec[T]): + validate_state_name(name) + if store is None: + raise KvError("value state store must not be None") + if codec is None: + raise KvError("value state codec must not be None") + self._store = store + self._codec = codec + state_name = name.strip() + self._ck = ComplexKey( + key_group=VALUE_PREFIX, + key=state_name.encode("utf-8"), + namespace=b"data", + user_key=b"", + ) + + def update(self, value: T) -> None: + self._store.put(self._ck, self._codec.encode(value)) + + def value(self) -> Tuple[Optional[T], bool]: + raw = self._store.get(self._ck) + if raw is None: + return (None, False) + return (self._codec.decode(raw), True) + + def clear(self) -> None: + self._store.delete(self._ck) + + +__all__ = ["ValueState"] diff --git a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py index 0dd9d624..c40537e3 100644 --- a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py +++ b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py @@ -13,7 +13,19 @@ from typing import Dict, List, Tuple from fs_api.context import Context -from fs_api.store import KvStore +from fs_api.store import ( + Codec, + KvStore, + ValueState, + MapState, + ListState, + PriorityQueueState, + KeyedStateFactory, + KeyedValueState, + KeyedMapState, + KeyedListState, + KeyedPriorityQueueState, +) from .fs_collector import emit, emit_watermark from .fs_store import FSStore @@ -60,6 +72,42 @@ def getOrCreateKVStore(self, name: str) -> KvStore: def getConfig(self) -> Dict[str, str]: return self._CONFIG.copy() + def getOrCreateValueState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> ValueState: + store = self.getOrCreateKVStore(store_name) + return ValueState(store, state_name, codec) + + def getOrCreateMapState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> MapState: + store = self.getOrCreateKVStore(store_name) + return MapState(store, state_name, key_codec, value_codec) + + def getOrCreateListState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> ListState: + store = self.getOrCreateKVStore(store_name) + return ListState(store, state_name, codec) + + def getOrCreatePriorityQueueState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> PriorityQueueState: + store = self.getOrCreateKVStore(store_name) + return PriorityQueueState(store, state_name, codec) + + def getOrCreateKeyedStateFactory(self, state_name: str, store_name: str = "__fssdk_structured_state__") -> KeyedStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedStateFactory(store, state_name) + + def getOrCreateKeyedValueState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedValueState: + factory = self.getOrCreateKeyedStateFactory(state_name, store_name) + return factory.new_keyed_value(key_codec, value_codec) + + def getOrCreateKeyedMapState(self, state_name: str, key_codec: Codec, map_key_codec: Codec, map_value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedMapState: + factory = self.getOrCreateKeyedStateFactory(state_name, store_name) + return factory.new_keyed_map(key_codec, map_key_codec, map_value_codec) + + def getOrCreateKeyedListState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedListState: + factory = self.getOrCreateKeyedStateFactory(state_name, store_name) + return factory.new_keyed_list(key_codec, value_codec) + + def getOrCreateKeyedPriorityQueueState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedPriorityQueueState: + factory = self.getOrCreateKeyedStateFactory(state_name, store_name) + return factory.new_keyed_priority_queue(key_codec, value_codec) + __all__ = ['WitContext', 'convert_config_to_dict'] From 16ac3b492676a2150dd4c26a6d63a056ec448b59 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Mon, 9 Mar 2026 00:23:12 +0800 Subject: [PATCH 2/9] update --- go-sdk/impl/context.go | 16 +- go-sdk/impl/state.go | 132 +++++++++++++++++ go-sdk/state/common/common.go | 11 -- go-sdk/state/context.go | 137 ------------------ go-sdk/state/state.go | 51 ------- go-sdk/state/structures/aggregating.go | 17 +-- go-sdk/state/structures/list.go | 16 +- go-sdk/state/structures/map.go | 20 +-- go-sdk/state/structures/priority_queue.go | 16 +- go-sdk/state/structures/reducing.go | 15 +- go-sdk/state/structures/value.go | 16 +- .../store/structures/aggregating_state.py | 8 +- .../src/fs_api/store/structures/list_state.py | 6 +- .../src/fs_api/store/structures/map_state.py | 17 ++- .../store/structures/priority_queue_state.py | 16 +- .../fs_api/store/structures/reducing_state.py | 8 +- .../fs_api/store/structures/value_state.py | 8 +- 17 files changed, 202 insertions(+), 308 deletions(-) create mode 100644 go-sdk/impl/state.go delete mode 100644 go-sdk/state/context.go delete mode 100644 go-sdk/state/state.go diff --git a/go-sdk/impl/context.go b/go-sdk/impl/context.go index 54d374a1..a6b5626b 100644 --- a/go-sdk/impl/context.go +++ b/go-sdk/impl/context.go @@ -13,15 +13,12 @@ package impl import ( - "strings" - "sync" - "github.com/functionstream/function-stream/go-sdk/api" "github.com/functionstream/function-stream/go-sdk/bindings/functionstream/core/collector" + "strings" ) type runtimeContext struct { - mu sync.RWMutex config map[string]string stores map[string]*storeImpl closed bool @@ -35,9 +32,7 @@ func newRuntimeContext(config map[string]string) *runtimeContext { } func (c *runtimeContext) Emit(targetID uint32, data []byte) error { - c.mu.RLock() closed := c.closed - c.mu.RUnlock() if closed { return api.NewError(api.ErrRuntimeClosed, "emit on closed context") } @@ -46,9 +41,7 @@ func (c *runtimeContext) Emit(targetID uint32, data []byte) error { } func (c *runtimeContext) EmitWatermark(targetID uint32, watermark uint64) error { - c.mu.RLock() closed := c.closed - c.mu.RUnlock() if closed { return api.NewError(api.ErrRuntimeClosed, "emit watermark on closed context") } @@ -62,8 +55,6 @@ func (c *runtimeContext) GetOrCreateStore(name string) (api.Store, error) { return nil, api.NewError(api.ErrStoreInvalidName, "store name must not be empty") } - c.mu.Lock() - defer c.mu.Unlock() if c.closed { return nil, api.NewError(api.ErrRuntimeClosed, "store request on closed context") } @@ -77,21 +68,16 @@ func (c *runtimeContext) GetOrCreateStore(name string) (api.Store, error) { } func (c *runtimeContext) Config() map[string]string { - c.mu.RLock() - defer c.mu.RUnlock() return cloneStringMap(c.config) } func (c *runtimeContext) Close() error { - c.mu.Lock() if c.closed { - c.mu.Unlock() return nil } c.closed = true stores := c.stores c.stores = make(map[string]*storeImpl) - c.mu.Unlock() var firstErr error for _, store := range stores { diff --git a/go-sdk/impl/state.go b/go-sdk/impl/state.go new file mode 100644 index 00000000..d01d0922 --- /dev/null +++ b/go-sdk/impl/state.go @@ -0,0 +1,132 @@ +package impl + +import ( + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/keyed" + "github.com/functionstream/function-stream/go-sdk/state/structures" +) + +func getStoreFromContext(ctx api.Context, storeName string) (*storeImpl, error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + s, ok := store.(*storeImpl) + if !ok { + return nil, api.NewError(api.ErrStoreInternal, "store %q is not the default implementation", storeName) + } + return s, nil +} + +func NewValueState[T any](ctx api.Context, storeName string, valueCodec codec.Codec[T]) (*structures.ValueState[T], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewValueState(s, valueCodec) +} + +func NewListState[T any](ctx api.Context, storeName string, itemCodec codec.Codec[T]) (*structures.ListState[T], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewListState(s, itemCodec) +} + +func NewMapState[K any, V any](ctx api.Context, storeName string, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*structures.MapState[K, V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewMapState(s, keyCodec, valueCodec) +} + +func NewMapStateAutoKeyCodec[K any, V any](ctx api.Context, storeName string, valueCodec codec.Codec[V]) (*structures.MapState[K, V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewMapStateAutoKeyCodec[K, V](s, valueCodec) +} + +func NewPriorityQueueState[T any](ctx api.Context, storeName string, itemCodec codec.Codec[T]) (*structures.PriorityQueueState[T], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewPriorityQueueState(s, itemCodec) +} + +func NewAggregatingState[T any, ACC any, R any](ctx api.Context, storeName string, accCodec codec.Codec[ACC], aggFunc structures.AggregateFunc[T, ACC, R]) (*structures.AggregatingState[T, ACC, R], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewAggregatingState(s, accCodec, aggFunc) +} + +func NewReducingState[V any](ctx api.Context, storeName string, valueCodec codec.Codec[V], reduceFunc structures.ReduceFunc[V]) (*structures.ReducingState[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewReducingState(s, valueCodec, reduceFunc) +} + +func NewKeyedListStateFactory[V any](ctx api.Context, storeName, name string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedListStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedListStateFactory(s, name, keyGroup, valueCodec) +} + +func NewKeyedListStateFactoryAutoCodec[V any](ctx api.Context, storeName, name string, keyGroup []byte) (*keyed.KeyedListStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedListStateFactoryAutoCodec[V](s, name, keyGroup) +} + +func NewKeyedValueStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedValueStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedValueStateFactory(s, keyGroup, valueCodec) +} + +func NewKeyedMapStateFactory[MK any, MV any](ctx api.Context, storeName string, keyGroup []byte, keyCodec codec.Codec[MK], valueCodec codec.Codec[MV]) (*keyed.KeyedMapStateFactory[MK, MV], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedMapStateFactory(s, keyGroup, keyCodec, valueCodec) +} + +func NewKeyedPriorityQueueStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, itemCodec codec.Codec[V]) (*keyed.KeyedPriorityQueueStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedPriorityQueueStateFactory(s, keyGroup, itemCodec) +} + +func NewKeyedAggregatingStateFactory[T any, ACC any, R any](ctx api.Context, storeName string, keyGroup []byte, accCodec codec.Codec[ACC], aggFunc keyed.AggregateFunc[T, ACC, R]) (*keyed.KeyedAggregatingStateFactory[T, ACC, R], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedAggregatingStateFactory(s, keyGroup, accCodec, aggFunc) +} + +func NewKeyedReducingStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V], reduceFunc keyed.ReduceFunc[V]) (*keyed.KeyedReducingStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedReducingStateFactory(s, keyGroup, valueCodec, reduceFunc) +} diff --git a/go-sdk/state/common/common.go b/go-sdk/state/common/common.go index de4a23b0..2551e884 100644 --- a/go-sdk/state/common/common.go +++ b/go-sdk/state/common/common.go @@ -10,17 +10,6 @@ import ( type Store = api.Store -const ( - StateValuePrefix = "__fssdk__/value/" - StateListPrefix = "__fssdk__/list/" - StatePQPrefix = "__fssdk__/priority_queue/" - StateMapGroup = "__fssdk__/map" - StateListGroup = "__fssdk__/list" - StatePQGroup = "__fssdk__/priority_queue" - StateAggregatingPrefix = "__fssdk__/aggregating/" - StateReducingPrefix = "__fssdk__/reducing/" -) - func ValidateStateName(name string) (string, error) { stateName := strings.TrimSpace(name) if stateName == "" { diff --git a/go-sdk/state/context.go b/go-sdk/state/context.go deleted file mode 100644 index 9ef76d3c..00000000 --- a/go-sdk/state/context.go +++ /dev/null @@ -1,137 +0,0 @@ -package state - -import "fmt" - -const DefaultStateStoreName = "__fssdk_structured_state__" - -func NewValueStateFromContext[T any](ctx Context, stateName string, codec Codec[T]) (*ValueState[T], error) { - return NewValueStateFromContextWithStore[T](ctx, DefaultStateStoreName, stateName, codec) -} - -func NewValueStateFromContextWithStore[T any](ctx Context, storeName string, stateName string, codec Codec[T]) (*ValueState[T], error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewValueState[T](store, stateName, codec) -} - -func NewMapStateFromContext[K any, V any](ctx Context, stateName string, keyCodec Codec[K], valueCodec Codec[V]) (*MapState[K, V], error) { - return NewMapStateFromContextWithStore[K, V](ctx, DefaultStateStoreName, stateName, keyCodec, valueCodec) -} - -func NewMapStateFromContextWithStore[K any, V any](ctx Context, storeName string, stateName string, keyCodec Codec[K], valueCodec Codec[V]) (*MapState[K, V], error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewMapState[K, V](store, stateName, keyCodec, valueCodec) -} - -func NewListStateFromContext[T any](ctx Context, stateName string, codec Codec[T]) (*ListState[T], error) { - return NewListStateFromContextWithStore[T](ctx, DefaultStateStoreName, stateName, codec) -} - -func NewListStateFromContextWithStore[T any](ctx Context, storeName string, stateName string, codec Codec[T]) (*ListState[T], error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewListState[T](store, stateName, codec) -} - -func NewPriorityQueueStateFromContext[T any](ctx Context, stateName string, codec Codec[T]) (*PriorityQueueState[T], error) { - return NewPriorityQueueStateFromContextWithStore[T](ctx, DefaultStateStoreName, stateName, codec) -} - -func NewPriorityQueueStateFromContextWithStore[T any](ctx Context, storeName string, stateName string, codec Codec[T]) (*PriorityQueueState[T], error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewPriorityQueueState[T](store, stateName, codec) -} - -func NewKeyedStateFactoryFromContext(ctx Context, stateName string) (*KeyedStateFactory, error) { - return NewKeyedStateFactoryFromContextWithStore(ctx, DefaultStateStoreName, stateName) -} - -func NewKeyedStateFactoryFromContextWithStore(ctx Context, storeName string, stateName string) (*KeyedStateFactory, error) { - return NewKeyedValueStateFactoryFromContextWithStore(ctx, storeName, stateName) -} - -func NewKeyedValueStateFactoryFromContext(ctx Context, stateName string) (*KeyedValueStateFactory, error) { - return NewKeyedValueStateFactoryFromContextWithStore(ctx, DefaultStateStoreName, stateName) -} - -func NewKeyedValueStateFactoryFromContextWithStore(ctx Context, storeName string, stateName string) (*KeyedValueStateFactory, error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewKeyedValueStateFactory(store, stateName) -} - -func NewKeyedMapStateFactoryFromContext[MK any, MV any](ctx Context, stateName string, mapKeyCodec Codec[MK], mapValueCodec Codec[MV]) (*KeyedMapStateFactory[MK, MV], error) { - return NewKeyedMapStateFactoryFromContextWithStore[MK, MV](ctx, DefaultStateStoreName, stateName, mapKeyCodec, mapValueCodec) -} - -func NewKeyedMapStateFactoryFromContextWithStore[MK any, MV any](ctx Context, storeName string, stateName string, mapKeyCodec Codec[MK], mapValueCodec Codec[MV]) (*KeyedMapStateFactory[MK, MV], error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewKeyedMapStateFactory[MK, MV](store, stateName, mapKeyCodec, mapValueCodec) -} - -func NewKeyedListStateFactoryFromContext[V any](ctx Context, stateName string, keyGroup []byte, valueCodec Codec[V]) (*KeyedListStateFactory[V], error) { - return NewKeyedListStateFactoryFromContextWithStore[V](ctx, DefaultStateStoreName, stateName, keyGroup, valueCodec) -} - -func NewKeyedListStateFactoryFromContextWithStore[V any](ctx Context, storeName string, stateName string, keyGroup []byte, valueCodec Codec[V]) (*KeyedListStateFactory[V], error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewKeyedListStateFactory[V](store, stateName, keyGroup, valueCodec) -} - -// NewKeyedListStateFactoryAutoCodecFromContext 从 Context 创建 KeyedListStateFactory，不显式传入 valueCodec， -// 由工厂根据类型 V 自动选择 codec（基础类型 / string / struct 用 JSON）。 -func NewKeyedListStateFactoryAutoCodecFromContext[V any](ctx Context, stateName string, keyGroup []byte) (*KeyedListStateFactory[V], error) { - return NewKeyedListStateFactoryAutoCodecFromContextWithStore[V](ctx, DefaultStateStoreName, stateName, keyGroup) -} - -func NewKeyedListStateFactoryAutoCodecFromContextWithStore[V any](ctx Context, storeName string, stateName string, keyGroup []byte) (*KeyedListStateFactory[V], error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewKeyedListStateFactoryAutoCodec[V](store, stateName, keyGroup) -} - -func NewKeyedPriorityQueueStateFactoryFromContext(ctx Context, stateName string) (*KeyedPriorityQueueStateFactory, error) { - return NewKeyedPriorityQueueStateFactoryFromContextWithStore(ctx, DefaultStateStoreName, stateName) -} - -func NewKeyedPriorityQueueStateFactoryFromContextWithStore(ctx Context, storeName string, stateName string) (*KeyedPriorityQueueStateFactory, error) { - store, err := requireStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return NewKeyedPriorityQueueStateFactory(store, stateName) -} - -func requireStoreFromContext(ctx Context, storeName string) (Store, error) { - if ctx == nil { - return nil, fmt.Errorf("context must not be nil") - } - store, err := ctx.GetOrCreateStore(storeName) - if err != nil { - return nil, err - } - if store == nil { - return nil, fmt.Errorf("context returned nil store for %q", storeName) - } - return store, nil -} diff --git a/go-sdk/state/state.go b/go-sdk/state/state.go deleted file mode 100644 index 2bfad332..00000000 --- a/go-sdk/state/state.go +++ /dev/null @@ -1,51 +0,0 @@ -package state - -import ( - "github.com/functionstream/function-stream/go-sdk/api" - statecodec "github.com/functionstream/function-stream/go-sdk/state/codec" -) - -type Store = api.Store -type Context = api.Context -// -//type Codec[T any] = statecodec.Codec[T] -// -//type JSONCodec[T any] = statecodec.JSONCodec[T] -type BytesCodec = statecodec.BytesCodec -type StringCodec = statecodec.StringCodec -type BoolCodec = statecodec.BoolCodec - -type Int64Codec = statecodec.Int64Codec -type Uint64Codec = statecodec.Uint64Codec -type Int32Codec = statecodec.Int32Codec -type Uint32Codec = statecodec.Uint32Codec -type Float64Codec = statecodec.Float64Codec -type Float32Codec = statecodec.Float32Codec - -type OrderedInt64Codec = statecodec.OrderedInt64Codec -type OrderedIntCodec = statecodec.OrderedIntCodec -type OrderedUint64Codec = statecodec.OrderedUint64Codec -type OrderedUintCodec = statecodec.OrderedUintCodec -type OrderedInt32Codec = statecodec.OrderedInt32Codec -type OrderedUint32Codec = statecodec.OrderedUint32Codec -type OrderedFloat64Codec = statecodec.OrderedFloat64Codec -type OrderedFloat32Codec = statecodec.OrderedFloat32Codec -// -//type ValueState[T any] = structuresstate.ValueState[T] -//type MapEntry[K any, V any] = structuresstate.MapEntry[K, V] -//type MapState[K any, V any] = structuresstate.MapState[K, V] -//type ListState[T any] = structuresstate.ListState[T] -//type PriorityQueueState[T any] = structuresstate.PriorityQueueState[T] -// -//type KeyedStateFactory = keyedstate.KeyedValueStateFactory -//type KeyedValueStateFactory = keyedstate.KeyedValueStateFactory -//type KeyedMapStateFactory[MK any, MV any] = keyedstate.KeyedMapStateFactory[MK, MV] -//type KeyedListStateFactory[V any] = keyedstate.KeyedListStateFactory[V] -//type KeyedPriorityQueueStateFactory = keyedstate.KeyedPriorityQueueStateFactory -//type KeyedValueState[K any, V any] = keyedstate.KeyedValueState[K, V] -//type KeyedMapEntry[K any, V any] = keyedstate.KeyedMapEntry[K, V] -//type KeyedMapState[MK any, MV any] = keyedstate.KeyedMapState[MK, MV] -//type KeyedListState[V any] = keyedstate.KeyedListState[V] -//type KeyedPriorityItem[T any] = keyedstate.KeyedPriorityItem[T] -//type KeyedPriorityQueueState[K any, V any] = keyedstate.KeyedPriorityQueueState[K, V] -// diff --git a/go-sdk/state/structures/aggregating.go b/go-sdk/state/structures/aggregating.go index 6e11c68c..24787e1d 100644 --- a/go-sdk/state/structures/aggregating.go +++ b/go-sdk/state/structures/aggregating.go @@ -24,27 +24,22 @@ type AggregatingState[T any, ACC any, R any] struct { func NewAggregatingState[T any, ACC any, R any]( store common.Store, - name string, accCodec codec.Codec[ACC], aggFunc AggregateFunc[T, ACC, R], ) (*AggregatingState[T, ACC, R], error) { - stateName, err := common.ValidateStateName(name) - if err != nil { - return nil, err - } if store == nil { - return nil, api.NewError(api.ErrStoreInternal, "aggregating state %q store must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "aggregating state store must not be nil") } if accCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "aggregating state %q acc codec must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "aggregating state acc codec must not be nil") } if aggFunc == nil { - return nil, api.NewError(api.ErrStoreInternal, "aggregating state %q agg func must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "aggregating state agg func must not be nil") } ck := api.ComplexKey{ - KeyGroup: []byte(common.StateAggregatingPrefix), - Key: []byte(stateName), - Namespace: []byte("data"), + KeyGroup: []byte{}, + Key: []byte{}, + Namespace: []byte{}, UserKey: []byte{}, } return &AggregatingState[T, ACC, R]{ diff --git a/go-sdk/state/structures/list.go b/go-sdk/state/structures/list.go index 2d5bbe55..7d48c2df 100644 --- a/go-sdk/state/structures/list.go +++ b/go-sdk/state/structures/list.go @@ -19,24 +19,20 @@ type ListState[T any] struct { serializeBatch func([]T) ([]byte, error) } -func NewListState[T any](store common.Store, name string, itemCodec codec.Codec[T]) (*ListState[T], error) { - stateName, err := common.ValidateStateName(name) - if err != nil { - return nil, err - } +func NewListState[T any](store common.Store, itemCodec codec.Codec[T]) (*ListState[T], error) { if store == nil { - return nil, api.NewError(api.ErrStoreInternal, "list state %q store must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "list state store must not be nil") } if itemCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "list state %q codec must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "list state codec must not be nil") } fixedSize, isFixed := codec.FixedEncodedSize[T](itemCodec) l := &ListState[T]{ store: store, complexKey: api.ComplexKey{ - KeyGroup: []byte(common.StateListGroup), - Key: []byte(stateName), - Namespace: []byte(""), + KeyGroup: []byte{}, + Key: []byte{}, + Namespace: []byte{}, UserKey: []byte{}, }, codec: itemCodec, diff --git a/go-sdk/state/structures/map.go b/go-sdk/state/structures/map.go index b14d1bcd..fb9fa7e2 100644 --- a/go-sdk/state/structures/map.go +++ b/go-sdk/state/structures/map.go @@ -23,32 +23,28 @@ type MapState[K any, V any] struct { valueCodec codec.Codec[V] } -func NewMapState[K any, V any](store common.Store, name string, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*MapState[K, V], error) { - stateName, err := common.ValidateStateName(name) - if err != nil { - return nil, err - } +func NewMapState[K any, V any](store common.Store, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*MapState[K, V], error) { if store == nil { - return nil, api.NewError(api.ErrStoreInternal, "map state %q store must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "map state store must not be nil") } if keyCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "map state %q key codec must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "map state key codec must not be nil") } if valueCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "map state %q value codec must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "map state value codec must not be nil") } if !keyCodec.IsOrderedKeyCodec() { - return nil, api.NewError(api.ErrStoreInternal, "map state %q key codec must be ordered (IsOrderedKeyCodec)", stateName) + return nil, api.NewError(api.ErrStoreInternal, "map state key codec must be ordered (IsOrderedKeyCodec)") } - return &MapState[K, V]{store: store, keyGroup: []byte(common.StateMapGroup), key: []byte(stateName), namespace: []byte("entries"), keyCodec: keyCodec, valueCodec: valueCodec}, nil + return &MapState[K, V]{store: store, keyGroup: []byte{}, key: []byte{}, namespace: []byte{}, keyCodec: keyCodec, valueCodec: valueCodec}, nil } -func NewMapStateAutoKeyCodec[K any, V any](store common.Store, name string, valueCodec codec.Codec[V]) (*MapState[K, V], error) { +func NewMapStateAutoKeyCodec[K any, V any](store common.Store, valueCodec codec.Codec[V]) (*MapState[K, V], error) { autoKeyCodec, err := inferOrderedKeyCodec[K]() if err != nil { return nil, err } - return NewMapState[K, V](store, name, autoKeyCodec, valueCodec) + return NewMapState[K, V](store, autoKeyCodec, valueCodec) } func (m *MapState[K, V]) Put(key K, value V) error { diff --git a/go-sdk/state/structures/priority_queue.go b/go-sdk/state/structures/priority_queue.go index 798833bb..d6fbb662 100644 --- a/go-sdk/state/structures/priority_queue.go +++ b/go-sdk/state/structures/priority_queue.go @@ -17,25 +17,21 @@ type PriorityQueueState[T any] struct { valueCodec codec.Codec[T] } -func NewPriorityQueueState[T any](store common.Store, name string, itemCodec codec.Codec[T]) (*PriorityQueueState[T], error) { - stateName, err := common.ValidateStateName(name) - if err != nil { - return nil, err - } +func NewPriorityQueueState[T any](store common.Store, itemCodec codec.Codec[T]) (*PriorityQueueState[T], error) { if store == nil { - return nil, api.NewError(api.ErrStoreInternal, "priority queue state %q store must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "priority queue state store must not be nil") } if itemCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "priority queue state %q codec must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "priority queue state codec must not be nil") } if !itemCodec.IsOrderedKeyCodec() { return nil, api.NewError(api.ErrStoreInternal, "priority queue value codec must be ordered") } return &PriorityQueueState[T]{ store: store, - keyGroup: []byte(common.StatePQGroup), - key: []byte(stateName), - namespace: []byte("items"), + keyGroup: []byte{}, + key: []byte{}, + namespace: []byte{}, valueCodec: itemCodec, }, nil } diff --git a/go-sdk/state/structures/reducing.go b/go-sdk/state/structures/reducing.go index 99f4b6a9..8af284fd 100644 --- a/go-sdk/state/structures/reducing.go +++ b/go-sdk/state/structures/reducing.go @@ -19,24 +19,19 @@ type ReducingState[V any] struct { func NewReducingState[V any]( store common.Store, - name string, valueCodec codec.Codec[V], reduceFunc ReduceFunc[V], ) (*ReducingState[V], error) { - stateName, err := common.ValidateStateName(name) - if err != nil { - return nil, err - } if store == nil { - return nil, api.NewError(api.ErrStoreInternal, "reducing state %q store must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "reducing state store must not be nil") } if valueCodec == nil || reduceFunc == nil { - return nil, api.NewError(api.ErrStoreInternal, "reducing state %q value codec and reduce function are required", stateName) + return nil, api.NewError(api.ErrStoreInternal, "reducing state value codec and reduce function are required") } ck := api.ComplexKey{ - KeyGroup: []byte(common.StateReducingPrefix), - Key: []byte(stateName), - Namespace: []byte("data"), + KeyGroup: []byte{}, + Key: []byte{}, + Namespace: []byte{}, UserKey: []byte{}, } return &ReducingState[V]{ diff --git a/go-sdk/state/structures/value.go b/go-sdk/state/structures/value.go index 0950def1..8f4e549e 100644 --- a/go-sdk/state/structures/value.go +++ b/go-sdk/state/structures/value.go @@ -14,21 +14,17 @@ type ValueState[T any] struct { codec codec.Codec[T] } -func NewValueState[T any](store common.Store, name string, valueCodec codec.Codec[T]) (*ValueState[T], error) { - stateName, err := common.ValidateStateName(name) - if err != nil { - return nil, err - } +func NewValueState[T any](store common.Store, valueCodec codec.Codec[T]) (*ValueState[T], error) { if store == nil { - return nil, api.NewError(api.ErrStoreInternal, "value state %q store must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "value state store must not be nil") } if valueCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "value state %q codec must not be nil", stateName) + return nil, api.NewError(api.ErrStoreInternal, "value state codec must not be nil") } ck := api.ComplexKey{ - KeyGroup: []byte(common.StateValuePrefix), - Key: []byte(stateName), - Namespace: []byte("data"), + KeyGroup: []byte{}, + Key: []byte{}, + Namespace: []byte{}, UserKey: []byte{}, } return &ValueState[T]{store: store, complexKey: ck, codec: valueCodec}, nil diff --git a/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py index 236952aa..1473c2d4 100644 --- a/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py @@ -12,7 +12,7 @@ from typing import Generic, Optional, Protocol, Tuple, TypeVar -from ..common import AGGREGATING_PREFIX, validate_state_name +from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -57,9 +57,9 @@ def __init__( self._agg_func = agg_func state_name = name.strip() self._ck = ComplexKey( - key_group=AGGREGATING_PREFIX, - key=state_name.encode("utf-8"), - namespace=b"data", + key_group=b"", + key=b"", + namespace=b"", user_key=b"", ) diff --git a/python/functionstream-api/src/fs_api/store/structures/list_state.py b/python/functionstream-api/src/fs_api/store/structures/list_state.py index 452cf588..0ec1970c 100644 --- a/python/functionstream-api/src/fs_api/store/structures/list_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/list_state.py @@ -13,7 +13,7 @@ import struct from typing import Generic, List, TypeVar -from ..common import LIST_GROUP, validate_state_name +from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -33,8 +33,8 @@ def __init__(self, store: KvStore, name: str, codec: Codec[T]): self._codec = codec state_name = name.strip() self._ck = ComplexKey( - key_group=LIST_GROUP, - key=state_name.encode("utf-8"), + key_group=b"", + key=b"", namespace=b"", user_key=b"", ) diff --git a/python/functionstream-api/src/fs_api/store/structures/map_state.py b/python/functionstream-api/src/fs_api/store/structures/map_state.py index 6d1bec6b..0c19edcb 100644 --- a/python/functionstream-api/src/fs_api/store/structures/map_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/map_state.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from typing import Any, Generator, Generic, List, Optional, Tuple, Type, TypeVar -from ..common import MAP_GROUP, validate_state_name +from ..common import validate_state_name from ..codec import ( BoolCodec, BytesCodec, @@ -48,8 +48,9 @@ def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: self._store = store self._key_codec = key_codec self._value_codec = value_codec - self._key = name.encode("utf-8") - self._namespace = b"entries" + self._key_group = b"" + self._key = b"" + self._namespace = b"" @classmethod def with_auto_key_codec( @@ -82,7 +83,7 @@ def clear(self) -> None: self._store.delete_prefix(self._ck(b"")) def all(self) -> Generator[Tuple[K, V], None, None]: - it = self._store.scan_complex(MAP_GROUP, self._key, self._namespace) + it = self._store.scan_complex(self._key_group, self._key, self._namespace) while it.has_next(): item = it.next() if item is None: @@ -91,7 +92,7 @@ def all(self) -> Generator[Tuple[K, V], None, None]: yield self._key_codec.decode(key_bytes), self._value_codec.decode(value_bytes) def len(self) -> int: - it = self._store.scan_complex(MAP_GROUP, self._key, self._namespace) + it = self._store.scan_complex(self._key_group, self._key, self._namespace) size = 0 while it.has_next(): item = it.next() @@ -101,7 +102,7 @@ def len(self) -> int: return size def entries(self) -> List[MapEntry[K, V]]: - it = self._store.scan_complex(MAP_GROUP, self._key, self._namespace) + it = self._store.scan_complex(self._key_group, self._key, self._namespace) out: List[MapEntry[K, V]] = [] while it.has_next(): item = it.next() @@ -119,7 +120,7 @@ def entries(self) -> List[MapEntry[K, V]]: def range(self, start_inclusive: K, end_exclusive: K) -> List[MapEntry[K, V]]: start_bytes = self._key_codec.encode(start_inclusive) end_bytes = self._key_codec.encode(end_exclusive) - user_keys = self._store.list_complex(MAP_GROUP, self._key, self._namespace, start_bytes, end_bytes) + user_keys = self._store.list_complex(self._key_group, self._key, self._namespace, start_bytes, end_bytes) out: List[MapEntry[K, V]] = [] for user_key in user_keys: raw = self._store.get(self._ck(user_key)) @@ -135,7 +136,7 @@ def range(self, start_inclusive: K, end_exclusive: K) -> List[MapEntry[K, V]]: def _ck(self, user_key: bytes) -> ComplexKey: return ComplexKey( - key_group=MAP_GROUP, + key_group=self._key_group, key=self._key, namespace=self._namespace, user_key=user_key, diff --git a/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py index 5f118304..0818e9fb 100644 --- a/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py @@ -12,7 +12,7 @@ from typing import Generic, Iterator, Optional, Tuple, TypeVar -from ..common import PQ_GROUP, validate_state_name +from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -32,13 +32,13 @@ def __init__(self, store: KvStore, name: str, codec: Codec[T]): raise KvError("priority queue value codec must be ordered") self._store = store self._codec = codec - state_name = name.strip() - self._key = state_name.encode("utf-8") - self._namespace = b"items" + self._key_group = b"" + self._key = b"" + self._namespace = b"" def _ck(self, user_key: bytes) -> ComplexKey: return ComplexKey( - key_group=PQ_GROUP, + key_group=self._key_group, key=self._key, namespace=self._namespace, user_key=user_key, @@ -49,7 +49,7 @@ def add(self, value: T) -> None: self._store.put(self._ck(user_key), b"") def peek(self) -> Tuple[Optional[T], bool]: - it = self._store.scan_complex(PQ_GROUP, self._key, self._namespace) + it = self._store.scan_complex(self._key_group, self._key, self._namespace) if not it.has_next(): return (None, False) item = it.next() @@ -68,11 +68,11 @@ def poll(self) -> Tuple[Optional[T], bool]: def clear(self) -> None: self._store.delete_prefix( - ComplexKey(key_group=PQ_GROUP, key=self._key, namespace=self._namespace, user_key=b"") + ComplexKey(key_group=self._key_group, key=self._key, namespace=self._namespace, user_key=b"") ) def all(self) -> Iterator[T]: - it = self._store.scan_complex(PQ_GROUP, self._key, self._namespace) + it = self._store.scan_complex(self._key_group, self._key, self._namespace) while it.has_next(): item = it.next() if item is None: diff --git a/python/functionstream-api/src/fs_api/store/structures/reducing_state.py b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py index e201a11c..b8e039a4 100644 --- a/python/functionstream-api/src/fs_api/store/structures/reducing_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py @@ -12,7 +12,7 @@ from typing import Callable, Generic, Optional, Tuple, TypeVar -from ..common import REDUCING_PREFIX, validate_state_name +from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -35,9 +35,9 @@ def __init__(self, store: KvStore, name: str, value_codec: Codec[V], reduce_func self._reduce_func = reduce_func state_name = name.strip() self._ck = ComplexKey( - key_group=REDUCING_PREFIX, - key=state_name.encode("utf-8"), - namespace=b"data", + key_group=b"", + key=b"", + namespace=b"", user_key=b"", ) diff --git a/python/functionstream-api/src/fs_api/store/structures/value_state.py b/python/functionstream-api/src/fs_api/store/structures/value_state.py index ad8dd4ec..ff3a51ac 100644 --- a/python/functionstream-api/src/fs_api/store/structures/value_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/value_state.py @@ -12,7 +12,7 @@ from typing import Generic, Optional, Tuple, TypeVar -from ..common import VALUE_PREFIX, validate_state_name +from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -32,9 +32,9 @@ def __init__(self, store: KvStore, name: str, codec: Codec[T]): self._codec = codec state_name = name.strip() self._ck = ComplexKey( - key_group=VALUE_PREFIX, - key=state_name.encode("utf-8"), - namespace=b"data", + key_group=b"", + key=b"", + namespace=b"", user_key=b"", ) From 8d60853ffe1dc8c3c98c812f8528f41efae9dfc7 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Mon, 9 Mar 2026 23:23:18 +0800 Subject: [PATCH 3/9] update --- go-sdk/impl/state.go | 12 + go-sdk/state/codec/bool_codec.go | 12 + go-sdk/state/codec/bytes_codec.go | 12 + go-sdk/state/codec/default_codec.go | 12 + go-sdk/state/codec/float32_codec.go | 12 + go-sdk/state/codec/float64_codec.go | 12 + go-sdk/state/codec/int32_codec.go | 12 + go-sdk/state/codec/int64_codec.go | 12 + go-sdk/state/codec/interface.go | 12 + go-sdk/state/codec/json_codec.go | 12 + go-sdk/state/codec/ordered_float32_codec.go | 12 + go-sdk/state/codec/ordered_float64_codec.go | 12 + go-sdk/state/codec/ordered_int32_codec.go | 12 + go-sdk/state/codec/ordered_int64_codec.go | 12 + go-sdk/state/codec/ordered_int_codec.go | 12 + go-sdk/state/codec/ordered_uint32_codec.go | 12 + go-sdk/state/codec/ordered_uint64_codec.go | 12 + go-sdk/state/codec/ordered_uint_codec.go | 12 + go-sdk/state/codec/string_codec.go | 12 + go-sdk/state/codec/uint32_codec.go | 12 + go-sdk/state/codec/uint64_codec.go | 12 + go-sdk/state/common/common.go | 12 + go-sdk/state/keyed/keyed_aggregating_state.go | 23 ++ go-sdk/state/keyed/keyed_list_state.go | 26 +- go-sdk/state/keyed/keyed_map_state.go | 17 +- .../state/keyed/keyed_priority_queue_state.go | 19 +- go-sdk/state/keyed/keyed_reducing_state.go | 17 +- go-sdk/state/keyed/keyed_state_factory.go | 14 +- go-sdk/state/keyed/keyed_value_state.go | 22 +- go-sdk/state/structures/aggregating.go | 12 + go-sdk/state/structures/list.go | 12 + go-sdk/state/structures/map.go | 12 + go-sdk/state/structures/priority_queue.go | 16 +- go-sdk/state/structures/reducing.go | 12 + go-sdk/state/structures/value.go | 12 + .../functionstream-api/src/fs_api/context.py | 141 ++++++++-- .../src/fs_api/store/__init__.py | 16 ++ .../src/fs_api/store/codec/__init__.py | 260 ++---------------- .../src/fs_api/store/codec/base.py | 25 ++ .../src/fs_api/store/codec/bool_codec.py | 29 ++ .../src/fs_api/store/codec/bytes_codec.py | 23 ++ .../src/fs_api/store/codec/default_codec.py | 51 ++++ .../src/fs_api/store/codec/float32_codec.py | 25 ++ .../src/fs_api/store/codec/float64_codec.py | 25 ++ .../src/fs_api/store/codec/int32_codec.py | 25 ++ .../src/fs_api/store/codec/int64_codec.py | 25 ++ .../src/fs_api/store/codec/json_codec.py | 26 ++ .../store/codec/ordered_float32_codec.py | 37 +++ .../store/codec/ordered_float64_codec.py | 37 +++ .../fs_api/store/codec/ordered_int32_codec.py | 32 +++ .../fs_api/store/codec/ordered_int64_codec.py | 32 +++ .../store/codec/ordered_uint32_codec.py | 29 ++ .../store/codec/ordered_uint64_codec.py | 29 ++ .../src/fs_api/store/codec/pickle_codec.py | 27 ++ .../src/fs_api/store/codec/string_codec.py | 23 ++ .../src/fs_api/store/codec/uint32_codec.py | 27 ++ .../src/fs_api/store/codec/uint64_codec.py | 27 ++ .../src/fs_api/store/common/__init__.py | 52 ++-- .../src/fs_api/store/keyed/__init__.py | 18 +- .../store/keyed/keyed_aggregating_state.py | 54 +++- .../fs_api/store/keyed/keyed_list_state.py | 58 +++- .../src/fs_api/store/keyed/keyed_map_state.py | 80 ++++-- .../store/keyed/keyed_priority_queue_state.py | 70 ++++- .../store/keyed/keyed_reducing_state.py | 54 +++- .../fs_api/store/keyed/keyed_state_factory.py | 41 ++- .../fs_api/store/keyed/keyed_value_state.py | 57 +++- .../store/structures/aggregating_state.py | 4 - .../src/fs_api/store/structures/list_state.py | 5 +- .../src/fs_api/store/structures/map_state.py | 10 +- .../store/structures/priority_queue_state.py | 8 +- .../fs_api/store/structures/reducing_state.py | 5 +- .../fs_api/store/structures/value_state.py | 5 +- .../src/fs_runtime/store/fs_context.py | 158 +++++++++-- 73 files changed, 1706 insertions(+), 422 deletions(-) create mode 100644 python/functionstream-api/src/fs_api/store/codec/base.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/bool_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/bytes_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/default_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/float32_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/float64_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/int32_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/int64_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/json_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/pickle_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/string_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/uint32_codec.py create mode 100644 python/functionstream-api/src/fs_api/store/codec/uint64_codec.py diff --git a/go-sdk/impl/state.go b/go-sdk/impl/state.go index d01d0922..2befbedb 100644 --- a/go-sdk/impl/state.go +++ b/go-sdk/impl/state.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package impl import ( diff --git a/go-sdk/state/codec/bool_codec.go b/go-sdk/state/codec/bool_codec.go index 6be75688..d18fbe63 100644 --- a/go-sdk/state/codec/bool_codec.go +++ b/go-sdk/state/codec/bool_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import "fmt" diff --git a/go-sdk/state/codec/bytes_codec.go b/go-sdk/state/codec/bytes_codec.go index d276fa69..b428d506 100644 --- a/go-sdk/state/codec/bytes_codec.go +++ b/go-sdk/state/codec/bytes_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import "github.com/functionstream/function-stream/go-sdk/state/common" diff --git a/go-sdk/state/codec/default_codec.go b/go-sdk/state/codec/default_codec.go index c8dc85d8..7a18321b 100644 --- a/go-sdk/state/codec/default_codec.go +++ b/go-sdk/state/codec/default_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/float32_codec.go b/go-sdk/state/codec/float32_codec.go index 719cdcaa..8967596a 100644 --- a/go-sdk/state/codec/float32_codec.go +++ b/go-sdk/state/codec/float32_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/float64_codec.go b/go-sdk/state/codec/float64_codec.go index cff6b2c0..1e5d484c 100644 --- a/go-sdk/state/codec/float64_codec.go +++ b/go-sdk/state/codec/float64_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/int32_codec.go b/go-sdk/state/codec/int32_codec.go index 25bbf245..40fe0507 100644 --- a/go-sdk/state/codec/int32_codec.go +++ b/go-sdk/state/codec/int32_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/int64_codec.go b/go-sdk/state/codec/int64_codec.go index 8db9a7a4..60046e9a 100644 --- a/go-sdk/state/codec/int64_codec.go +++ b/go-sdk/state/codec/int64_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/interface.go b/go-sdk/state/codec/interface.go index b5a958b1..2e35b4f7 100644 --- a/go-sdk/state/codec/interface.go +++ b/go-sdk/state/codec/interface.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec // Codec is the core encoding interface. EncodedSize reports encoded byte length: diff --git a/go-sdk/state/codec/json_codec.go b/go-sdk/state/codec/json_codec.go index 9b45bb87..abf65b8d 100644 --- a/go-sdk/state/codec/json_codec.go +++ b/go-sdk/state/codec/json_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import "encoding/json" diff --git a/go-sdk/state/codec/ordered_float32_codec.go b/go-sdk/state/codec/ordered_float32_codec.go index a8860613..ea9a3c68 100644 --- a/go-sdk/state/codec/ordered_float32_codec.go +++ b/go-sdk/state/codec/ordered_float32_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/ordered_float64_codec.go b/go-sdk/state/codec/ordered_float64_codec.go index f481c7d4..d032e2f0 100644 --- a/go-sdk/state/codec/ordered_float64_codec.go +++ b/go-sdk/state/codec/ordered_float64_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/ordered_int32_codec.go b/go-sdk/state/codec/ordered_int32_codec.go index 258f8bd4..1b48e92d 100644 --- a/go-sdk/state/codec/ordered_int32_codec.go +++ b/go-sdk/state/codec/ordered_int32_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/ordered_int64_codec.go b/go-sdk/state/codec/ordered_int64_codec.go index e228dbfc..dc8b5346 100644 --- a/go-sdk/state/codec/ordered_int64_codec.go +++ b/go-sdk/state/codec/ordered_int64_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import "github.com/functionstream/function-stream/go-sdk/state/common" diff --git a/go-sdk/state/codec/ordered_int_codec.go b/go-sdk/state/codec/ordered_int_codec.go index 35899ee7..ebdc9e90 100644 --- a/go-sdk/state/codec/ordered_int_codec.go +++ b/go-sdk/state/codec/ordered_int_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import "strconv" diff --git a/go-sdk/state/codec/ordered_uint32_codec.go b/go-sdk/state/codec/ordered_uint32_codec.go index f115e039..a6a5ce1e 100644 --- a/go-sdk/state/codec/ordered_uint32_codec.go +++ b/go-sdk/state/codec/ordered_uint32_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/ordered_uint64_codec.go b/go-sdk/state/codec/ordered_uint64_codec.go index f3fda43d..239b21ff 100644 --- a/go-sdk/state/codec/ordered_uint64_codec.go +++ b/go-sdk/state/codec/ordered_uint64_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/ordered_uint_codec.go b/go-sdk/state/codec/ordered_uint_codec.go index 507ccb68..6642d8d1 100644 --- a/go-sdk/state/codec/ordered_uint_codec.go +++ b/go-sdk/state/codec/ordered_uint_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import "strconv" diff --git a/go-sdk/state/codec/string_codec.go b/go-sdk/state/codec/string_codec.go index d530e668..0f4f2e21 100644 --- a/go-sdk/state/codec/string_codec.go +++ b/go-sdk/state/codec/string_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec type StringCodec struct{} diff --git a/go-sdk/state/codec/uint32_codec.go b/go-sdk/state/codec/uint32_codec.go index 5e6d3aaa..958b9609 100644 --- a/go-sdk/state/codec/uint32_codec.go +++ b/go-sdk/state/codec/uint32_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/codec/uint64_codec.go b/go-sdk/state/codec/uint64_codec.go index c7aed98a..f4001e24 100644 --- a/go-sdk/state/codec/uint64_codec.go +++ b/go-sdk/state/codec/uint64_codec.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codec import ( diff --git a/go-sdk/state/common/common.go b/go-sdk/state/common/common.go index 2551e884..a31c2a37 100644 --- a/go-sdk/state/common/common.go +++ b/go-sdk/state/common/common.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package common import ( diff --git a/go-sdk/state/keyed/keyed_aggregating_state.go b/go-sdk/state/keyed/keyed_aggregating_state.go index 930840bc..4787889c 100644 --- a/go-sdk/state/keyed/keyed_aggregating_state.go +++ b/go-sdk/state/keyed/keyed_aggregating_state.go @@ -1,7 +1,20 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package keyed import ( "fmt" + "github.com/functionstream/function-stream/go-sdk/api" "github.com/functionstream/function-stream/go-sdk/state/codec" "github.com/functionstream/function-stream/go-sdk/state/common" @@ -33,6 +46,16 @@ func NewKeyedAggregatingStateFactory[T any, ACC any, R any]( return nil, err } + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed aggregating state factory key_group must not be nil") + } + if accCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed aggregating state factory acc_codec must not be nil") + } + if aggFunc == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed aggregating state factory agg_func must not be nil") + } + return &KeyedAggregatingStateFactory[T, ACC, R]{ inner: inner, groupKey: common.DupBytes(keyGroup), diff --git a/go-sdk/state/keyed/keyed_list_state.go b/go-sdk/state/keyed/keyed_list_state.go index b52dc6d4..4c02a0b8 100644 --- a/go-sdk/state/keyed/keyed_list_state.go +++ b/go-sdk/state/keyed/keyed_list_state.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package keyed import ( @@ -102,13 +114,13 @@ func (s *KeyedListState[V]) Add(value V) error { } func (s *KeyedListState[V]) AddAll(values []V) error { - payload, err := s.serializeBatch(values) - if err != nil { - return err - } - if err := s.factory.inner.store.Merge(s.complexKey, payload); err != nil { - return err - } + payload, err := s.serializeBatch(values) + if err != nil { + return err + } + if err := s.factory.inner.store.Merge(s.complexKey, payload); err != nil { + return err + } return nil } diff --git a/go-sdk/state/keyed/keyed_map_state.go b/go-sdk/state/keyed/keyed_map_state.go index ffc6e1c4..8fc7e09b 100644 --- a/go-sdk/state/keyed/keyed_map_state.go +++ b/go-sdk/state/keyed/keyed_map_state.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package keyed import ( @@ -28,8 +40,11 @@ func NewKeyedMapStateFactory[MK any, MV any]( return nil, err } + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed map state factory key_group must not be nil") + } if mapKeyCodec == nil || mapValueCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "codec must not be nil") + return nil, api.NewError(api.ErrStoreInternal, "keyed map state factory map_key_codec and map_value_codec must not be nil") } if !mapKeyCodec.IsOrderedKeyCodec() { diff --git a/go-sdk/state/keyed/keyed_priority_queue_state.go b/go-sdk/state/keyed/keyed_priority_queue_state.go index 94790bdb..0e524664 100644 --- a/go-sdk/state/keyed/keyed_priority_queue_state.go +++ b/go-sdk/state/keyed/keyed_priority_queue_state.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package keyed import ( @@ -26,8 +38,11 @@ func NewKeyedPriorityQueueStateFactory[V any]( return nil, err } + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed priority queue state factory key_group must not be nil") + } if valueCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "value codec must not be nil") + return nil, api.NewError(api.ErrStoreInternal, "keyed priority queue state factory value codec must not be nil") } if !valueCodec.IsOrderedKeyCodec() { @@ -159,4 +174,4 @@ func (s *KeyedPriorityQueueState[V]) All() iter.Seq[V] { } } } -} \ No newline at end of file +} diff --git a/go-sdk/state/keyed/keyed_reducing_state.go b/go-sdk/state/keyed/keyed_reducing_state.go index 3911ec32..e608ee3d 100644 --- a/go-sdk/state/keyed/keyed_reducing_state.go +++ b/go-sdk/state/keyed/keyed_reducing_state.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package keyed import ( @@ -29,8 +41,11 @@ func NewKeyedReducingStateFactory[V any]( return nil, err } + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed reducing state factory key_group must not be nil") + } if valueCodec == nil || reduceFunc == nil { - return nil, api.NewError(api.ErrStoreInternal, "value codec and reduce function are required") + return nil, api.NewError(api.ErrStoreInternal, "keyed reducing state factory value_codec and reduce_func must not be nil") } return &KeyedReducingStateFactory[V]{ diff --git a/go-sdk/state/keyed/keyed_state_factory.go b/go-sdk/state/keyed/keyed_state_factory.go index 914e2418..acc601ec 100644 --- a/go-sdk/state/keyed/keyed_state_factory.go +++ b/go-sdk/state/keyed/keyed_state_factory.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package keyed import ( @@ -25,8 +37,6 @@ func newKeyedStateFactory(store common.Store, name string, kind string) (*keyedS return &keyedStateFactory{store: store, name: stateName}, nil } - - func keyedSubStateName[K any](base string, keyCodec codec.Codec[K], key K, kind string) (string, error) { if keyCodec == nil { return "", api.NewError(api.ErrStoreInternal, "key codec must not be nil") diff --git a/go-sdk/state/keyed/keyed_value_state.go b/go-sdk/state/keyed/keyed_value_state.go index 5e6c2187..c181e992 100644 --- a/go-sdk/state/keyed/keyed_value_state.go +++ b/go-sdk/state/keyed/keyed_value_state.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package keyed import ( @@ -7,6 +19,7 @@ import ( "github.com/functionstream/function-stream/go-sdk/state/codec" "github.com/functionstream/function-stream/go-sdk/state/common" ) + type KeyedValueStateFactory[V any] struct { inner *keyedStateFactory groupKey []byte @@ -24,8 +37,11 @@ func NewKeyedValueStateFactory[V any]( return nil, err } + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed value state factory key_group must not be nil") + } if valueCodec == nil { - return nil, api.NewError(api.ErrStoreInternal, "value codec must not be nil") + return nil, api.NewError(api.ErrStoreInternal, "keyed value state factory value codec must not be nil") } return &KeyedValueStateFactory[V]{ @@ -57,7 +73,7 @@ func (s *KeyedValueState[V]) buildCK() api.ComplexKey { KeyGroup: s.factory.groupKey, Key: s.primaryKey, Namespace: s.namespace, - UserKey: []byte{}, + UserKey: []byte{}, } } @@ -86,4 +102,4 @@ func (s *KeyedValueState[V]) Value() (V, bool, error) { func (s *KeyedValueState[V]) Clear() error { return s.factory.inner.store.Delete(s.buildCK()) -} \ No newline at end of file +} diff --git a/go-sdk/state/structures/aggregating.go b/go-sdk/state/structures/aggregating.go index 24787e1d..ab6f3270 100644 --- a/go-sdk/state/structures/aggregating.go +++ b/go-sdk/state/structures/aggregating.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package structures import ( diff --git a/go-sdk/state/structures/list.go b/go-sdk/state/structures/list.go index 7d48c2df..83d2c845 100644 --- a/go-sdk/state/structures/list.go +++ b/go-sdk/state/structures/list.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package structures import ( diff --git a/go-sdk/state/structures/map.go b/go-sdk/state/structures/map.go index fb9fa7e2..a4b89352 100644 --- a/go-sdk/state/structures/map.go +++ b/go-sdk/state/structures/map.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package structures import ( diff --git a/go-sdk/state/structures/priority_queue.go b/go-sdk/state/structures/priority_queue.go index d6fbb662..16518893 100644 --- a/go-sdk/state/structures/priority_queue.go +++ b/go-sdk/state/structures/priority_queue.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package structures import ( @@ -9,6 +21,7 @@ import ( "github.com/functionstream/function-stream/go-sdk/state/common" ) +// PriorityQueueState holds a priority queue. itemCodec must be ordered (IsOrderedKeyCodec() true). type PriorityQueueState[T any] struct { store common.Store keyGroup []byte @@ -17,6 +30,7 @@ type PriorityQueueState[T any] struct { valueCodec codec.Codec[T] } +// NewPriorityQueueState creates a priority queue state. itemCodec must support ordered key encoding. func NewPriorityQueueState[T any](store common.Store, itemCodec codec.Codec[T]) (*PriorityQueueState[T], error) { if store == nil { return nil, api.NewError(api.ErrStoreInternal, "priority queue state store must not be nil") @@ -25,7 +39,7 @@ func NewPriorityQueueState[T any](store common.Store, itemCodec codec.Codec[T]) return nil, api.NewError(api.ErrStoreInternal, "priority queue state codec must not be nil") } if !itemCodec.IsOrderedKeyCodec() { - return nil, api.NewError(api.ErrStoreInternal, "priority queue value codec must be ordered") + return nil, api.NewError(api.ErrStoreInternal, "priority queue codec must support ordered key encoding") } return &PriorityQueueState[T]{ store: store, diff --git a/go-sdk/state/structures/reducing.go b/go-sdk/state/structures/reducing.go index 8af284fd..19ca5b9c 100644 --- a/go-sdk/state/structures/reducing.go +++ b/go-sdk/state/structures/reducing.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package structures import ( diff --git a/go-sdk/state/structures/value.go b/go-sdk/state/structures/value.go index 8f4e549e..b77ab4f9 100644 --- a/go-sdk/state/structures/value.go +++ b/go-sdk/state/structures/value.go @@ -1,3 +1,15 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package structures import ( diff --git a/python/functionstream-api/src/fs_api/context.py b/python/functionstream-api/src/fs_api/context.py index c9578cbe..5c053b17 100644 --- a/python/functionstream-api/src/fs_api/context.py +++ b/python/functionstream-api/src/fs_api/context.py @@ -10,13 +10,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -fs_api.context - -Context: Context object -""" import abc -from typing import Dict +from typing import Dict, Optional, Type + from .store import ( Codec, KvStore, @@ -24,17 +20,18 @@ MapState, ListState, PriorityQueueState, - KeyedStateFactory, - KeyedValueState, - KeyedMapState, - KeyedListState, - KeyedPriorityQueueState, + AggregatingState, + ReducingState, + KeyedListStateFactory, + KeyedValueStateFactory, + KeyedMapStateFactory, + KeyedPriorityQueueStateFactory, + KeyedAggregatingStateFactory, + KeyedReducingStateFactory, ) class Context(abc.ABC): - """Context object""" - @abc.abstractmethod def emit(self, data: bytes, channel: int = 0): pass @@ -49,47 +46,135 @@ def getOrCreateKVStore(self, name: str) -> KvStore: @abc.abstractmethod def getConfig(self) -> Dict[str, str]: - """ - Get global configuration Map + pass - Returns: - Dict[str, str]: Configuration dictionary - """ + @abc.abstractmethod + def getOrCreateValueState(self, store_name: str, codec: Codec) -> ValueState: + pass @abc.abstractmethod - def getOrCreateValueState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> ValueState: + def getOrCreateValueStateAutoCodec(self, store_name: str) -> ValueState: pass @abc.abstractmethod - def getOrCreateMapState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> MapState: + def getOrCreateMapState(self, store_name: str, key_codec: Codec, value_codec: Codec) -> MapState: pass @abc.abstractmethod - def getOrCreateListState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> ListState: + def getOrCreateMapStateAutoKeyCodec(self, store_name: str, value_codec: Codec) -> MapState: pass @abc.abstractmethod - def getOrCreatePriorityQueueState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> PriorityQueueState: + def getOrCreateListState(self, store_name: str, codec: Codec) -> ListState: pass @abc.abstractmethod - def getOrCreateKeyedStateFactory(self, state_name: str, store_name: str = "__fssdk_structured_state__") -> KeyedStateFactory: + def getOrCreateListStateAutoCodec(self, store_name: str) -> ListState: pass @abc.abstractmethod - def getOrCreateKeyedValueState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedValueState: + def getOrCreatePriorityQueueState(self, store_name: str, codec: Codec) -> PriorityQueueState: pass @abc.abstractmethod - def getOrCreateKeyedMapState(self, state_name: str, key_codec: Codec, map_key_codec: Codec, map_value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedMapState: + def getOrCreatePriorityQueueStateAutoCodec(self, store_name: str) -> PriorityQueueState: pass @abc.abstractmethod - def getOrCreateKeyedListState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedListState: + def getOrCreateAggregatingState( + self, store_name: str, acc_codec: Codec, agg_func: object + ) -> AggregatingState: pass @abc.abstractmethod - def getOrCreateKeyedPriorityQueueState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedPriorityQueueState: + def getOrCreateAggregatingStateAutoCodec( + self, store_name: str, agg_func: object + ) -> AggregatingState: pass -__all__ = ['Context'] + @abc.abstractmethod + def getOrCreateReducingState( + self, store_name: str, value_codec: Codec, reduce_func: object + ) -> ReducingState: + pass + + @abc.abstractmethod + def getOrCreateReducingStateAutoCodec( + self, store_name: str, reduce_func: object + ) -> ReducingState: + pass + + @abc.abstractmethod + def getOrCreateKeyedListStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedListStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedListStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_type: Optional[Type] = None + ) -> KeyedListStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedValueStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedValueStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedValueStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_type: Optional[Type] = None + ) -> KeyedValueStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedMapStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, key_codec: Codec, value_codec: Codec + ) -> KeyedMapStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedMapStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedMapStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedPriorityQueueStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, item_codec: Codec + ) -> KeyedPriorityQueueStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedPriorityQueueStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, item_type: Optional[Type] = None + ) -> KeyedPriorityQueueStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedAggregatingStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, acc_codec: Codec, agg_func: object + ) -> KeyedAggregatingStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedAggregatingStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, agg_func: object, acc_type: Optional[Type] = None + ) -> KeyedAggregatingStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedReducingStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec, reduce_func: object + ) -> KeyedReducingStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedReducingStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, reduce_func: object, value_type: Optional[Type] = None + ) -> KeyedReducingStateFactory: + pass + + +__all__ = ["Context"] diff --git a/python/functionstream-api/src/fs_api/store/__init__.py b/python/functionstream-api/src/fs_api/store/__init__.py index 454c99a2..1cf1eb8a 100644 --- a/python/functionstream-api/src/fs_api/store/__init__.py +++ b/python/functionstream-api/src/fs_api/store/__init__.py @@ -14,6 +14,7 @@ from .complexkey import ComplexKey from .iterator import KvIterator from .store import KvStore +from .common import StateKind from .codec import ( Codec, @@ -34,6 +35,7 @@ OrderedUint32Codec, OrderedFloat64Codec, OrderedFloat32Codec, + default_codec_for, ) from .structures import ( @@ -52,6 +54,12 @@ from .keyed import ( KeyedStateFactory, + KeyedListStateFactory, + KeyedValueStateFactory, + KeyedMapStateFactory, + KeyedPriorityQueueStateFactory, + KeyedAggregatingStateFactory, + KeyedReducingStateFactory, KeyedValueState, KeyedMapState, KeyedListState, @@ -68,6 +76,7 @@ "ComplexKey", "KvIterator", "KvStore", + "StateKind", "Codec", "JsonCodec", "PickleCodec", @@ -86,6 +95,7 @@ "OrderedUint32Codec", "OrderedFloat64Codec", "OrderedFloat32Codec", + "default_codec_for", "ValueState", "MapEntry", "MapState", @@ -98,6 +108,12 @@ "ReduceFunc", "ReducingState", "KeyedStateFactory", + "KeyedListStateFactory", + "KeyedValueStateFactory", + "KeyedMapStateFactory", + "KeyedPriorityQueueStateFactory", + "KeyedAggregatingStateFactory", + "KeyedReducingStateFactory", "KeyedValueState", "KeyedMapState", "KeyedListState", diff --git a/python/functionstream-api/src/fs_api/store/codec/__init__.py b/python/functionstream-api/src/fs_api/store/codec/__init__.py index 376cf8c7..1438430f 100644 --- a/python/functionstream-api/src/fs_api/store/codec/__init__.py +++ b/python/functionstream-api/src/fs_api/store/codec/__init__.py @@ -10,246 +10,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import struct -from typing import Generic, TypeVar - -import cloudpickle - -T = TypeVar("T") - - -class Codec(Generic[T]): - supports_ordered_keys: bool = False - - def encode(self, value: T) -> bytes: - raise NotImplementedError - - def decode(self, data: bytes) -> T: - raise NotImplementedError - - -class JsonCodec(Codec[T]): - def encode(self, value: T) -> bytes: - return json.dumps(value).encode("utf-8") - - def decode(self, data: bytes) -> T: - return json.loads(data.decode("utf-8")) - - -class PickleCodec(Codec[T]): - def encode(self, value: T) -> bytes: - return cloudpickle.dumps(value) - - def decode(self, data: bytes) -> T: - return cloudpickle.loads(data) - - -class BytesCodec(Codec[bytes]): - supports_ordered_keys = True - - def encode(self, value: bytes) -> bytes: - return bytes(value) - - def decode(self, data: bytes) -> bytes: - return bytes(data) - - -class StringCodec(Codec[str]): - supports_ordered_keys = True - - def encode(self, value: str) -> bytes: - return value.encode("utf-8") - - def decode(self, data: bytes) -> str: - return data.decode("utf-8") - - -class BoolCodec(Codec[bool]): - supports_ordered_keys = True - - def encode(self, value: bool) -> bytes: - return b"\x01" if value else b"\x00" - - def decode(self, data: bytes) -> bool: - if len(data) != 1: - raise ValueError(f"invalid bool payload length: {len(data)}") - if data == b"\x00": - return False - if data == b"\x01": - return True - raise ValueError(f"invalid bool payload byte: {data[0]}") - - -class Int64Codec(Codec[int]): - def encode(self, value: int) -> bytes: - return struct.pack(">q", value) - - def decode(self, data: bytes) -> int: - if len(data) != 8: - raise ValueError(f"invalid int64 payload length: {len(data)}") - return struct.unpack(">q", data)[0] - - -class Uint64Codec(Codec[int]): - def encode(self, value: int) -> bytes: - if value < 0: - raise ValueError("uint64 value must be >= 0") - return struct.pack(">Q", value) - - def decode(self, data: bytes) -> int: - if len(data) != 8: - raise ValueError(f"invalid uint64 payload length: {len(data)}") - return struct.unpack(">Q", data)[0] - - -class Int32Codec(Codec[int]): - def encode(self, value: int) -> bytes: - return struct.pack(">i", value) - - def decode(self, data: bytes) -> int: - if len(data) != 4: - raise ValueError(f"invalid int32 payload length: {len(data)}") - return struct.unpack(">i", data)[0] - - -class Uint32Codec(Codec[int]): - def encode(self, value: int) -> bytes: - if value < 0: - raise ValueError("uint32 value must be >= 0") - return struct.pack(">I", value) - - def decode(self, data: bytes) -> int: - if len(data) != 4: - raise ValueError(f"invalid uint32 payload length: {len(data)}") - return struct.unpack(">I", data)[0] - - -class Float64Codec(Codec[float]): - def encode(self, value: float) -> bytes: - return struct.pack(">d", value) - - def decode(self, data: bytes) -> float: - if len(data) != 8: - raise ValueError(f"invalid float64 payload length: {len(data)}") - return struct.unpack(">d", data)[0] - - -class Float32Codec(Codec[float]): - def encode(self, value: float) -> bytes: - return struct.pack(">f", value) - - def decode(self, data: bytes) -> float: - if len(data) != 4: - raise ValueError(f"invalid float32 payload length: {len(data)}") - return struct.unpack(">f", data)[0] - - -class OrderedInt64Codec(Codec[int]): - supports_ordered_keys = True - - def encode(self, value: int) -> bytes: - mapped = (value & 0xFFFFFFFFFFFFFFFF) ^ (1 << 63) - return struct.pack(">Q", mapped) - - def decode(self, data: bytes) -> int: - if len(data) != 8: - raise ValueError(f"invalid ordered int64 payload length: {len(data)}") - mapped = struct.unpack(">Q", data)[0] - raw = mapped ^ (1 << 63) - if raw >= (1 << 63): - return raw - (1 << 64) - return raw - - -class OrderedUint64Codec(Codec[int]): - supports_ordered_keys = True - - def encode(self, value: int) -> bytes: - if value < 0: - raise ValueError("ordered uint64 value must be >= 0") - return struct.pack(">Q", value) - - def decode(self, data: bytes) -> int: - if len(data) != 8: - raise ValueError(f"invalid ordered uint64 payload length: {len(data)}") - return struct.unpack(">Q", data)[0] - - -class OrderedInt32Codec(Codec[int]): - supports_ordered_keys = True - - def encode(self, value: int) -> bytes: - mapped = (value & 0xFFFFFFFF) ^ (1 << 31) - return struct.pack(">I", mapped) - - def decode(self, data: bytes) -> int: - if len(data) != 4: - raise ValueError(f"invalid ordered int32 payload length: {len(data)}") - mapped = struct.unpack(">I", data)[0] - raw = mapped ^ (1 << 31) - if raw >= (1 << 31): - return raw - (1 << 32) - return raw - - -class OrderedUint32Codec(Codec[int]): - supports_ordered_keys = True - - def encode(self, value: int) -> bytes: - if value < 0: - raise ValueError("ordered uint32 value must be >= 0") - return struct.pack(">I", value) - - def decode(self, data: bytes) -> int: - if len(data) != 4: - raise ValueError(f"invalid ordered uint32 payload length: {len(data)}") - return struct.unpack(">I", data)[0] - - -class OrderedFloat64Codec(Codec[float]): - supports_ordered_keys = True - - def encode(self, value: float) -> bytes: - bits = struct.unpack(">Q", struct.pack(">d", value))[0] - if bits & (1 << 63): - mapped = (~bits) & 0xFFFFFFFFFFFFFFFF - else: - mapped = bits ^ (1 << 63) - return struct.pack(">Q", mapped) - - def decode(self, data: bytes) -> float: - if len(data) != 8: - raise ValueError(f"invalid ordered float64 payload length: {len(data)}") - mapped = struct.unpack(">Q", data)[0] - if mapped & (1 << 63): - bits = mapped ^ (1 << 63) - else: - bits = (~mapped) & 0xFFFFFFFFFFFFFFFF - return struct.unpack(">d", struct.pack(">Q", bits))[0] - - -class OrderedFloat32Codec(Codec[float]): - supports_ordered_keys = True - - def encode(self, value: float) -> bytes: - bits = struct.unpack(">I", struct.pack(">f", value))[0] - if bits & (1 << 31): - mapped = (~bits) & 0xFFFFFFFF - else: - mapped = bits ^ (1 << 31) - return struct.pack(">I", mapped) - - def decode(self, data: bytes) -> float: - if len(data) != 4: - raise ValueError(f"invalid ordered float32 payload length: {len(data)}") - mapped = struct.unpack(">I", data)[0] - if mapped & (1 << 31): - bits = mapped ^ (1 << 31) - else: - bits = (~mapped) & 0xFFFFFFFF - return struct.unpack(">f", struct.pack(">I", bits))[0] - +from .base import Codec +from .json_codec import JsonCodec +from .pickle_codec import PickleCodec +from .bytes_codec import BytesCodec +from .string_codec import StringCodec +from .bool_codec import BoolCodec +from .int64_codec import Int64Codec +from .uint64_codec import Uint64Codec +from .int32_codec import Int32Codec +from .uint32_codec import Uint32Codec +from .float64_codec import Float64Codec +from .float32_codec import Float32Codec +from .ordered_int64_codec import OrderedInt64Codec +from .ordered_uint64_codec import OrderedUint64Codec +from .ordered_int32_codec import OrderedInt32Codec +from .ordered_uint32_codec import OrderedUint32Codec +from .ordered_float64_codec import OrderedFloat64Codec +from .ordered_float32_codec import OrderedFloat32Codec +from .default_codec import default_codec_for __all__ = [ "Codec", @@ -270,4 +49,5 @@ def decode(self, data: bytes) -> float: "OrderedUint32Codec", "OrderedFloat64Codec", "OrderedFloat32Codec", + "default_codec_for", ] diff --git a/python/functionstream-api/src/fs_api/store/codec/base.py b/python/functionstream-api/src/fs_api/store/codec/base.py new file mode 100644 index 00000000..604d87ab --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/base.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, TypeVar + +T = TypeVar("T") + + +class Codec(Generic[T]): + supports_ordered_keys: bool = False + + def encode(self, value: T) -> bytes: + raise NotImplementedError + + def decode(self, data: bytes) -> T: + raise NotImplementedError diff --git a/python/functionstream-api/src/fs_api/store/codec/bool_codec.py b/python/functionstream-api/src/fs_api/store/codec/bool_codec.py new file mode 100644 index 00000000..6f259010 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/bool_codec.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import Codec + + +class BoolCodec(Codec[bool]): + supports_ordered_keys = True + + def encode(self, value: bool) -> bytes: + return b"\x01" if value else b"\x00" + + def decode(self, data: bytes) -> bool: + if len(data) != 1: + raise ValueError(f"invalid bool payload length: {len(data)}") + if data == b"\x00": + return False + if data == b"\x01": + return True + raise ValueError(f"invalid bool payload byte: {data[0]}") diff --git a/python/functionstream-api/src/fs_api/store/codec/bytes_codec.py b/python/functionstream-api/src/fs_api/store/codec/bytes_codec.py new file mode 100644 index 00000000..3ac38b15 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/bytes_codec.py @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import Codec + + +class BytesCodec(Codec[bytes]): + supports_ordered_keys = True + + def encode(self, value: bytes) -> bytes: + return bytes(value) + + def decode(self, data: bytes) -> bytes: + return bytes(data) diff --git a/python/functionstream-api/src/fs_api/store/codec/default_codec.py b/python/functionstream-api/src/fs_api/store/codec/default_codec.py new file mode 100644 index 00000000..edc10d2e --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/default_codec.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Type + +from .base import Codec +from .bool_codec import BoolCodec +from .bytes_codec import BytesCodec +from .json_codec import JsonCodec +from .ordered_float32_codec import OrderedFloat32Codec +from .ordered_float64_codec import OrderedFloat64Codec +from .ordered_int32_codec import OrderedInt32Codec +from .ordered_int64_codec import OrderedInt64Codec +from .ordered_uint32_codec import OrderedUint32Codec +from .ordered_uint64_codec import OrderedUint64Codec +from .pickle_codec import PickleCodec +from .string_codec import StringCodec + + +def default_codec_for(value_type: Type[Any]) -> Codec[Any]: + """ + Return a default Codec for the given type, aligned with Go DefaultCodecFor. + Built-in types use ordered codecs where applicable; list/dict use JsonCodec; else PickleCodec. + """ + if value_type is bool: + return BoolCodec() + if value_type is int: + return OrderedInt64Codec() + if value_type is float: + return OrderedFloat64Codec() + if value_type is str: + return StringCodec() + if value_type is bytes: + return BytesCodec() + try: + if issubclass(value_type, int) and value_type is not bool: + return OrderedInt64Codec() + except TypeError: + pass + if value_type is list or value_type is dict: + return JsonCodec() + return PickleCodec() diff --git a/python/functionstream-api/src/fs_api/store/codec/float32_codec.py b/python/functionstream-api/src/fs_api/store/codec/float32_codec.py new file mode 100644 index 00000000..5dd6150f --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/float32_codec.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Float32Codec(Codec[float]): + def encode(self, value: float) -> bytes: + return struct.pack(">f", value) + + def decode(self, data: bytes) -> float: + if len(data) != 4: + raise ValueError(f"invalid float32 payload length: {len(data)}") + return struct.unpack(">f", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/float64_codec.py b/python/functionstream-api/src/fs_api/store/codec/float64_codec.py new file mode 100644 index 00000000..42879eaf --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/float64_codec.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Float64Codec(Codec[float]): + def encode(self, value: float) -> bytes: + return struct.pack(">d", value) + + def decode(self, data: bytes) -> float: + if len(data) != 8: + raise ValueError(f"invalid float64 payload length: {len(data)}") + return struct.unpack(">d", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/int32_codec.py b/python/functionstream-api/src/fs_api/store/codec/int32_codec.py new file mode 100644 index 00000000..eb850e7e --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/int32_codec.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Int32Codec(Codec[int]): + def encode(self, value: int) -> bytes: + return struct.pack(">i", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid int32 payload length: {len(data)}") + return struct.unpack(">i", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/int64_codec.py b/python/functionstream-api/src/fs_api/store/codec/int64_codec.py new file mode 100644 index 00000000..31208faa --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/int64_codec.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Int64Codec(Codec[int]): + def encode(self, value: int) -> bytes: + return struct.pack(">q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid int64 payload length: {len(data)}") + return struct.unpack(">q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/json_codec.py b/python/functionstream-api/src/fs_api/store/codec/json_codec.py new file mode 100644 index 00000000..b35416d2 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/json_codec.py @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import TypeVar + +from .base import Codec + +T = TypeVar("T") + + +class JsonCodec(Codec[T]): + def encode(self, value: T) -> bytes: + return json.dumps(value).encode("utf-8") + + def decode(self, data: bytes) -> T: + return json.loads(data.decode("utf-8")) diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py new file mode 100644 index 00000000..0dced99a --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedFloat32Codec(Codec[float]): + supports_ordered_keys = True + + def encode(self, value: float) -> bytes: + bits = struct.unpack(">I", struct.pack(">f", value))[0] + if bits & (1 << 31): + mapped = (~bits) & 0xFFFFFFFF + else: + mapped = bits ^ (1 << 31) + return struct.pack(">I", mapped) + + def decode(self, data: bytes) -> float: + if len(data) != 4: + raise ValueError(f"invalid ordered float32 payload length: {len(data)}") + mapped = struct.unpack(">I", data)[0] + if mapped & (1 << 31): + bits = mapped ^ (1 << 31) + else: + bits = (~mapped) & 0xFFFFFFFF + return struct.unpack(">f", struct.pack(">I", bits))[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py new file mode 100644 index 00000000..37e5100a --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedFloat64Codec(Codec[float]): + supports_ordered_keys = True + + def encode(self, value: float) -> bytes: + bits = struct.unpack(">Q", struct.pack(">d", value))[0] + if bits & (1 << 63): + mapped = (~bits) & 0xFFFFFFFFFFFFFFFF + else: + mapped = bits ^ (1 << 63) + return struct.pack(">Q", mapped) + + def decode(self, data: bytes) -> float: + if len(data) != 8: + raise ValueError(f"invalid ordered float64 payload length: {len(data)}") + mapped = struct.unpack(">Q", data)[0] + if mapped & (1 << 63): + bits = mapped ^ (1 << 63) + else: + bits = (~mapped) & 0xFFFFFFFFFFFFFFFF + return struct.unpack(">d", struct.pack(">Q", bits))[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py new file mode 100644 index 00000000..9787bf6c --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedInt32Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + mapped = (value & 0xFFFFFFFF) ^ (1 << 31) + return struct.pack(">I", mapped) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid ordered int32 payload length: {len(data)}") + mapped = struct.unpack(">I", data)[0] + raw = mapped ^ (1 << 31) + if raw >= (1 << 31): + return raw - (1 << 32) + return raw diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py new file mode 100644 index 00000000..dadd9c68 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedInt64Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + mapped = (value & 0xFFFFFFFFFFFFFFFF) ^ (1 << 63) + return struct.pack(">Q", mapped) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid ordered int64 payload length: {len(data)}") + mapped = struct.unpack(">Q", data)[0] + raw = mapped ^ (1 << 63) + if raw >= (1 << 63): + return raw - (1 << 64) + return raw diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py new file mode 100644 index 00000000..9f1ac2e1 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedUint32Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("ordered uint32 value must be >= 0") + return struct.pack(">I", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid ordered uint32 payload length: {len(data)}") + return struct.unpack(">I", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py new file mode 100644 index 00000000..dab4a927 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedUint64Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("ordered uint64 value must be >= 0") + return struct.pack(">Q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid ordered uint64 payload length: {len(data)}") + return struct.unpack(">Q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/pickle_codec.py b/python/functionstream-api/src/fs_api/store/codec/pickle_codec.py new file mode 100644 index 00000000..84066fc2 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/pickle_codec.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TypeVar + +import cloudpickle + +from .base import Codec + +T = TypeVar("T") + + +class PickleCodec(Codec[T]): + def encode(self, value: T) -> bytes: + return cloudpickle.dumps(value) + + def decode(self, data: bytes) -> T: + return cloudpickle.loads(data) diff --git a/python/functionstream-api/src/fs_api/store/codec/string_codec.py b/python/functionstream-api/src/fs_api/store/codec/string_codec.py new file mode 100644 index 00000000..6bb112a4 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/string_codec.py @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import Codec + + +class StringCodec(Codec[str]): + supports_ordered_keys = True + + def encode(self, value: str) -> bytes: + return value.encode("utf-8") + + def decode(self, data: bytes) -> str: + return data.decode("utf-8") diff --git a/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py b/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py new file mode 100644 index 00000000..281531d1 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Uint32Codec(Codec[int]): + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("uint32 value must be >= 0") + return struct.pack(">I", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid uint32 payload length: {len(data)}") + return struct.unpack(">I", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py b/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py new file mode 100644 index 00000000..634d7dcb --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Uint64Codec(Codec[int]): + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("uint64 value must be >= 0") + return struct.pack(">Q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid uint64 payload length: {len(data)}") + return struct.unpack(">Q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/common/__init__.py b/python/functionstream-api/src/fs_api/store/common/__init__.py index 64b03dfb..4056a4e2 100644 --- a/python/functionstream-api/src/fs_api/store/common/__init__.py +++ b/python/functionstream-api/src/fs_api/store/common/__init__.py @@ -11,18 +11,45 @@ # limitations under the License. import struct +from enum import IntEnum from typing import Tuple from ..error import KvError -VALUE_PREFIX = b"__fssdk__/value/" -LIST_PREFIX = b"__fssdk__/list/" -PQ_PREFIX = b"__fssdk__/priority_queue/" -AGGREGATING_PREFIX = b"__fssdk__/aggregating/" -REDUCING_PREFIX = b"__fssdk__/reducing/" -MAP_GROUP = b"__fssdk__/map" -LIST_GROUP = b"__fssdk__/list" -PQ_GROUP = b"__fssdk__/priority_queue" + +class StateKind(IntEnum): + VALUE = 0 + LIST = 1 + PRIORITY_QUEUE = 2 + MAP = 3 + AGGREGATING = 4 + REDUCING = 5 + + def prefix(self) -> bytes: + if self == StateKind.VALUE: + return b"__fssdk__/value/" + if self == StateKind.LIST: + return b"__fssdk__/list/" + if self == StateKind.PRIORITY_QUEUE: + return b"__fssdk__/priority_queue/" + if self == StateKind.MAP: + return b"" + if self == StateKind.AGGREGATING: + return b"__fssdk__/aggregating/" + if self == StateKind.REDUCING: + return b"__fssdk__/reducing/" + return b"" + + def group(self) -> bytes: + if self in (StateKind.VALUE, StateKind.AGGREGATING, StateKind.REDUCING): + return b"" + if self == StateKind.LIST: + return b"__fssdk__/list" + if self == StateKind.PRIORITY_QUEUE: + return b"__fssdk__/priority_queue" + if self == StateKind.MAP: + return b"__fssdk__/map" + return b"" def validate_state_name(name: str) -> None: @@ -53,14 +80,7 @@ def decode_priority_key(data: bytes) -> Tuple[int, int]: __all__ = [ - "VALUE_PREFIX", - "LIST_PREFIX", - "PQ_PREFIX", - "AGGREGATING_PREFIX", - "REDUCING_PREFIX", - "MAP_GROUP", - "LIST_GROUP", - "PQ_GROUP", + "StateKind", "validate_state_name", "encode_int64_lex", "encode_priority_key", diff --git a/python/functionstream-api/src/fs_api/store/keyed/__init__.py b/python/functionstream-api/src/fs_api/store/keyed/__init__.py index 802ca888..f12cdfbb 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/__init__.py +++ b/python/functionstream-api/src/fs_api/store/keyed/__init__.py @@ -11,15 +11,21 @@ # limitations under the License. from .keyed_state_factory import KeyedStateFactory -from .keyed_value_state import KeyedValueState -from .keyed_list_state import KeyedListState -from .keyed_map_state import KeyedMapEntry, KeyedMapState -from .keyed_priority_queue_state import KeyedPriorityQueueState -from .keyed_aggregating_state import AggregateFunc, KeyedAggregatingState -from .keyed_reducing_state import KeyedReducingState, ReduceFunc +from .keyed_value_state import KeyedValueState, KeyedValueStateFactory +from .keyed_list_state import KeyedListState, KeyedListStateFactory +from .keyed_map_state import KeyedMapEntry, KeyedMapState, KeyedMapStateFactory +from .keyed_priority_queue_state import KeyedPriorityQueueState, KeyedPriorityQueueStateFactory +from .keyed_aggregating_state import AggregateFunc, KeyedAggregatingState, KeyedAggregatingStateFactory +from .keyed_reducing_state import KeyedReducingState, KeyedReducingStateFactory, ReduceFunc __all__ = [ "KeyedStateFactory", + "KeyedListStateFactory", + "KeyedValueStateFactory", + "KeyedMapStateFactory", + "KeyedPriorityQueueStateFactory", + "KeyedAggregatingStateFactory", + "KeyedReducingStateFactory", "KeyedValueState", "KeyedListState", "KeyedMapEntry", diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py index 06a8aed0..c0e7173a 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py @@ -14,6 +14,7 @@ from ..codec import Codec from ..complexkey import ComplexKey +from ..error import KvError from ..store import KvStore from ._keyed_common import KEYED_AGGREGATING_GROUP, ensure_ordered_key_codec @@ -31,17 +32,60 @@ def get_result(self, accumulator: ACC) -> R: ... def merge(self, a: ACC, b: ACC) -> ACC: ... +class KeyedAggregatingStateFactory(Generic[T_agg, ACC, R]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + acc_codec: Codec[ACC], + agg_func: "AggregateFunc[T_agg, ACC, R]", + ): + if store is None: + raise KvError("keyed aggregating state factory store must not be None") + if namespace is None: + raise KvError("keyed aggregating state factory namespace must not be None") + if key_group is None: + raise KvError("keyed aggregating state factory key_group must not be None") + if acc_codec is None or agg_func is None: + raise KvError("keyed aggregating state factory acc_codec and agg_func must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._acc_codec = acc_codec + self._agg_func = agg_func + + def new_aggregating(self, key_codec: Codec[K]) -> "KeyedAggregatingState[K, T_agg, ACC, R]": + ensure_ordered_key_codec(key_codec, "keyed aggregating") + return KeyedAggregatingState( + self._store, + self._namespace, + key_codec, + self._acc_codec, + self._agg_func, + self._key_group, + ) + + class KeyedAggregatingState(Generic[K, T_agg, ACC, R]): def __init__( self, store: KvStore, - name: str, + namespace: bytes, key_codec: Codec[K], acc_codec: Codec[ACC], agg_func: AggregateFunc[T_agg, ACC, R], + key_group: bytes, ): + if namespace is None: + raise KvError("keyed aggregating state namespace must not be None") + if key_group is None: + raise KvError("keyed aggregating state key_group must not be None") + if acc_codec is None: + raise KvError("keyed aggregating state acc_codec must not be None") self._store = store - self._name = name.strip() + self._namespace = namespace + self._key_group = key_group self._key_codec = key_codec self._acc_codec = acc_codec self._agg_func = agg_func @@ -49,9 +93,9 @@ def __init__( def _build_ck(self, key: K) -> ComplexKey: return ComplexKey( - key_group=KEYED_AGGREGATING_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=b"", ) @@ -77,4 +121,4 @@ def clear(self, key: K) -> None: self._store.delete(self._build_ck(key)) -__all__ = ["AggregateFunc", "KeyedAggregatingState"] +__all__ = ["AggregateFunc", "KeyedAggregatingState", "KeyedAggregatingStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py index 8635217b..ee49d213 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py @@ -11,7 +11,7 @@ # limitations under the License. import struct -from typing import Generic, List, TypeVar +from typing import Generic, List, Optional, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -24,19 +24,65 @@ V = TypeVar("V") +class KeyedListStateFactory(Generic[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ): + if store is None: + raise KvError("keyed list state factory store must not be None") + if namespace is None: + raise KvError("keyed list state factory namespace must not be None") + if key_group is None: + raise KvError("keyed list state factory key_group must not be None") + if value_codec is None: + raise KvError("keyed list value codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._value_codec = value_codec + + def new_list(self, key_codec: Codec[K]) -> "KeyedListState[K, V]": + ensure_ordered_key_codec(key_codec, "keyed list") + return KeyedListState( + self._store, + self._namespace, + key_codec, + self._value_codec, + self._key_group, + ) + + class KeyedListState(Generic[K, V]): - def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: Codec[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + value_codec: Codec[V], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed list state namespace must not be None") + if key_group is None: + raise KvError("keyed list state key_group must not be None") + if value_codec is None: + raise KvError("keyed list state value_codec must not be None") self._store = store - self._name = name.strip() + self._namespace = namespace + self._key_group = key_group self._key_codec = key_codec self._value_codec = value_codec ensure_ordered_key_codec(key_codec, "keyed list") def _build_ck(self, key: K) -> ComplexKey: return ComplexKey( - key_group=KEYED_LIST_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=b"", ) @@ -91,4 +137,4 @@ def _deserialize(self, raw: bytes) -> List[V]: return out -__all__ = ["KeyedListState"] +__all__ = ["KeyedListState", "KeyedListStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py index 79d01146..0234e002 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py @@ -31,17 +31,61 @@ class KeyedMapEntry(Generic[MK, MV]): value: MV +class KeyedMapStateFactory(Generic[MK, MV]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + map_key_codec: Codec[MK], + map_value_codec: Codec[MV], + ): + if store is None: + raise KvError("keyed map state factory store must not be None") + if namespace is None: + raise KvError("keyed map state factory namespace must not be None") + if key_group is None: + raise KvError("keyed map state factory key_group must not be None") + if map_key_codec is None or map_value_codec is None: + raise KvError("keyed map state factory map_key_codec and map_value_codec must not be None") + ensure_ordered_key_codec(map_key_codec, "keyed map inner") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._map_key_codec = map_key_codec + self._map_value_codec = map_value_codec + + def new_map(self, key_codec: Codec[K]) -> "KeyedMapState[K, MK, MV]": + ensure_ordered_key_codec(key_codec, "keyed map") + return KeyedMapState( + self._store, + self._namespace, + key_codec, + self._map_key_codec, + self._map_value_codec, + self._key_group, + ) + + class KeyedMapState(Generic[K, MK, MV]): def __init__( self, store: KvStore, - name: str, + namespace: bytes, key_codec: Codec[K], map_key_codec: Codec[MK], map_value_codec: Codec[MV], + key_group: bytes, ): + if namespace is None: + raise KvError("keyed map state namespace must not be None") + if key_group is None: + raise KvError("keyed map state key_group must not be None") + if map_key_codec is None or map_value_codec is None: + raise KvError("keyed map state map_key_codec and map_value_codec must not be None") self._store = store - self._name = name.strip() + self._namespace = namespace + self._key_group = key_group self._key_codec = key_codec self._map_key_codec = map_key_codec self._map_value_codec = map_value_codec @@ -50,18 +94,18 @@ def __init__( def put(self, key: K, map_key: MK, value: MV) -> None: ck = ComplexKey( - key_group=KEYED_MAP_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=self._map_key_codec.encode(map_key), ) self._store.put(ck, self._map_value_codec.encode(value)) def get(self, key: K, map_key: MK) -> Optional[MV]: ck = ComplexKey( - key_group=KEYED_MAP_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=self._map_key_codec.encode(map_key), ) raw = self._store.get(ck) @@ -71,27 +115,27 @@ def get(self, key: K, map_key: MK) -> Optional[MV]: def delete(self, key: K, map_key: MK) -> None: ck = ComplexKey( - key_group=KEYED_MAP_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=self._map_key_codec.encode(map_key), ) self._store.delete(ck) def clear(self, key: K) -> None: prefix_ck = ComplexKey( - key_group=KEYED_MAP_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=b"", ) self._store.delete_prefix(prefix_ck) def all(self, key: K): it = self._store.scan_complex( - KEYED_MAP_GROUP, + self._key_group, self._key_codec.encode(key), - self._name.encode("utf-8"), + self._namespace, ) while it.has_next(): item = it.next() @@ -102,9 +146,9 @@ def all(self, key: K): def len(self, key: K) -> int: it = self._store.scan_complex( - KEYED_MAP_GROUP, + self._key_group, self._key_codec.encode(key), - self._name.encode("utf-8"), + self._namespace, ) n = 0 while it.has_next(): @@ -125,15 +169,15 @@ def range( start_bytes = self._map_key_codec.encode(start_inclusive) end_bytes = self._map_key_codec.encode(end_exclusive) user_keys = self._store.list_complex( - KEYED_MAP_GROUP, key_bytes, self._name.encode("utf-8"), + self._key_group, key_bytes, self._namespace, start_bytes, end_bytes, ) out = [] for uk in user_keys: ck = ComplexKey( - key_group=KEYED_MAP_GROUP, + key_group=self._key_group, key=key_bytes, - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=uk, ) raw = self._store.get(ck) @@ -146,4 +190,4 @@ def range( return out -__all__ = ["KeyedMapEntry", "KeyedMapState"] +__all__ = ["KeyedMapEntry", "KeyedMapState", "KeyedMapStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py index 58183f53..1bd3857a 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py @@ -14,6 +14,7 @@ from ..codec import Codec from ..complexkey import ComplexKey +from ..error import KvError from ..store import KvStore from ._keyed_common import KEYED_PQ_GROUP, ensure_ordered_key_codec @@ -22,10 +23,57 @@ V = TypeVar("V") +class KeyedPriorityQueueStateFactory(Generic[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ): + if store is None: + raise KvError("keyed priority queue state factory store must not be None") + if namespace is None: + raise KvError("keyed priority queue state factory namespace must not be None") + if key_group is None: + raise KvError("keyed priority queue state factory key_group must not be None") + if value_codec is None: + raise KvError("keyed priority queue state factory value codec must not be None") + ensure_ordered_key_codec(value_codec, "keyed priority queue value") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._value_codec = value_codec + + def new_priority_queue(self, key_codec: Codec[K]) -> "KeyedPriorityQueueState[K, V]": + ensure_ordered_key_codec(key_codec, "keyed priority queue") + return KeyedPriorityQueueState( + self._store, + self._namespace, + key_codec, + self._value_codec, + self._key_group, + ) + + class KeyedPriorityQueueState(Generic[K, V]): - def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: Codec[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + value_codec: Codec[V], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed priority queue state namespace must not be None") + if key_group is None: + raise KvError("keyed priority queue state key_group must not be None") + if value_codec is None: + raise KvError("keyed priority queue state value_codec must not be None") self._store = store - self._name = name.strip() + self._namespace = namespace + self._key_group = key_group self._key_codec = key_codec self._value_codec = value_codec ensure_ordered_key_codec(key_codec, "keyed priority queue") @@ -33,17 +81,17 @@ def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: def _ck(self, key: K, user_key: bytes) -> ComplexKey: return ComplexKey( - key_group=KEYED_PQ_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=user_key, ) def _prefix_ck(self, key: K) -> ComplexKey: return ComplexKey( - key_group=KEYED_PQ_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=b"", ) @@ -53,9 +101,9 @@ def add(self, key: K, value: V) -> None: def peek(self, key: K) -> Tuple[Optional[V], bool]: it = self._store.scan_complex( - KEYED_PQ_GROUP, + self._key_group, self._key_codec.encode(key), - self._name.encode("utf-8"), + self._namespace, ) if not it.has_next(): return (None, False) @@ -77,9 +125,9 @@ def clear(self, key: K) -> None: def all(self, key: K) -> Iterator[V]: it = self._store.scan_complex( - KEYED_PQ_GROUP, + self._key_group, self._key_codec.encode(key), - self._name.encode("utf-8"), + self._namespace, ) while it.has_next(): item = it.next() @@ -89,4 +137,4 @@ def all(self, key: K) -> Iterator[V]: yield self._value_codec.decode(user_key) -__all__ = ["KeyedPriorityQueueState"] +__all__ = ["KeyedPriorityQueueState", "KeyedPriorityQueueStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py index 0656eb4c..c9c176c0 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py @@ -14,6 +14,7 @@ from ..codec import Codec from ..complexkey import ComplexKey +from ..error import KvError from ..store import KvStore from ._keyed_common import KEYED_REDUCING_GROUP, ensure_ordered_key_codec @@ -24,17 +25,60 @@ ReduceFunc = Callable[[V, V], V] +class KeyedReducingStateFactory(Generic[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + reduce_func: ReduceFunc[V], + ): + if store is None: + raise KvError("keyed reducing state factory store must not be None") + if namespace is None: + raise KvError("keyed reducing state factory namespace must not be None") + if key_group is None: + raise KvError("keyed reducing state factory key_group must not be None") + if value_codec is None or reduce_func is None: + raise KvError("keyed reducing state factory value_codec and reduce_func must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._value_codec = value_codec + self._reduce_func = reduce_func + + def new_reducing(self, key_codec: Codec[K]) -> "KeyedReducingState[K, V]": + ensure_ordered_key_codec(key_codec, "keyed reducing") + return KeyedReducingState( + self._store, + self._namespace, + key_codec, + self._value_codec, + self._reduce_func, + self._key_group, + ) + + class KeyedReducingState(Generic[K, V]): def __init__( self, store: KvStore, - name: str, + namespace: bytes, key_codec: Codec[K], value_codec: Codec[V], reduce_func: ReduceFunc[V], + key_group: bytes, ): + if namespace is None: + raise KvError("keyed reducing state namespace must not be None") + if key_group is None: + raise KvError("keyed reducing state key_group must not be None") + if value_codec is None: + raise KvError("keyed reducing state value_codec must not be None") self._store = store - self._name = name.strip() + self._namespace = namespace + self._key_group = key_group self._key_codec = key_codec self._value_codec = value_codec self._reduce_func = reduce_func @@ -42,9 +86,9 @@ def __init__( def _build_ck(self, key: K) -> ComplexKey: return ComplexKey( - key_group=KEYED_REDUCING_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=b"", ) @@ -67,4 +111,4 @@ def clear(self, key: K) -> None: self._store.delete(self._build_ck(key)) -__all__ = ["ReduceFunc", "KeyedReducingState"] +__all__ = ["ReduceFunc", "KeyedReducingState", "KeyedReducingStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py index c2563bff..ebbfa1b8 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py @@ -33,36 +33,49 @@ class KeyedStateFactory: - def __init__(self, store: KvStore, name: str): + def __init__(self, store: KvStore, namespace: bytes, key_group: bytes): if store is None: raise KvError("keyed state factory store must not be None") - if not isinstance(name, str) or not name.strip(): - raise KvError("keyed state factory name must be non-empty") + if namespace is None: + raise KvError("keyed state factory namespace must not be None") + if key_group is None: + raise KvError("keyed state factory key_group must not be None") self._store = store - self._name = name.strip() + self._namespace = namespace + self._key_group = key_group self._kind = None def new_keyed_value(self, key_codec: Codec[K], value_codec: Codec[V]) -> KeyedValueState[K, V]: self._claim_kind("value") - return KeyedValueState(self._store, self._name, key_codec, value_codec) + if value_codec is None: + raise KvError("keyed value state value_codec must not be None") + return KeyedValueState(self._store, self._namespace, key_codec, value_codec, self._key_group) def new_keyed_map( self, key_codec: Codec[K], map_key_codec: Codec[MK], map_value_codec: Codec[MV] ) -> KeyedMapState[K, MK, MV]: self._claim_kind("map") + if map_key_codec is None or map_value_codec is None: + raise KvError("keyed map state map_key_codec and map_value_codec must not be None") return KeyedMapState( - self._store, self._name, key_codec, map_key_codec, map_value_codec + self._store, self._namespace, key_codec, map_key_codec, map_value_codec, self._key_group ) def new_keyed_list(self, key_codec: Codec[K], value_codec: Codec[V]) -> KeyedListState[K, V]: self._claim_kind("list") - return KeyedListState(self._store, self._name, key_codec, value_codec) + if value_codec is None: + raise KvError("keyed list state value_codec must not be None") + return KeyedListState(self._store, self._namespace, key_codec, value_codec, self._key_group) def new_keyed_priority_queue( self, key_codec: Codec[K], value_codec: Codec[V] ) -> KeyedPriorityQueueState[K, V]: self._claim_kind("priority_queue") - return KeyedPriorityQueueState(self._store, self._name, key_codec, value_codec) + if value_codec is None: + raise KvError("keyed priority queue state value_codec must not be None") + return KeyedPriorityQueueState( + self._store, self._namespace, key_codec, value_codec, self._key_group + ) def new_keyed_aggregating( self, @@ -71,15 +84,21 @@ def new_keyed_aggregating( agg_func: AggregateFunc[T_agg, ACC, R], ) -> KeyedAggregatingState[K, T_agg, ACC, R]: self._claim_kind("aggregating") + if acc_codec is None or agg_func is None: + raise KvError("keyed aggregating state acc_codec and agg_func must not be None") return KeyedAggregatingState( - self._store, self._name, key_codec, acc_codec, agg_func + self._store, self._namespace, key_codec, acc_codec, agg_func, self._key_group ) def new_keyed_reducing( self, key_codec: Codec[K], value_codec: Codec[V], reduce_func: ReduceFunc[V] ) -> KeyedReducingState[K, V]: self._claim_kind("reducing") - return KeyedReducingState(self._store, self._name, key_codec, value_codec, reduce_func) + if value_codec is None or reduce_func is None: + raise KvError("keyed reducing state value_codec and reduce_func must not be None") + return KeyedReducingState( + self._store, self._namespace, key_codec, value_codec, reduce_func, self._key_group + ) def _claim_kind(self, kind: str) -> None: if self._kind is None: @@ -87,7 +106,7 @@ def _claim_kind(self, kind: str) -> None: return if self._kind != kind: raise KvError( - f"keyed state factory '{self._name}' already bound to '{self._kind}', cannot create '{kind}'" + f"keyed state factory already bound to '{self._kind}', cannot create '{kind}'" ) diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py index 0455a739..334c4337 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py @@ -14,6 +14,7 @@ from ..codec import Codec from ..complexkey import ComplexKey +from ..error import KvError from ..store import KvStore from ._keyed_common import KEYED_VALUE_GROUP, ensure_ordered_key_codec @@ -22,19 +23,65 @@ V = TypeVar("V") +class KeyedValueStateFactory(Generic[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ): + if store is None: + raise KvError("keyed value state factory store must not be None") + if namespace is None: + raise KvError("keyed value state factory namespace must not be None") + if key_group is None: + raise KvError("keyed value state factory key_group must not be None") + if value_codec is None: + raise KvError("keyed value state factory value codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._value_codec = value_codec + + def new_value(self, key_codec: Codec[K]) -> "KeyedValueState[K, V]": + ensure_ordered_key_codec(key_codec, "keyed value") + return KeyedValueState( + self._store, + self._namespace, + key_codec, + self._value_codec, + self._key_group, + ) + + class KeyedValueState(Generic[K, V]): - def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: Codec[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + value_codec: Codec[V], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed value state namespace must not be None") + if key_group is None: + raise KvError("keyed value state key_group must not be None") + if value_codec is None: + raise KvError("keyed value state value_codec must not be None") self._store = store - self._name = name.strip() + self._namespace = namespace + self._key_group = key_group self._key_codec = key_codec self._value_codec = value_codec ensure_ordered_key_codec(key_codec, "keyed value") def _build_ck(self, key: K) -> ComplexKey: return ComplexKey( - key_group=KEYED_VALUE_GROUP, + key_group=self._key_group, key=self._key_codec.encode(key), - namespace=self._name.encode("utf-8"), + namespace=self._namespace, user_key=b"", ) @@ -53,4 +100,4 @@ def delete(self, key: K) -> None: self._store.delete(self._build_ck(key)) -__all__ = ["KeyedValueState"] +__all__ = ["KeyedValueState", "KeyedValueStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py index 1473c2d4..52daa5c0 100644 --- a/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py @@ -12,7 +12,6 @@ from typing import Generic, Optional, Protocol, Tuple, TypeVar -from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -41,11 +40,9 @@ class AggregatingState(Generic[T, ACC, R]): def __init__( self, store: KvStore, - name: str, acc_codec: Codec[ACC], agg_func: AggregateFunc[T, ACC, R], ): - validate_state_name(name) if store is None: raise KvError("aggregating state store must not be None") if acc_codec is None: @@ -55,7 +52,6 @@ def __init__( self._store = store self._acc_codec = acc_codec self._agg_func = agg_func - state_name = name.strip() self._ck = ComplexKey( key_group=b"", key=b"", diff --git a/python/functionstream-api/src/fs_api/store/structures/list_state.py b/python/functionstream-api/src/fs_api/store/structures/list_state.py index 0ec1970c..bfa53e13 100644 --- a/python/functionstream-api/src/fs_api/store/structures/list_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/list_state.py @@ -13,7 +13,6 @@ import struct from typing import Generic, List, TypeVar -from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -23,15 +22,13 @@ class ListState(Generic[T]): - def __init__(self, store: KvStore, name: str, codec: Codec[T]): - validate_state_name(name) + def __init__(self, store: KvStore, codec: Codec[T]): if store is None: raise KvError("list state store must not be None") if codec is None: raise KvError("list state codec must not be None") self._store = store self._codec = codec - state_name = name.strip() self._ck = ComplexKey( key_group=b"", key=b"", diff --git a/python/functionstream-api/src/fs_api/store/structures/map_state.py b/python/functionstream-api/src/fs_api/store/structures/map_state.py index 0c19edcb..a1ceb8ee 100644 --- a/python/functionstream-api/src/fs_api/store/structures/map_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/map_state.py @@ -13,7 +13,6 @@ from dataclasses import dataclass from typing import Any, Generator, Generic, List, Optional, Tuple, Type, TypeVar -from ..common import validate_state_name from ..codec import ( BoolCodec, BytesCodec, @@ -37,8 +36,7 @@ class MapEntry(Generic[K, V]): class MapState(Generic[K, V]): - def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: Codec[V]): - validate_state_name(name) + def __init__(self, store: KvStore, key_codec: Codec[K], value_codec: Codec[V]): if store is None: raise KvError("map state store must not be None") if key_codec is None or value_codec is None: @@ -56,12 +54,11 @@ def __init__(self, store: KvStore, name: str, key_codec: Codec[K], value_codec: def with_auto_key_codec( cls, store: KvStore, - name: str, key_type: Type[K], value_codec: Codec[V], ) -> "MapState[K, V]": key_codec = infer_ordered_key_codec(key_type) - return cls(store, name, key_codec, value_codec) + return cls(store, key_codec, value_codec) def put(self, key: K, value: V) -> None: encoded_key = self._key_codec.encode(key) @@ -159,11 +156,10 @@ def infer_ordered_key_codec(key_type: Type[Any]) -> Codec[Any]: def create_map_state_auto_key_codec( store: KvStore, - name: str, key_type: Type[K], value_codec: Codec[V], ) -> MapState[K, V]: - return MapState.with_auto_key_codec(store, name, key_type, value_codec) + return MapState.with_auto_key_codec(store, key_type, value_codec) __all__ = ["MapEntry", "MapState", "infer_ordered_key_codec", "create_map_state_auto_key_codec"] diff --git a/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py index 0818e9fb..6386181d 100644 --- a/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py @@ -12,7 +12,6 @@ from typing import Generic, Iterator, Optional, Tuple, TypeVar -from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -22,14 +21,15 @@ class PriorityQueueState(Generic[T]): - def __init__(self, store: KvStore, name: str, codec: Codec[T]): - validate_state_name(name) + """State for a priority queue. codec must support ordered key encoding (supports_ordered_keys=True).""" + + def __init__(self, store: KvStore, codec: Codec[T]): if store is None: raise KvError("priority queue store must not be None") if codec is None: raise KvError("priority queue codec must not be None") if not getattr(codec, "supports_ordered_keys", False): - raise KvError("priority queue value codec must be ordered") + raise KvError("priority queue codec must support ordered key encoding") self._store = store self._codec = codec self._key_group = b"" diff --git a/python/functionstream-api/src/fs_api/store/structures/reducing_state.py b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py index b8e039a4..7c1b298d 100644 --- a/python/functionstream-api/src/fs_api/store/structures/reducing_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py @@ -12,7 +12,6 @@ from typing import Callable, Generic, Optional, Tuple, TypeVar -from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -24,8 +23,7 @@ class ReducingState(Generic[V]): - def __init__(self, store: KvStore, name: str, value_codec: Codec[V], reduce_func: ReduceFunc[V]): - validate_state_name(name) + def __init__(self, store: KvStore, value_codec: Codec[V], reduce_func: ReduceFunc[V]): if store is None: raise KvError("reducing state store must not be None") if value_codec is None or reduce_func is None: @@ -33,7 +31,6 @@ def __init__(self, store: KvStore, name: str, value_codec: Codec[V], reduce_func self._store = store self._value_codec = value_codec self._reduce_func = reduce_func - state_name = name.strip() self._ck = ComplexKey( key_group=b"", key=b"", diff --git a/python/functionstream-api/src/fs_api/store/structures/value_state.py b/python/functionstream-api/src/fs_api/store/structures/value_state.py index ff3a51ac..1e5a46aa 100644 --- a/python/functionstream-api/src/fs_api/store/structures/value_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/value_state.py @@ -12,7 +12,6 @@ from typing import Generic, Optional, Tuple, TypeVar -from ..common import validate_state_name from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError @@ -22,15 +21,13 @@ class ValueState(Generic[T]): - def __init__(self, store: KvStore, name: str, codec: Codec[T]): - validate_state_name(name) + def __init__(self, store: KvStore, codec: Codec[T]): if store is None: raise KvError("value state store must not be None") if codec is None: raise KvError("value state codec must not be None") self._store = store self._codec = codec - state_name = name.strip() self._ck = ComplexKey( key_group=b"", key=b"", diff --git a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py index c40537e3..ee86e62e 100644 --- a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py +++ b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py @@ -20,11 +20,18 @@ MapState, ListState, PriorityQueueState, - KeyedStateFactory, - KeyedValueState, - KeyedMapState, - KeyedListState, - KeyedPriorityQueueState, + AggregatingState, + ReducingState, + KeyedListStateFactory, + KeyedValueStateFactory, + KeyedMapStateFactory, + KeyedPriorityQueueStateFactory, + KeyedAggregatingStateFactory, + KeyedReducingStateFactory, + PickleCodec, + BytesCodec, + OrderedInt64Codec, + default_codec_for, ) from .fs_collector import emit, emit_watermark @@ -72,41 +79,138 @@ def getOrCreateKVStore(self, name: str) -> KvStore: def getConfig(self) -> Dict[str, str]: return self._CONFIG.copy() - def getOrCreateValueState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> ValueState: + def getOrCreateValueState(self, store_name: str, codec: Codec) -> ValueState: store = self.getOrCreateKVStore(store_name) - return ValueState(store, state_name, codec) + return ValueState(store, codec) - def getOrCreateMapState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> MapState: + def getOrCreateValueStateAutoCodec(self, store_name: str) -> ValueState: store = self.getOrCreateKVStore(store_name) - return MapState(store, state_name, key_codec, value_codec) + return ValueState(store, PickleCodec()) - def getOrCreateListState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> ListState: + def getOrCreateMapState(self, store_name: str, key_codec: Codec, value_codec: Codec) -> MapState: store = self.getOrCreateKVStore(store_name) - return ListState(store, state_name, codec) + return MapState(store, key_codec, value_codec) - def getOrCreatePriorityQueueState(self, state_name: str, codec: Codec, store_name: str = "__fssdk_structured_state__") -> PriorityQueueState: + def getOrCreateMapStateAutoKeyCodec(self, store_name: str, value_codec: Codec) -> MapState: store = self.getOrCreateKVStore(store_name) - return PriorityQueueState(store, state_name, codec) + return MapState(store, BytesCodec(), value_codec) - def getOrCreateKeyedStateFactory(self, state_name: str, store_name: str = "__fssdk_structured_state__") -> KeyedStateFactory: + def getOrCreateListState(self, store_name: str, codec: Codec) -> ListState: store = self.getOrCreateKVStore(store_name) - return KeyedStateFactory(store, state_name) + return ListState(store, codec) - def getOrCreateKeyedValueState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedValueState: - factory = self.getOrCreateKeyedStateFactory(state_name, store_name) - return factory.new_keyed_value(key_codec, value_codec) + def getOrCreateListStateAutoCodec(self, store_name: str) -> ListState: + store = self.getOrCreateKVStore(store_name) + return ListState(store, PickleCodec()) + + def getOrCreatePriorityQueueState(self, store_name: str, codec: Codec) -> PriorityQueueState: + store = self.getOrCreateKVStore(store_name) + return PriorityQueueState(store, codec) + + def getOrCreatePriorityQueueStateAutoCodec(self, store_name: str) -> PriorityQueueState: + store = self.getOrCreateKVStore(store_name) + return PriorityQueueState(store, OrderedInt64Codec()) + + def getOrCreateAggregatingState( + self, store_name: str, acc_codec: Codec, agg_func: object + ) -> AggregatingState: + store = self.getOrCreateKVStore(store_name) + return AggregatingState(store, acc_codec, agg_func) + + def getOrCreateAggregatingStateAutoCodec( + self, store_name: str, agg_func: object + ) -> AggregatingState: + store = self.getOrCreateKVStore(store_name) + return AggregatingState(store, PickleCodec(), agg_func) + + def getOrCreateReducingState( + self, store_name: str, value_codec: Codec, reduce_func: object + ) -> ReducingState: + store = self.getOrCreateKVStore(store_name) + return ReducingState(store, value_codec, reduce_func) + + def getOrCreateReducingStateAutoCodec( + self, store_name: str, reduce_func: object + ) -> ReducingState: + store = self.getOrCreateKVStore(store_name) + return ReducingState(store, PickleCodec(), reduce_func) + + def getOrCreateKeyedListStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedListStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedListStateFactory(store, namespace, key_group, value_codec) + + def getOrCreateKeyedListStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_type=None + ) -> KeyedListStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return KeyedListStateFactory(store, namespace, key_group, codec) - def getOrCreateKeyedMapState(self, state_name: str, key_codec: Codec, map_key_codec: Codec, map_value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedMapState: - factory = self.getOrCreateKeyedStateFactory(state_name, store_name) - return factory.new_keyed_map(key_codec, map_key_codec, map_value_codec) + def getOrCreateKeyedValueStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedValueStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedValueStateFactory(store, namespace, key_group, value_codec) - def getOrCreateKeyedListState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedListState: - factory = self.getOrCreateKeyedStateFactory(state_name, store_name) - return factory.new_keyed_list(key_codec, value_codec) + def getOrCreateKeyedValueStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_type=None + ) -> KeyedValueStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return KeyedValueStateFactory(store, namespace, key_group, codec) - def getOrCreateKeyedPriorityQueueState(self, state_name: str, key_codec: Codec, value_codec: Codec, store_name: str = "__fssdk_structured_state__") -> KeyedPriorityQueueState: - factory = self.getOrCreateKeyedStateFactory(state_name, store_name) - return factory.new_keyed_priority_queue(key_codec, value_codec) + def getOrCreateKeyedMapStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, key_codec: Codec, value_codec: Codec + ) -> KeyedMapStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedMapStateFactory(store, namespace, key_group, key_codec, value_codec) + + def getOrCreateKeyedMapStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedMapStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedMapStateFactory(store, namespace, key_group, BytesCodec(), value_codec) + + def getOrCreateKeyedPriorityQueueStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, item_codec: Codec + ) -> KeyedPriorityQueueStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedPriorityQueueStateFactory(store, namespace, key_group, item_codec) + + def getOrCreateKeyedPriorityQueueStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, item_type=None + ) -> KeyedPriorityQueueStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(item_type) if item_type is not None else OrderedInt64Codec() + return KeyedPriorityQueueStateFactory(store, namespace, key_group, codec) + + def getOrCreateKeyedAggregatingStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, acc_codec: Codec, agg_func: object + ) -> KeyedAggregatingStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedAggregatingStateFactory(store, namespace, key_group, acc_codec, agg_func) + + def getOrCreateKeyedAggregatingStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, agg_func: object, acc_type=None + ) -> KeyedAggregatingStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(acc_type) if acc_type is not None else PickleCodec() + return KeyedAggregatingStateFactory(store, namespace, key_group, codec, agg_func) + + def getOrCreateKeyedReducingStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec, reduce_func: object + ) -> KeyedReducingStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedReducingStateFactory(store, namespace, key_group, value_codec, reduce_func) + + def getOrCreateKeyedReducingStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, reduce_func: object, value_type=None + ) -> KeyedReducingStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return KeyedReducingStateFactory(store, namespace, key_group, codec, reduce_func) __all__ = ['WitContext', 'convert_config_to_dict'] From fe629082475c3afeac3ece75f3695e15dca54397 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Mon, 9 Mar 2026 23:31:09 +0800 Subject: [PATCH 4/9] fix --- python/functionstream-api/pyproject.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/functionstream-api/pyproject.toml b/python/functionstream-api/pyproject.toml index b7c107c6..adc2651b 100644 --- a/python/functionstream-api/pyproject.toml +++ b/python/functionstream-api/pyproject.toml @@ -25,7 +25,9 @@ dependencies = [ "cloudpickle>=2.0.0", ] -[tool.setuptools] -package-dir = {"" = "src"} -packages = ["fs_api", "fs_api.store"] +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-dir] +"" = "src" From 631e40603a40ef824389a81ed5d79db4b92c7c74 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 11 Mar 2026 00:31:29 +0800 Subject: [PATCH 5/9] update --- go-sdk/impl/state.go | 8 +- go-sdk/state/codec/default_codec.go | 26 ++-- go-sdk/state/codec/float32_codec.go | 21 ++- go-sdk/state/codec/float64_codec.go | 21 ++- ...ordered_uint64_codec.go => int16_codec.go} | 20 +-- go-sdk/state/codec/int32_codec.go | 9 +- go-sdk/state/codec/int64_codec.go | 12 +- .../{ordered_int64_codec.go => int8_codec.go} | 21 ++- go-sdk/state/codec/ordered_float32_codec.go | 50 ------- go-sdk/state/codec/ordered_float64_codec.go | 50 ------- go-sdk/state/codec/ordered_int32_codec.go | 37 ----- go-sdk/state/codec/ordered_int_codec.go | 37 ----- go-sdk/state/codec/ordered_uint_codec.go | 37 ----- ...rdered_uint32_codec.go => uint16_codec.go} | 20 +-- go-sdk/state/codec/uint32_codec.go | 5 +- go-sdk/state/codec/uint64_codec.go | 5 +- .../codec/{bytes_codec.go => uint8_codec.go} | 21 ++- go-sdk/state/common/common.go | 39 ------ go-sdk/state/keyed/keyed_aggregating_state.go | 18 ++- go-sdk/state/keyed/keyed_list_state.go | 27 ++-- go-sdk/state/keyed/keyed_map_state.go | 20 ++- .../state/keyed/keyed_priority_queue_state.go | 20 ++- go-sdk/state/keyed/keyed_reducing_state.go | 18 ++- go-sdk/state/keyed/keyed_state_factory.go | 49 ------- go-sdk/state/keyed/keyed_value_state.go | 16 +-- go-sdk/state/structures/map.go | 127 +----------------- .../functionstream-api/src/fs_api/__init__.py | 30 +---- .../src/fs_api/store/__init__.py | 32 +---- .../src/fs_api/store/codec/__init__.py | 28 +--- .../src/fs_api/store/codec/default_codec.py | 14 +- .../src/fs_api/store/codec/float32_codec.py | 25 ---- .../src/fs_api/store/codec/float64_codec.py | 25 ---- ...rdered_float64_codec.py => float_codec.py} | 5 +- .../src/fs_api/store/codec/int32_codec.py | 25 ---- .../src/fs_api/store/codec/int64_codec.py | 25 ---- .../{ordered_int64_codec.py => int_codec.py} | 5 +- .../store/codec/ordered_float32_codec.py | 37 ----- .../fs_api/store/codec/ordered_int32_codec.py | 32 ----- .../store/codec/ordered_uint32_codec.py | 29 ---- .../store/codec/ordered_uint64_codec.py | 29 ---- .../src/fs_api/store/codec/uint32_codec.py | 27 ---- .../src/fs_api/store/codec/uint64_codec.py | 27 ---- .../src/fs_api/store/common/__init__.py | 88 ------------ .../src/fs_api/store/keyed/__init__.py | 2 - .../src/fs_api/store/keyed/_keyed_common.py | 14 -- .../store/keyed/keyed_aggregating_state.py | 2 +- .../fs_api/store/keyed/keyed_list_state.py | 2 +- .../src/fs_api/store/keyed/keyed_map_state.py | 51 +------ .../store/keyed/keyed_priority_queue_state.py | 2 +- .../store/keyed/keyed_reducing_state.py | 2 +- .../fs_api/store/keyed/keyed_state_factory.py | 113 ---------------- .../fs_api/store/keyed/keyed_value_state.py | 2 - .../src/fs_api/store/structures/map_state.py | 76 ++--------- .../src/fs_runtime/store/fs_context.py | 6 +- 54 files changed, 222 insertions(+), 1267 deletions(-) rename go-sdk/state/codec/{ordered_uint64_codec.go => int16_codec.go} (55%) rename go-sdk/state/codec/{ordered_int64_codec.go => int8_codec.go} (56%) delete mode 100644 go-sdk/state/codec/ordered_float32_codec.go delete mode 100644 go-sdk/state/codec/ordered_float64_codec.go delete mode 100644 go-sdk/state/codec/ordered_int32_codec.go delete mode 100644 go-sdk/state/codec/ordered_int_codec.go delete mode 100644 go-sdk/state/codec/ordered_uint_codec.go rename go-sdk/state/codec/{ordered_uint32_codec.go => uint16_codec.go} (55%) rename go-sdk/state/codec/{bytes_codec.go => uint8_codec.go} (57%) delete mode 100644 go-sdk/state/keyed/keyed_state_factory.go delete mode 100644 python/functionstream-api/src/fs_api/store/codec/float32_codec.py delete mode 100644 python/functionstream-api/src/fs_api/store/codec/float64_codec.py rename python/functionstream-api/src/fs_api/store/codec/{ordered_float64_codec.py => float_codec.py} (87%) delete mode 100644 python/functionstream-api/src/fs_api/store/codec/int32_codec.py delete mode 100644 python/functionstream-api/src/fs_api/store/codec/int64_codec.py rename python/functionstream-api/src/fs_api/store/codec/{ordered_int64_codec.py => int_codec.py} (85%) delete mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py delete mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py delete mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py delete mode 100644 python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py delete mode 100644 python/functionstream-api/src/fs_api/store/codec/uint32_codec.py delete mode 100644 python/functionstream-api/src/fs_api/store/codec/uint64_codec.py delete mode 100644 python/functionstream-api/src/fs_api/store/common/__init__.py delete mode 100644 python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py diff --git a/go-sdk/impl/state.go b/go-sdk/impl/state.go index 2befbedb..530a03db 100644 --- a/go-sdk/impl/state.go +++ b/go-sdk/impl/state.go @@ -87,20 +87,20 @@ func NewReducingState[V any](ctx api.Context, storeName string, valueCodec codec return structures.NewReducingState(s, valueCodec, reduceFunc) } -func NewKeyedListStateFactory[V any](ctx api.Context, storeName, name string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedListStateFactory[V], error) { +func NewKeyedListStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedListStateFactory[V], error) { s, err := getStoreFromContext(ctx, storeName) if err != nil { return nil, err } - return keyed.NewKeyedListStateFactory(s, name, keyGroup, valueCodec) + return keyed.NewKeyedListStateFactory(s, keyGroup, valueCodec) } -func NewKeyedListStateFactoryAutoCodec[V any](ctx api.Context, storeName, name string, keyGroup []byte) (*keyed.KeyedListStateFactory[V], error) { +func NewKeyedListStateFactoryAutoCodec[V any](ctx api.Context, storeName string, keyGroup []byte) (*keyed.KeyedListStateFactory[V], error) { s, err := getStoreFromContext(ctx, storeName) if err != nil { return nil, err } - return keyed.NewKeyedListStateFactoryAutoCodec[V](s, name, keyGroup) + return keyed.NewKeyedListStateFactoryAutoCodec[V](s, keyGroup) } func NewKeyedValueStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedValueStateFactory[V], error) { diff --git a/go-sdk/state/codec/default_codec.go b/go-sdk/state/codec/default_codec.go index 7a18321b..16fa868d 100644 --- a/go-sdk/state/codec/default_codec.go +++ b/go-sdk/state/codec/default_codec.go @@ -28,21 +28,31 @@ func DefaultCodecFor[V any]() (Codec[V], error) { case reflect.Bool: return any(BoolCodec{}).(Codec[V]), nil case reflect.Int32: - return any(OrderedInt32Codec{}).(Codec[V]), nil + return any(Int32Codec{}).(Codec[V]), nil case reflect.Int64: - return any(OrderedInt64Codec{}).(Codec[V]), nil + return any(Int64Codec{}).(Codec[V]), nil case reflect.Uint32: - return any(OrderedUint32Codec{}).(Codec[V]), nil + return any(Uint32Codec{}).(Codec[V]), nil case reflect.Uint64: - return any(OrderedUint64Codec{}).(Codec[V]), nil + return any(Uint64Codec{}).(Codec[V]), nil case reflect.Float32: - return any(OrderedFloat32Codec{}).(Codec[V]), nil + return any(Float32Codec{}).(Codec[V]), nil case reflect.Float64: - return any(OrderedFloat64Codec{}).(Codec[V]), nil + return any(Float64Codec{}).(Codec[V]), nil case reflect.String: return any(StringCodec{}).(Codec[V]), nil - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Uint, reflect.Uint8, reflect.Uint16: - return any(JSONCodec[V]{}).(Codec[V]), nil + case reflect.Int: + return any(Int64Codec{}).(Codec[V]), nil + case reflect.Int8: + return any(Int8Codec{}).(Codec[V]), nil + case reflect.Int16: + return any(Int16Codec{}).(Codec[V]), nil + case reflect.Uint: + return any(Uint64Codec{}).(Codec[V]), nil + case reflect.Uint8: + return any(Uint8Codec{}).(Codec[V]), nil + case reflect.Uint16: + return any(Uint16Codec{}).(Codec[V]), nil case reflect.Struct, reflect.Map, reflect.Slice, reflect.Array, reflect.Interface: return any(JSONCodec[V]{}).(Codec[V]), nil default: diff --git a/go-sdk/state/codec/float32_codec.go b/go-sdk/state/codec/float32_codec.go index 8967596a..139f76ec 100644 --- a/go-sdk/state/codec/float32_codec.go +++ b/go-sdk/state/codec/float32_codec.go @@ -21,8 +21,14 @@ import ( type Float32Codec struct{} func (c Float32Codec) Encode(value float32) ([]byte, error) { + bits := math.Float32bits(value) + if (bits & (uint32(1) << 31)) != 0 { + bits = ^bits + } else { + bits ^= uint32(1) << 31 + } out := make([]byte, 4) - binary.BigEndian.PutUint32(out, math.Float32bits(value)) + binary.BigEndian.PutUint32(out, bits) return out, nil } @@ -30,8 +36,15 @@ func (c Float32Codec) Decode(data []byte) (float32, error) { if len(data) != 4 { return 0, fmt.Errorf("invalid float32 payload length: %d", len(data)) } - return math.Float32frombits(binary.BigEndian.Uint32(data)), nil + encoded := binary.BigEndian.Uint32(data) + if (encoded & (uint32(1) << 31)) != 0 { + encoded ^= uint32(1) << 31 + } else { + encoded = ^encoded + } + return math.Float32frombits(encoded), nil } -func (c Float32Codec) EncodedSize() int { return 4 } -func (c Float32Codec) IsOrderedKeyCodec() bool { return false } +func (c Float32Codec) EncodedSize() int { return 4 } + +func (c Float32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/float64_codec.go b/go-sdk/state/codec/float64_codec.go index 1e5d484c..895fb810 100644 --- a/go-sdk/state/codec/float64_codec.go +++ b/go-sdk/state/codec/float64_codec.go @@ -21,8 +21,14 @@ import ( type Float64Codec struct{} func (c Float64Codec) Encode(value float64) ([]byte, error) { + bits := math.Float64bits(value) + if (bits & (uint64(1) << 63)) != 0 { + bits = ^bits + } else { + bits ^= uint64(1) << 63 + } out := make([]byte, 8) - binary.BigEndian.PutUint64(out, math.Float64bits(value)) + binary.BigEndian.PutUint64(out, bits) return out, nil } @@ -30,8 +36,15 @@ func (c Float64Codec) Decode(data []byte) (float64, error) { if len(data) != 8 { return 0, fmt.Errorf("invalid float64 payload length: %d", len(data)) } - return math.Float64frombits(binary.BigEndian.Uint64(data)), nil + encoded := binary.BigEndian.Uint64(data) + if (encoded & (uint64(1) << 63)) != 0 { + encoded ^= uint64(1) << 63 + } else { + encoded = ^encoded + } + return math.Float64frombits(encoded), nil } -func (c Float64Codec) EncodedSize() int { return 8 } -func (c Float64Codec) IsOrderedKeyCodec() bool { return false } +func (c Float64Codec) EncodedSize() int { return 8 } + +func (c Float64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint64_codec.go b/go-sdk/state/codec/int16_codec.go similarity index 55% rename from go-sdk/state/codec/ordered_uint64_codec.go rename to go-sdk/state/codec/int16_codec.go index 239b21ff..f678f792 100644 --- a/go-sdk/state/codec/ordered_uint64_codec.go +++ b/go-sdk/state/codec/int16_codec.go @@ -17,21 +17,21 @@ import ( "fmt" ) -type OrderedUint64Codec struct{} +type Int16Codec struct{} -func (c OrderedUint64Codec) Encode(value uint64) ([]byte, error) { - out := make([]byte, 8) - binary.BigEndian.PutUint64(out, value) +func (c Int16Codec) Encode(value int16) ([]byte, error) { + out := make([]byte, 2) + binary.BigEndian.PutUint16(out, uint16(value)^(uint16(1)<<15)) return out, nil } -func (c OrderedUint64Codec) Decode(data []byte) (uint64, error) { - if len(data) != 8 { - return 0, fmt.Errorf("invalid ordered uint64 payload length: %d", len(data)) +func (c Int16Codec) Decode(data []byte) (int16, error) { + if len(data) != 2 { + return 0, fmt.Errorf("invalid int16 payload length: %d", len(data)) } - return binary.BigEndian.Uint64(data), nil + return int16(binary.BigEndian.Uint16(data) ^ (uint16(1) << 15)), nil } -func (c OrderedUint64Codec) EncodedSize() int { return 8 } +func (c Int16Codec) EncodedSize() int { return 2 } -func (c OrderedUint64Codec) IsOrderedKeyCodec() bool { return true } +func (c Int16Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/int32_codec.go b/go-sdk/state/codec/int32_codec.go index 40fe0507..03792712 100644 --- a/go-sdk/state/codec/int32_codec.go +++ b/go-sdk/state/codec/int32_codec.go @@ -21,7 +21,7 @@ type Int32Codec struct{} func (c Int32Codec) Encode(value int32) ([]byte, error) { out := make([]byte, 4) - binary.BigEndian.PutUint32(out, uint32(value)) + binary.BigEndian.PutUint32(out, uint32(value)^(uint32(1)<<31)) return out, nil } @@ -29,8 +29,9 @@ func (c Int32Codec) Decode(data []byte) (int32, error) { if len(data) != 4 { return 0, fmt.Errorf("invalid int32 payload length: %d", len(data)) } - return int32(binary.BigEndian.Uint32(data)), nil + return int32(binary.BigEndian.Uint32(data) ^ (uint32(1) << 31)), nil } -func (c Int32Codec) EncodedSize() int { return 4 } -func (c Int32Codec) IsOrderedKeyCodec() bool { return false } +func (c Int32Codec) EncodedSize() int { return 4 } + +func (c Int32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/int64_codec.go b/go-sdk/state/codec/int64_codec.go index 60046e9a..cb632bda 100644 --- a/go-sdk/state/codec/int64_codec.go +++ b/go-sdk/state/codec/int64_codec.go @@ -21,7 +21,8 @@ type Int64Codec struct{} func (c Int64Codec) Encode(value int64) ([]byte, error) { out := make([]byte, 8) - binary.BigEndian.PutUint64(out, uint64(value)) + mapped := uint64(value) ^ (uint64(1) << 63) + binary.BigEndian.PutUint64(out, mapped) return out, nil } @@ -29,8 +30,11 @@ func (c Int64Codec) Decode(data []byte) (int64, error) { if len(data) != 8 { return 0, fmt.Errorf("invalid int64 payload length: %d", len(data)) } - return int64(binary.BigEndian.Uint64(data)), nil + mapped := binary.BigEndian.Uint64(data) + raw := int64(mapped ^ (uint64(1) << 63)) + return raw, nil } -func (c Int64Codec) EncodedSize() int { return 8 } -func (c Int64Codec) IsOrderedKeyCodec() bool { return false } +func (c Int64Codec) EncodedSize() int { return 8 } + +func (c Int64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int64_codec.go b/go-sdk/state/codec/int8_codec.go similarity index 56% rename from go-sdk/state/codec/ordered_int64_codec.go rename to go-sdk/state/codec/int8_codec.go index dc8b5346..fac2dd81 100644 --- a/go-sdk/state/codec/ordered_int64_codec.go +++ b/go-sdk/state/codec/int8_codec.go @@ -12,16 +12,23 @@ package codec -import "github.com/functionstream/function-stream/go-sdk/state/common" +import ( + "fmt" +) -type OrderedInt64Codec struct{} +type Int8Codec struct{} -func (c OrderedInt64Codec) Encode(value int64) ([]byte, error) { - return common.EncodeInt64Lex(value), nil +func (c Int8Codec) Encode(value int8) ([]byte, error) { + return []byte{byte(uint8(value) ^ (1 << 7))}, nil } -func (c OrderedInt64Codec) Decode(data []byte) (int64, error) { return common.DecodeInt64Lex(data) } +func (c Int8Codec) Decode(data []byte) (int8, error) { + if len(data) != 1 { + return 0, fmt.Errorf("invalid int8 payload length: %d", len(data)) + } + return int8(data[0] ^ (1 << 7)), nil +} -func (c OrderedInt64Codec) EncodedSize() int { return 8 } +func (c Int8Codec) EncodedSize() int { return 1 } -func (c OrderedInt64Codec) IsOrderedKeyCodec() bool { return true } +func (c Int8Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_float32_codec.go b/go-sdk/state/codec/ordered_float32_codec.go deleted file mode 100644 index ea9a3c68..00000000 --- a/go-sdk/state/codec/ordered_float32_codec.go +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package codec - -import ( - "encoding/binary" - "fmt" - "math" -) - -type OrderedFloat32Codec struct{} - -func (c OrderedFloat32Codec) Encode(value float32) ([]byte, error) { - bits := math.Float32bits(value) - if (bits & (uint32(1) << 31)) != 0 { - bits = ^bits - } else { - bits ^= uint32(1) << 31 - } - out := make([]byte, 4) - binary.BigEndian.PutUint32(out, bits) - return out, nil -} - -func (c OrderedFloat32Codec) Decode(data []byte) (float32, error) { - if len(data) != 4 { - return 0, fmt.Errorf("invalid ordered float32 payload length: %d", len(data)) - } - encoded := binary.BigEndian.Uint32(data) - if (encoded & (uint32(1) << 31)) != 0 { - encoded ^= uint32(1) << 31 - } else { - encoded = ^encoded - } - return math.Float32frombits(encoded), nil -} - -func (c OrderedFloat32Codec) EncodedSize() int { return 4 } - -func (c OrderedFloat32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_float64_codec.go b/go-sdk/state/codec/ordered_float64_codec.go deleted file mode 100644 index d032e2f0..00000000 --- a/go-sdk/state/codec/ordered_float64_codec.go +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package codec - -import ( - "encoding/binary" - "fmt" - "math" -) - -type OrderedFloat64Codec struct{} - -func (c OrderedFloat64Codec) Encode(value float64) ([]byte, error) { - bits := math.Float64bits(value) - if (bits & (uint64(1) << 63)) != 0 { - bits = ^bits - } else { - bits ^= uint64(1) << 63 - } - out := make([]byte, 8) - binary.BigEndian.PutUint64(out, bits) - return out, nil -} - -func (c OrderedFloat64Codec) Decode(data []byte) (float64, error) { - if len(data) != 8 { - return 0, fmt.Errorf("invalid ordered float64 payload length: %d", len(data)) - } - encoded := binary.BigEndian.Uint64(data) - if (encoded & (uint64(1) << 63)) != 0 { - encoded ^= uint64(1) << 63 - } else { - encoded = ^encoded - } - return math.Float64frombits(encoded), nil -} - -func (c OrderedFloat64Codec) EncodedSize() int { return 8 } - -func (c OrderedFloat64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int32_codec.go b/go-sdk/state/codec/ordered_int32_codec.go deleted file mode 100644 index 1b48e92d..00000000 --- a/go-sdk/state/codec/ordered_int32_codec.go +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package codec - -import ( - "encoding/binary" - "fmt" -) - -type OrderedInt32Codec struct{} - -func (c OrderedInt32Codec) Encode(value int32) ([]byte, error) { - out := make([]byte, 4) - binary.BigEndian.PutUint32(out, uint32(value)^(uint32(1)<<31)) - return out, nil -} - -func (c OrderedInt32Codec) Decode(data []byte) (int32, error) { - if len(data) != 4 { - return 0, fmt.Errorf("invalid ordered int32 payload length: %d", len(data)) - } - return int32(binary.BigEndian.Uint32(data) ^ (uint32(1) << 31)), nil -} - -func (c OrderedInt32Codec) EncodedSize() int { return 4 } - -func (c OrderedInt32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int_codec.go b/go-sdk/state/codec/ordered_int_codec.go deleted file mode 100644 index ebdc9e90..00000000 --- a/go-sdk/state/codec/ordered_int_codec.go +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package codec - -import "strconv" - -type OrderedIntCodec struct{} - -func (c OrderedIntCodec) Encode(value int) ([]byte, error) { - if strconv.IntSize == 32 { - return OrderedInt32Codec{}.Encode(int32(value)) - } - return OrderedInt64Codec{}.Encode(int64(value)) -} - -func (c OrderedIntCodec) Decode(data []byte) (int, error) { - if strconv.IntSize == 32 { - v, err := OrderedInt32Codec{}.Decode(data) - return int(v), err - } - v, err := OrderedInt64Codec{}.Decode(data) - return int(v), err -} - -func (c OrderedIntCodec) EncodedSize() int { return strconv.IntSize / 8 } - -func (c OrderedIntCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint_codec.go b/go-sdk/state/codec/ordered_uint_codec.go deleted file mode 100644 index 6642d8d1..00000000 --- a/go-sdk/state/codec/ordered_uint_codec.go +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package codec - -import "strconv" - -type OrderedUintCodec struct{} - -func (c OrderedUintCodec) Encode(value uint) ([]byte, error) { - if strconv.IntSize == 32 { - return OrderedUint32Codec{}.Encode(uint32(value)) - } - return OrderedUint64Codec{}.Encode(uint64(value)) -} - -func (c OrderedUintCodec) Decode(data []byte) (uint, error) { - if strconv.IntSize == 32 { - v, err := OrderedUint32Codec{}.Decode(data) - return uint(v), err - } - v, err := OrderedUint64Codec{}.Decode(data) - return uint(v), err -} - -func (c OrderedUintCodec) EncodedSize() int { return strconv.IntSize / 8 } - -func (c OrderedUintCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint32_codec.go b/go-sdk/state/codec/uint16_codec.go similarity index 55% rename from go-sdk/state/codec/ordered_uint32_codec.go rename to go-sdk/state/codec/uint16_codec.go index a6a5ce1e..524553a5 100644 --- a/go-sdk/state/codec/ordered_uint32_codec.go +++ b/go-sdk/state/codec/uint16_codec.go @@ -17,21 +17,21 @@ import ( "fmt" ) -type OrderedUint32Codec struct{} +type Uint16Codec struct{} -func (c OrderedUint32Codec) Encode(value uint32) ([]byte, error) { - out := make([]byte, 4) - binary.BigEndian.PutUint32(out, value) +func (c Uint16Codec) Encode(value uint16) ([]byte, error) { + out := make([]byte, 2) + binary.BigEndian.PutUint16(out, value) return out, nil } -func (c OrderedUint32Codec) Decode(data []byte) (uint32, error) { - if len(data) != 4 { - return 0, fmt.Errorf("invalid ordered uint32 payload length: %d", len(data)) +func (c Uint16Codec) Decode(data []byte) (uint16, error) { + if len(data) != 2 { + return 0, fmt.Errorf("invalid uint16 payload length: %d", len(data)) } - return binary.BigEndian.Uint32(data), nil + return binary.BigEndian.Uint16(data), nil } -func (c OrderedUint32Codec) EncodedSize() int { return 4 } +func (c Uint16Codec) EncodedSize() int { return 2 } -func (c OrderedUint32Codec) IsOrderedKeyCodec() bool { return true } +func (c Uint16Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/uint32_codec.go b/go-sdk/state/codec/uint32_codec.go index 958b9609..a9555278 100644 --- a/go-sdk/state/codec/uint32_codec.go +++ b/go-sdk/state/codec/uint32_codec.go @@ -32,5 +32,6 @@ func (c Uint32Codec) Decode(data []byte) (uint32, error) { return binary.BigEndian.Uint32(data), nil } -func (c Uint32Codec) EncodedSize() int { return 4 } -func (c Uint32Codec) IsOrderedKeyCodec() bool { return false } +func (c Uint32Codec) EncodedSize() int { return 4 } + +func (c Uint32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/uint64_codec.go b/go-sdk/state/codec/uint64_codec.go index f4001e24..485df695 100644 --- a/go-sdk/state/codec/uint64_codec.go +++ b/go-sdk/state/codec/uint64_codec.go @@ -32,5 +32,6 @@ func (c Uint64Codec) Decode(data []byte) (uint64, error) { return binary.BigEndian.Uint64(data), nil } -func (c Uint64Codec) EncodedSize() int { return 8 } -func (c Uint64Codec) IsOrderedKeyCodec() bool { return false } +func (c Uint64Codec) EncodedSize() int { return 8 } + +func (c Uint64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/bytes_codec.go b/go-sdk/state/codec/uint8_codec.go similarity index 57% rename from go-sdk/state/codec/bytes_codec.go rename to go-sdk/state/codec/uint8_codec.go index b428d506..45e1ce36 100644 --- a/go-sdk/state/codec/bytes_codec.go +++ b/go-sdk/state/codec/uint8_codec.go @@ -12,14 +12,23 @@ package codec -import "github.com/functionstream/function-stream/go-sdk/state/common" +import ( + "fmt" +) -type BytesCodec struct{} +type Uint8Codec struct{} -func (c BytesCodec) Encode(value []byte) ([]byte, error) { return common.DupBytes(value), nil } +func (c Uint8Codec) Encode(value uint8) ([]byte, error) { + return []byte{byte(value)}, nil +} -func (c BytesCodec) Decode(data []byte) ([]byte, error) { return common.DupBytes(data), nil } +func (c Uint8Codec) Decode(data []byte) (uint8, error) { + if len(data) != 1 { + return 0, fmt.Errorf("invalid uint8 payload length: %d", len(data)) + } + return data[0], nil +} -func (c BytesCodec) EncodedSize() int { return -1 } +func (c Uint8Codec) EncodedSize() int { return 1 } -func (c BytesCodec) IsOrderedKeyCodec() bool { return true } +func (c Uint8Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/common/common.go b/go-sdk/state/common/common.go index a31c2a37..b889fb28 100644 --- a/go-sdk/state/common/common.go +++ b/go-sdk/state/common/common.go @@ -13,50 +13,11 @@ package common import ( - "encoding/binary" - "fmt" - "strings" - "github.com/functionstream/function-stream/go-sdk/api" ) type Store = api.Store -func ValidateStateName(name string) (string, error) { - stateName := strings.TrimSpace(name) - if stateName == "" { - return "", api.NewError(api.ErrStoreInvalidName, "state name must not be empty") - } - return stateName, nil -} - -func EncodeInt64Lex(v int64) []byte { - out := make([]byte, 8) - binary.BigEndian.PutUint64(out, uint64(v)^(uint64(1)<<63)) - return out -} - -func DecodeInt64Lex(data []byte) (int64, error) { - if len(data) != 8 { - return 0, fmt.Errorf("invalid int64 lex key length: %d", len(data)) - } - return int64(binary.BigEndian.Uint64(data) ^ (uint64(1) << 63)), nil -} - -func EncodePriorityUserKey(priority int64, seq uint64) []byte { - out := make([]byte, 16) - copy(out[:8], EncodeInt64Lex(priority)) - binary.BigEndian.PutUint64(out[8:], seq) - return out -} - -func DecodePriorityUserKey(data []byte) (int64, error) { - if len(data) != 16 { - return 0, fmt.Errorf("invalid priority key length: %d", len(data)) - } - return DecodeInt64Lex(data[:8]) -} - func DupBytes(input []byte) []byte { if input == nil { return nil diff --git a/go-sdk/state/keyed/keyed_aggregating_state.go b/go-sdk/state/keyed/keyed_aggregating_state.go index 4787889c..748ed2b6 100644 --- a/go-sdk/state/keyed/keyed_aggregating_state.go +++ b/go-sdk/state/keyed/keyed_aggregating_state.go @@ -28,7 +28,7 @@ type AggregateFunc[T any, ACC any, R any] interface { } type KeyedAggregatingStateFactory[T any, ACC any, R any] struct { - inner *keyedStateFactory + store common.Store groupKey []byte accCodec codec.Codec[ACC] aggFunc AggregateFunc[T, ACC, R] @@ -41,11 +41,9 @@ func NewKeyedAggregatingStateFactory[T any, ACC any, R any]( aggFunc AggregateFunc[T, ACC, R], ) (*KeyedAggregatingStateFactory[T, ACC, R], error) { - inner, err := newKeyedStateFactory(store, "", "aggregating") - if err != nil { - return nil, err + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed aggregating state factory store must not be nil") } - if keyGroup == nil { return nil, api.NewError(api.ErrStoreInternal, "keyed aggregating state factory key_group must not be nil") } @@ -57,7 +55,7 @@ func NewKeyedAggregatingStateFactory[T any, ACC any, R any]( } return &KeyedAggregatingStateFactory[T, ACC, R]{ - inner: inner, + store: store, groupKey: common.DupBytes(keyGroup), accCodec: accCodec, aggFunc: aggFunc, @@ -90,7 +88,7 @@ func (s *KeyedAggregatingState[T, ACC, R]) buildCK() api.ComplexKey { func (s *KeyedAggregatingState[T, ACC, R]) Add(value T) error { ck := s.buildCK() - raw, found, err := s.factory.inner.store.Get(ck) + raw, found, err := s.factory.store.Get(ck) if err != nil { return fmt.Errorf("failed to get accumulator: %w", err) } @@ -112,13 +110,13 @@ func (s *KeyedAggregatingState[T, ACC, R]) Add(value T) error { if err != nil { return fmt.Errorf("failed to encode new accumulator: %w", err) } - return s.factory.inner.store.Put(ck, encoded) + return s.factory.store.Put(ck, encoded) } func (s *KeyedAggregatingState[T, ACC, R]) Get() (R, bool, error) { var zero R ck := s.buildCK() - raw, found, err := s.factory.inner.store.Get(ck) + raw, found, err := s.factory.store.Get(ck) if err != nil || !found { return zero, found, err } @@ -132,5 +130,5 @@ func (s *KeyedAggregatingState[T, ACC, R]) Get() (R, bool, error) { } func (s *KeyedAggregatingState[T, ACC, R]) Clear() error { - return s.factory.inner.store.Delete(s.buildCK()) + return s.factory.store.Delete(s.buildCK()) } diff --git a/go-sdk/state/keyed/keyed_list_state.go b/go-sdk/state/keyed/keyed_list_state.go index 4c02a0b8..b33b44f0 100644 --- a/go-sdk/state/keyed/keyed_list_state.go +++ b/go-sdk/state/keyed/keyed_list_state.go @@ -22,27 +22,26 @@ import ( ) type KeyedListStateFactory[V any] struct { - inner *keyedStateFactory + store common.Store keyGroup []byte fixedSize int valueCodec codec.Codec[V] isFixed bool } -func NewKeyedListStateFactory[V any](store common.Store, name string, keyGroup []byte, valueCodec codec.Codec[V]) (*KeyedListStateFactory[V], error) { - inner, err := newKeyedStateFactory(store, name, "list") - if err != nil { - return nil, err +func NewKeyedListStateFactory[V any](store common.Store,keyGroup []byte, valueCodec codec.Codec[V]) (*KeyedListStateFactory[V], error) { + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list state factory store must not be nil") } if keyGroup == nil { - return nil, api.NewError(api.ErrStoreInternal, "keyed list state factory %q key group must not be nil", inner.name) + return nil, api.NewError(api.ErrStoreInternal, "keyed list state factory key group must not be nil") } if valueCodec == nil { return nil, api.NewError(api.ErrStoreInternal, "keyed list value codec must not be nil") } fixedSize, isFixed := codec.FixedEncodedSize[V](valueCodec) return &KeyedListStateFactory[V]{ - inner: inner, + store: store, keyGroup: common.DupBytes(keyGroup), fixedSize: fixedSize, valueCodec: valueCodec, @@ -50,12 +49,12 @@ func NewKeyedListStateFactory[V any](store common.Store, name string, keyGroup [ }, nil } -func NewKeyedListStateFactoryAutoCodec[V any](store common.Store, name string, keyGroup []byte) (*KeyedListStateFactory[V], error) { +func NewKeyedListStateFactoryAutoCodec[V any](store common.Store, keyGroup []byte) (*KeyedListStateFactory[V], error) { valueCodec, err := codec.DefaultCodecFor[V]() if err != nil { return nil, err } - return NewKeyedListStateFactory[V](store, name, keyGroup, valueCodec) + return NewKeyedListStateFactory[V](store, keyGroup, valueCodec) } type KeyedListState[V any] struct { @@ -110,7 +109,7 @@ func (s *KeyedListState[V]) Add(value V) error { if err != nil { return err } - return s.factory.inner.store.Merge(s.complexKey, payload) + return s.factory.store.Merge(s.complexKey, payload) } func (s *KeyedListState[V]) AddAll(values []V) error { @@ -118,14 +117,14 @@ func (s *KeyedListState[V]) AddAll(values []V) error { if err != nil { return err } - if err := s.factory.inner.store.Merge(s.complexKey, payload); err != nil { + if err := s.factory.store.Merge(s.complexKey, payload); err != nil { return err } return nil } func (s *KeyedListState[V]) Get() ([]V, error) { - raw, found, err := s.factory.inner.store.Get(s.complexKey) + raw, found, err := s.factory.store.Get(s.complexKey) if err != nil { return nil, err } @@ -144,11 +143,11 @@ func (s *KeyedListState[V]) Update(values []V) error { if err != nil { return err } - return s.factory.inner.store.Put(s.complexKey, payload) + return s.factory.store.Put(s.complexKey, payload) } func (s *KeyedListState[V]) Clear() error { - return s.factory.inner.store.Delete(s.complexKey) + return s.factory.store.Delete(s.complexKey) } func (s *KeyedListState[V]) serializeValueVarLen(value V) ([]byte, error) { diff --git a/go-sdk/state/keyed/keyed_map_state.go b/go-sdk/state/keyed/keyed_map_state.go index 8fc7e09b..937cffe8 100644 --- a/go-sdk/state/keyed/keyed_map_state.go +++ b/go-sdk/state/keyed/keyed_map_state.go @@ -22,7 +22,7 @@ import ( ) type KeyedMapStateFactory[MK any, MV any] struct { - inner *keyedStateFactory + store common.Store groupKey []byte mapKeyCodec codec.Codec[MK] mapValueCodec codec.Codec[MV] @@ -35,11 +35,9 @@ func NewKeyedMapStateFactory[MK any, MV any]( mapValueCodec codec.Codec[MV], ) (*KeyedMapStateFactory[MK, MV], error) { - inner, err := newKeyedStateFactory(store, "", "map") - if err != nil { - return nil, err + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed map state factory store must not be nil") } - if keyGroup == nil { return nil, api.NewError(api.ErrStoreInternal, "keyed map state factory key_group must not be nil") } @@ -52,7 +50,7 @@ func NewKeyedMapStateFactory[MK any, MV any]( } return &KeyedMapStateFactory[MK, MV]{ - inner: inner, + store: store, groupKey: common.DupBytes(keyGroup), mapKeyCodec: mapKeyCodec, mapValueCodec: mapValueCodec, @@ -98,7 +96,7 @@ func (s *KeyedMapState[MK, MV]) Put(mapKey MK, value MV) error { if err != nil { return err } - return s.factory.inner.store.Put(ck, encodedValue) + return s.factory.store.Put(ck, encodedValue) } func (s *KeyedMapState[MK, MV]) Get(mapKey MK) (MV, bool, error) { @@ -107,7 +105,7 @@ func (s *KeyedMapState[MK, MV]) Get(mapKey MK) (MV, bool, error) { if err != nil { return zero, false, err } - raw, found, err := s.factory.inner.store.Get(ck) + raw, found, err := s.factory.store.Get(ck) if err != nil || !found { return zero, found, err } @@ -123,11 +121,11 @@ func (s *KeyedMapState[MK, MV]) Delete(mapKey MK) error { if err != nil { return err } - return s.factory.inner.store.Delete(ck) + return s.factory.store.Delete(ck) } func (s *KeyedMapState[MK, MV]) Clear() error { - return s.factory.inner.store.DeletePrefix(api.ComplexKey{ + return s.factory.store.DeletePrefix(api.ComplexKey{ KeyGroup: s.factory.groupKey, Key: s.primaryKey, Namespace: s.namespace, @@ -137,7 +135,7 @@ func (s *KeyedMapState[MK, MV]) Clear() error { func (s *KeyedMapState[MK, MV]) All() iter.Seq2[MK, MV] { return func(yield func(MK, MV) bool) { - iter, err := s.factory.inner.store.ScanComplex( + iter, err := s.factory.store.ScanComplex( s.factory.groupKey, s.primaryKey, s.namespace, diff --git a/go-sdk/state/keyed/keyed_priority_queue_state.go b/go-sdk/state/keyed/keyed_priority_queue_state.go index 0e524664..49273755 100644 --- a/go-sdk/state/keyed/keyed_priority_queue_state.go +++ b/go-sdk/state/keyed/keyed_priority_queue_state.go @@ -22,7 +22,7 @@ import ( ) type KeyedPriorityQueueStateFactory[V any] struct { - inner *keyedStateFactory + store common.Store groupKey []byte valueCodec codec.Codec[V] } @@ -33,11 +33,9 @@ func NewKeyedPriorityQueueStateFactory[V any]( valueCodec codec.Codec[V], ) (*KeyedPriorityQueueStateFactory[V], error) { - inner, err := newKeyedStateFactory(store, "", "pq") - if err != nil { - return nil, err + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed priority queue state factory store must not be nil") } - if keyGroup == nil { return nil, api.NewError(api.ErrStoreInternal, "keyed priority queue state factory key_group must not be nil") } @@ -50,7 +48,7 @@ func NewKeyedPriorityQueueStateFactory[V any]( } return &KeyedPriorityQueueStateFactory[V]{ - inner: inner, + store: store, groupKey: common.DupBytes(keyGroup), valueCodec: valueCodec, }, nil @@ -86,13 +84,13 @@ func (s *KeyedPriorityQueueState[V]) Add(value V) error { UserKey: userKey, } - return s.factory.inner.store.Put(ck, []byte{}) + return s.factory.store.Put(ck, []byte{}) } func (s *KeyedPriorityQueueState[V]) Peek() (V, bool, error) { var zero V - iter, err := s.factory.inner.store.ScanComplex( + iter, err := s.factory.store.ScanComplex( s.factory.groupKey, s.primaryKey, s.namespace, @@ -133,12 +131,12 @@ func (s *KeyedPriorityQueueState[V]) Poll() (V, bool, error) { UserKey: userKey, } - err = s.factory.inner.store.Delete(ck) + err = s.factory.store.Delete(ck) return val, true, err } func (s *KeyedPriorityQueueState[V]) Clear() error { - return s.factory.inner.store.DeletePrefix(api.ComplexKey{ + return s.factory.store.DeletePrefix(api.ComplexKey{ KeyGroup: s.factory.groupKey, Key: s.primaryKey, Namespace: s.namespace, @@ -148,7 +146,7 @@ func (s *KeyedPriorityQueueState[V]) Clear() error { func (s *KeyedPriorityQueueState[V]) All() iter.Seq[V] { return func(yield func(V) bool) { - iter, err := s.factory.inner.store.ScanComplex( + iter, err := s.factory.store.ScanComplex( s.factory.groupKey, s.primaryKey, s.namespace, diff --git a/go-sdk/state/keyed/keyed_reducing_state.go b/go-sdk/state/keyed/keyed_reducing_state.go index e608ee3d..45a854bf 100644 --- a/go-sdk/state/keyed/keyed_reducing_state.go +++ b/go-sdk/state/keyed/keyed_reducing_state.go @@ -23,7 +23,7 @@ import ( type ReduceFunc[V any] func(value1 V, value2 V) (V, error) type KeyedReducingStateFactory[V any] struct { - inner *keyedStateFactory + store common.Store groupKey []byte valueCodec codec.Codec[V] reduceFunc ReduceFunc[V] @@ -36,11 +36,9 @@ func NewKeyedReducingStateFactory[V any]( reduceFunc ReduceFunc[V], ) (*KeyedReducingStateFactory[V], error) { - inner, err := newKeyedStateFactory(store, "", "reducing") - if err != nil { - return nil, err + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed reducing state factory store must not be nil") } - if keyGroup == nil { return nil, api.NewError(api.ErrStoreInternal, "keyed reducing state factory key_group must not be nil") } @@ -49,7 +47,7 @@ func NewKeyedReducingStateFactory[V any]( } return &KeyedReducingStateFactory[V]{ - inner: inner, + store: store, groupKey: common.DupBytes(keyGroup), valueCodec: valueCodec, reduceFunc: reduceFunc, @@ -84,7 +82,7 @@ func (s *KeyedReducingState[V]) buildCK() api.ComplexKey { func (s *KeyedReducingState[V]) Add(value V) error { ck := s.buildCK() - raw, found, err := s.factory.inner.store.Get(ck) + raw, found, err := s.factory.store.Get(ck) if err != nil { return fmt.Errorf("failed to get old value for reducing state: %w", err) } @@ -109,13 +107,13 @@ func (s *KeyedReducingState[V]) Add(value V) error { return fmt.Errorf("failed to encode reduced value: %w", err) } - return s.factory.inner.store.Put(ck, encoded) + return s.factory.store.Put(ck, encoded) } func (s *KeyedReducingState[V]) Get() (V, bool, error) { var zero V ck := s.buildCK() - raw, found, err := s.factory.inner.store.Get(ck) + raw, found, err := s.factory.store.Get(ck) if err != nil || !found { return zero, found, err } @@ -127,5 +125,5 @@ func (s *KeyedReducingState[V]) Get() (V, bool, error) { } func (s *KeyedReducingState[V]) Clear() error { - return s.factory.inner.store.Delete(s.buildCK()) + return s.factory.store.Delete(s.buildCK()) } diff --git a/go-sdk/state/keyed/keyed_state_factory.go b/go-sdk/state/keyed/keyed_state_factory.go deleted file mode 100644 index acc601ec..00000000 --- a/go-sdk/state/keyed/keyed_state_factory.go +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package keyed - -import ( - "encoding/base64" - "fmt" - - "github.com/functionstream/function-stream/go-sdk/api" - "github.com/functionstream/function-stream/go-sdk/state/codec" - "github.com/functionstream/function-stream/go-sdk/state/common" -) - -type keyedStateFactory struct { - store common.Store - name string -} - -func newKeyedStateFactory(store common.Store, name string, kind string) (*keyedStateFactory, error) { - stateName, err := common.ValidateStateName(name) - if err != nil { - return nil, err - } - if store == nil { - return nil, api.NewError(api.ErrStoreInternal, "keyed %s state factory %q store must not be nil", kind, stateName) - } - return &keyedStateFactory{store: store, name: stateName}, nil -} - -func keyedSubStateName[K any](base string, keyCodec codec.Codec[K], key K, kind string) (string, error) { - if keyCodec == nil { - return "", api.NewError(api.ErrStoreInternal, "key codec must not be nil") - } - encoded, err := keyCodec.Encode(key) - if err != nil { - return "", fmt.Errorf("encode keyed state key failed: %w", err) - } - return fmt.Sprintf("%s/%s/%s", base, kind, base64.RawURLEncoding.EncodeToString(encoded)), nil -} diff --git a/go-sdk/state/keyed/keyed_value_state.go b/go-sdk/state/keyed/keyed_value_state.go index c181e992..b8bb9987 100644 --- a/go-sdk/state/keyed/keyed_value_state.go +++ b/go-sdk/state/keyed/keyed_value_state.go @@ -21,7 +21,7 @@ import ( ) type KeyedValueStateFactory[V any] struct { - inner *keyedStateFactory + store common.Store groupKey []byte valueCodec codec.Codec[V] } @@ -32,11 +32,9 @@ func NewKeyedValueStateFactory[V any]( valueCodec codec.Codec[V], ) (*KeyedValueStateFactory[V], error) { - inner, err := newKeyedStateFactory(store, "", "value") - if err != nil { - return nil, err + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed value state factory store must not be nil") } - if keyGroup == nil { return nil, api.NewError(api.ErrStoreInternal, "keyed value state factory key_group must not be nil") } @@ -45,7 +43,7 @@ func NewKeyedValueStateFactory[V any]( } return &KeyedValueStateFactory[V]{ - inner: inner, + store: store, groupKey: common.DupBytes(keyGroup), valueCodec: valueCodec, }, nil @@ -83,13 +81,13 @@ func (s *KeyedValueState[V]) Update(value V) error { if err != nil { return fmt.Errorf("encode value state failed: %w", err) } - return s.factory.inner.store.Put(ck, encoded) + return s.factory.store.Put(ck, encoded) } func (s *KeyedValueState[V]) Value() (V, bool, error) { var zero V ck := s.buildCK() - raw, found, err := s.factory.inner.store.Get(ck) + raw, found, err := s.factory.store.Get(ck) if err != nil || !found { return zero, found, err } @@ -101,5 +99,5 @@ func (s *KeyedValueState[V]) Value() (V, bool, error) { } func (s *KeyedValueState[V]) Clear() error { - return s.factory.inner.store.Delete(s.buildCK()) + return s.factory.store.Delete(s.buildCK()) } diff --git a/go-sdk/state/structures/map.go b/go-sdk/state/structures/map.go index a4b89352..ff19bacc 100644 --- a/go-sdk/state/structures/map.go +++ b/go-sdk/state/structures/map.go @@ -52,7 +52,7 @@ func NewMapState[K any, V any](store common.Store, keyCodec codec.Codec[K], valu } func NewMapStateAutoKeyCodec[K any, V any](store common.Store, valueCodec codec.Codec[V]) (*MapState[K, V], error) { - autoKeyCodec, err := inferOrderedKeyCodec[K]() + autoKeyCodec, err := codec.DefaultCodecFor[K]() if err != nil { return nil, err } @@ -99,101 +99,6 @@ func (m *MapState[K, V]) Delete(key K) error { return m.store.Delete(m.ck(encodedKey)) } -func (m *MapState[K, V]) Len() (uint64, error) { - iter, err := m.store.ScanComplex(m.keyGroup, m.key, m.namespace) - if err != nil { - return 0, err - } - defer iter.Close() - var count uint64 - for { - has, err := iter.HasNext() - if err != nil { - return 0, err - } - if !has { - return count, nil - } - _, _, ok, err := iter.Next() - if err != nil { - return 0, err - } - if !ok { - return count, nil - } - count++ - } -} - -func (m *MapState[K, V]) Entries() ([]MapEntry[K, V], error) { - iter, err := m.store.ScanComplex(m.keyGroup, m.key, m.namespace) - if err != nil { - return nil, err - } - defer iter.Close() - out := make([]MapEntry[K, V], 0, 16) - for { - has, err := iter.HasNext() - if err != nil { - return nil, err - } - if !has { - return out, nil - } - keyBytes, valueBytes, ok, err := iter.Next() - if err != nil { - return nil, err - } - if !ok { - return out, nil - } - decodedKey, err := m.keyCodec.Decode(keyBytes) - if err != nil { - return nil, fmt.Errorf("decode map key failed: %w", err) - } - decodedValue, err := m.valueCodec.Decode(valueBytes) - if err != nil { - return nil, fmt.Errorf("decode map value failed: %w", err) - } - out = append(out, MapEntry[K, V]{Key: decodedKey, Value: decodedValue}) - } -} - -func (m *MapState[K, V]) Range(startInclusive K, endExclusive K) ([]MapEntry[K, V], error) { - startBytes, err := m.keyCodec.Encode(startInclusive) - if err != nil { - return nil, fmt.Errorf("encode map start key failed: %w", err) - } - endBytes, err := m.keyCodec.Encode(endExclusive) - if err != nil { - return nil, fmt.Errorf("encode map end key failed: %w", err) - } - userKeys, err := m.store.ListComplex(m.keyGroup, m.key, m.namespace, startBytes, endBytes) - if err != nil { - return nil, err - } - out := make([]MapEntry[K, V], 0, len(userKeys)) - for _, userKey := range userKeys { - raw, found, err := m.store.Get(m.ck(userKey)) - if err != nil { - return nil, err - } - if !found { - return nil, api.NewError(api.ErrResultUnexpected, "map range key disappeared during scan") - } - decodedKey, err := m.keyCodec.Decode(userKey) - if err != nil { - return nil, fmt.Errorf("decode map key failed: %w", err) - } - decodedValue, err := m.valueCodec.Decode(raw) - if err != nil { - return nil, fmt.Errorf("decode map value failed: %w", err) - } - out = append(out, MapEntry[K, V]{Key: decodedKey, Value: decodedValue}) - } - return out, nil -} - func (m *MapState[K, V]) Clear() error { return m.store.DeletePrefix(api.ComplexKey{KeyGroup: m.keyGroup, Key: m.key, Namespace: m.namespace, UserKey: nil}) } @@ -229,33 +134,3 @@ func (m *MapState[K, V]) All() iter.Seq2[K, V] { func (m *MapState[K, V]) ck(userKey []byte) api.ComplexKey { return api.ComplexKey{KeyGroup: m.keyGroup, Key: m.key, Namespace: m.namespace, UserKey: userKey} } - -func inferOrderedKeyCodec[K any]() (codec.Codec[K], error) { - var zero K - switch any(zero).(type) { - case string: - return any(codec.StringCodec{}).(codec.Codec[K]), nil - case []byte: - return any(codec.BytesCodec{}).(codec.Codec[K]), nil - case bool: - return any(codec.BoolCodec{}).(codec.Codec[K]), nil - case int: - return any(codec.OrderedIntCodec{}).(codec.Codec[K]), nil - case uint: - return any(codec.OrderedUintCodec{}).(codec.Codec[K]), nil - case int32: - return any(codec.OrderedInt32Codec{}).(codec.Codec[K]), nil - case uint32: - return any(codec.OrderedUint32Codec{}).(codec.Codec[K]), nil - case int64: - return any(codec.OrderedInt64Codec{}).(codec.Codec[K]), nil - case uint64: - return any(codec.OrderedUint64Codec{}).(codec.Codec[K]), nil - case float32: - return any(codec.OrderedFloat32Codec{}).(codec.Codec[K]), nil - case float64: - return any(codec.OrderedFloat64Codec{}).(codec.Codec[K]), nil - default: - return nil, api.NewError(api.ErrStoreInternal, "unsupported map key type for auto codec") - } -} diff --git a/python/functionstream-api/src/fs_api/__init__.py b/python/functionstream-api/src/fs_api/__init__.py index 7c185282..c599933e 100644 --- a/python/functionstream-api/src/fs_api/__init__.py +++ b/python/functionstream-api/src/fs_api/__init__.py @@ -29,18 +29,8 @@ BytesCodec, StringCodec, BoolCodec, - Int64Codec, - Uint64Codec, - Int32Codec, - Uint32Codec, - Float64Codec, - Float32Codec, - OrderedInt64Codec, - OrderedUint64Codec, - OrderedInt32Codec, - OrderedUint32Codec, - OrderedFloat64Codec, - OrderedFloat32Codec, + IntCodec, + FloatCodec, ValueState, MapEntry, MapState, @@ -52,7 +42,6 @@ AggregatingState, ReduceFunc, ReducingState, - KeyedStateFactory, KeyedValueState, KeyedMapState, KeyedListState, @@ -77,18 +66,8 @@ "BytesCodec", "StringCodec", "BoolCodec", - "Int64Codec", - "Uint64Codec", - "Int32Codec", - "Uint32Codec", - "Float64Codec", - "Float32Codec", - "OrderedInt64Codec", - "OrderedUint64Codec", - "OrderedInt32Codec", - "OrderedUint32Codec", - "OrderedFloat64Codec", - "OrderedFloat32Codec", + "IntCodec", + "FloatCodec", "ValueState", "MapEntry", "MapState", @@ -100,7 +79,6 @@ "AggregatingState", "ReduceFunc", "ReducingState", - "KeyedStateFactory", "KeyedValueState", "KeyedMapState", "KeyedListState", diff --git a/python/functionstream-api/src/fs_api/store/__init__.py b/python/functionstream-api/src/fs_api/store/__init__.py index 1cf1eb8a..1c10d5bc 100644 --- a/python/functionstream-api/src/fs_api/store/__init__.py +++ b/python/functionstream-api/src/fs_api/store/__init__.py @@ -14,7 +14,6 @@ from .complexkey import ComplexKey from .iterator import KvIterator from .store import KvStore -from .common import StateKind from .codec import ( Codec, @@ -23,18 +22,8 @@ BytesCodec, StringCodec, BoolCodec, - Int64Codec, - Uint64Codec, - Int32Codec, - Uint32Codec, - Float64Codec, - Float32Codec, - OrderedInt64Codec, - OrderedUint64Codec, - OrderedInt32Codec, - OrderedUint32Codec, - OrderedFloat64Codec, - OrderedFloat32Codec, + IntCodec, + FloatCodec, default_codec_for, ) @@ -53,7 +42,6 @@ ) from .keyed import ( - KeyedStateFactory, KeyedListStateFactory, KeyedValueStateFactory, KeyedMapStateFactory, @@ -76,25 +64,14 @@ "ComplexKey", "KvIterator", "KvStore", - "StateKind", "Codec", "JsonCodec", "PickleCodec", "BytesCodec", "StringCodec", "BoolCodec", - "Int64Codec", - "Uint64Codec", - "Int32Codec", - "Uint32Codec", - "Float64Codec", - "Float32Codec", - "OrderedInt64Codec", - "OrderedUint64Codec", - "OrderedInt32Codec", - "OrderedUint32Codec", - "OrderedFloat64Codec", - "OrderedFloat32Codec", + "IntCodec", + "FloatCodec", "default_codec_for", "ValueState", "MapEntry", @@ -107,7 +84,6 @@ "AggregatingState", "ReduceFunc", "ReducingState", - "KeyedStateFactory", "KeyedListStateFactory", "KeyedValueStateFactory", "KeyedMapStateFactory", diff --git a/python/functionstream-api/src/fs_api/store/codec/__init__.py b/python/functionstream-api/src/fs_api/store/codec/__init__.py index 1438430f..3c3d4ba1 100644 --- a/python/functionstream-api/src/fs_api/store/codec/__init__.py +++ b/python/functionstream-api/src/fs_api/store/codec/__init__.py @@ -16,18 +16,8 @@ from .bytes_codec import BytesCodec from .string_codec import StringCodec from .bool_codec import BoolCodec -from .int64_codec import Int64Codec -from .uint64_codec import Uint64Codec -from .int32_codec import Int32Codec -from .uint32_codec import Uint32Codec -from .float64_codec import Float64Codec -from .float32_codec import Float32Codec -from .ordered_int64_codec import OrderedInt64Codec -from .ordered_uint64_codec import OrderedUint64Codec -from .ordered_int32_codec import OrderedInt32Codec -from .ordered_uint32_codec import OrderedUint32Codec -from .ordered_float64_codec import OrderedFloat64Codec -from .ordered_float32_codec import OrderedFloat32Codec +from .int_codec import IntCodec +from .float_codec import FloatCodec from .default_codec import default_codec_for __all__ = [ @@ -37,17 +27,7 @@ "BytesCodec", "StringCodec", "BoolCodec", - "Int64Codec", - "Uint64Codec", - "Int32Codec", - "Uint32Codec", - "Float64Codec", - "Float32Codec", - "OrderedInt64Codec", - "OrderedUint64Codec", - "OrderedInt32Codec", - "OrderedUint32Codec", - "OrderedFloat64Codec", - "OrderedFloat32Codec", + "IntCodec", + "FloatCodec", "default_codec_for", ] diff --git a/python/functionstream-api/src/fs_api/store/codec/default_codec.py b/python/functionstream-api/src/fs_api/store/codec/default_codec.py index edc10d2e..67aa17fd 100644 --- a/python/functionstream-api/src/fs_api/store/codec/default_codec.py +++ b/python/functionstream-api/src/fs_api/store/codec/default_codec.py @@ -15,13 +15,9 @@ from .base import Codec from .bool_codec import BoolCodec from .bytes_codec import BytesCodec +from .float_codec import FloatCodec +from .int_codec import IntCodec from .json_codec import JsonCodec -from .ordered_float32_codec import OrderedFloat32Codec -from .ordered_float64_codec import OrderedFloat64Codec -from .ordered_int32_codec import OrderedInt32Codec -from .ordered_int64_codec import OrderedInt64Codec -from .ordered_uint32_codec import OrderedUint32Codec -from .ordered_uint64_codec import OrderedUint64Codec from .pickle_codec import PickleCodec from .string_codec import StringCodec @@ -34,16 +30,16 @@ def default_codec_for(value_type: Type[Any]) -> Codec[Any]: if value_type is bool: return BoolCodec() if value_type is int: - return OrderedInt64Codec() + return IntCodec() if value_type is float: - return OrderedFloat64Codec() + return FloatCodec() if value_type is str: return StringCodec() if value_type is bytes: return BytesCodec() try: if issubclass(value_type, int) and value_type is not bool: - return OrderedInt64Codec() + return IntCodec() except TypeError: pass if value_type is list or value_type is dict: diff --git a/python/functionstream-api/src/fs_api/store/codec/float32_codec.py b/python/functionstream-api/src/fs_api/store/codec/float32_codec.py deleted file mode 100644 index 5dd6150f..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/float32_codec.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class Float32Codec(Codec[float]): - def encode(self, value: float) -> bytes: - return struct.pack(">f", value) - - def decode(self, data: bytes) -> float: - if len(data) != 4: - raise ValueError(f"invalid float32 payload length: {len(data)}") - return struct.unpack(">f", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/float64_codec.py b/python/functionstream-api/src/fs_api/store/codec/float64_codec.py deleted file mode 100644 index 42879eaf..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/float64_codec.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class Float64Codec(Codec[float]): - def encode(self, value: float) -> bytes: - return struct.pack(">d", value) - - def decode(self, data: bytes) -> float: - if len(data) != 8: - raise ValueError(f"invalid float64 payload length: {len(data)}") - return struct.unpack(">d", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py b/python/functionstream-api/src/fs_api/store/codec/float_codec.py similarity index 87% rename from python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py rename to python/functionstream-api/src/fs_api/store/codec/float_codec.py index 37e5100a..1e87c3d0 100644 --- a/python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py +++ b/python/functionstream-api/src/fs_api/store/codec/float_codec.py @@ -15,7 +15,8 @@ from .base import Codec -class OrderedFloat64Codec(Codec[float]): +class FloatCodec(Codec[float]): + """Ordered float codec for range scans (lexicographic byte order).""" supports_ordered_keys = True def encode(self, value: float) -> bytes: @@ -28,7 +29,7 @@ def encode(self, value: float) -> bytes: def decode(self, data: bytes) -> float: if len(data) != 8: - raise ValueError(f"invalid ordered float64 payload length: {len(data)}") + raise ValueError(f"invalid float payload length: {len(data)}") mapped = struct.unpack(">Q", data)[0] if mapped & (1 << 63): bits = mapped ^ (1 << 63) diff --git a/python/functionstream-api/src/fs_api/store/codec/int32_codec.py b/python/functionstream-api/src/fs_api/store/codec/int32_codec.py deleted file mode 100644 index eb850e7e..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/int32_codec.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class Int32Codec(Codec[int]): - def encode(self, value: int) -> bytes: - return struct.pack(">i", value) - - def decode(self, data: bytes) -> int: - if len(data) != 4: - raise ValueError(f"invalid int32 payload length: {len(data)}") - return struct.unpack(">i", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/int64_codec.py b/python/functionstream-api/src/fs_api/store/codec/int64_codec.py deleted file mode 100644 index 31208faa..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/int64_codec.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class Int64Codec(Codec[int]): - def encode(self, value: int) -> bytes: - return struct.pack(">q", value) - - def decode(self, data: bytes) -> int: - if len(data) != 8: - raise ValueError(f"invalid int64 payload length: {len(data)}") - return struct.unpack(">q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py b/python/functionstream-api/src/fs_api/store/codec/int_codec.py similarity index 85% rename from python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py rename to python/functionstream-api/src/fs_api/store/codec/int_codec.py index dadd9c68..712bb947 100644 --- a/python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py +++ b/python/functionstream-api/src/fs_api/store/codec/int_codec.py @@ -15,7 +15,8 @@ from .base import Codec -class OrderedInt64Codec(Codec[int]): +class IntCodec(Codec[int]): + """Ordered int codec for range scans (lexicographic byte order).""" supports_ordered_keys = True def encode(self, value: int) -> bytes: @@ -24,7 +25,7 @@ def encode(self, value: int) -> bytes: def decode(self, data: bytes) -> int: if len(data) != 8: - raise ValueError(f"invalid ordered int64 payload length: {len(data)}") + raise ValueError(f"invalid int payload length: {len(data)}") mapped = struct.unpack(">Q", data)[0] raw = mapped ^ (1 << 63) if raw >= (1 << 63): diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py deleted file mode 100644 index 0dced99a..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class OrderedFloat32Codec(Codec[float]): - supports_ordered_keys = True - - def encode(self, value: float) -> bytes: - bits = struct.unpack(">I", struct.pack(">f", value))[0] - if bits & (1 << 31): - mapped = (~bits) & 0xFFFFFFFF - else: - mapped = bits ^ (1 << 31) - return struct.pack(">I", mapped) - - def decode(self, data: bytes) -> float: - if len(data) != 4: - raise ValueError(f"invalid ordered float32 payload length: {len(data)}") - mapped = struct.unpack(">I", data)[0] - if mapped & (1 << 31): - bits = mapped ^ (1 << 31) - else: - bits = (~mapped) & 0xFFFFFFFF - return struct.unpack(">f", struct.pack(">I", bits))[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py deleted file mode 100644 index 9787bf6c..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class OrderedInt32Codec(Codec[int]): - supports_ordered_keys = True - - def encode(self, value: int) -> bytes: - mapped = (value & 0xFFFFFFFF) ^ (1 << 31) - return struct.pack(">I", mapped) - - def decode(self, data: bytes) -> int: - if len(data) != 4: - raise ValueError(f"invalid ordered int32 payload length: {len(data)}") - mapped = struct.unpack(">I", data)[0] - raw = mapped ^ (1 << 31) - if raw >= (1 << 31): - return raw - (1 << 32) - return raw diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py deleted file mode 100644 index 9f1ac2e1..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class OrderedUint32Codec(Codec[int]): - supports_ordered_keys = True - - def encode(self, value: int) -> bytes: - if value < 0: - raise ValueError("ordered uint32 value must be >= 0") - return struct.pack(">I", value) - - def decode(self, data: bytes) -> int: - if len(data) != 4: - raise ValueError(f"invalid ordered uint32 payload length: {len(data)}") - return struct.unpack(">I", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py deleted file mode 100644 index dab4a927..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class OrderedUint64Codec(Codec[int]): - supports_ordered_keys = True - - def encode(self, value: int) -> bytes: - if value < 0: - raise ValueError("ordered uint64 value must be >= 0") - return struct.pack(">Q", value) - - def decode(self, data: bytes) -> int: - if len(data) != 8: - raise ValueError(f"invalid ordered uint64 payload length: {len(data)}") - return struct.unpack(">Q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py b/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py deleted file mode 100644 index 281531d1..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class Uint32Codec(Codec[int]): - def encode(self, value: int) -> bytes: - if value < 0: - raise ValueError("uint32 value must be >= 0") - return struct.pack(">I", value) - - def decode(self, data: bytes) -> int: - if len(data) != 4: - raise ValueError(f"invalid uint32 payload length: {len(data)}") - return struct.unpack(">I", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py b/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py deleted file mode 100644 index 634d7dcb..00000000 --- a/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct - -from .base import Codec - - -class Uint64Codec(Codec[int]): - def encode(self, value: int) -> bytes: - if value < 0: - raise ValueError("uint64 value must be >= 0") - return struct.pack(">Q", value) - - def decode(self, data: bytes) -> int: - if len(data) != 8: - raise ValueError(f"invalid uint64 payload length: {len(data)}") - return struct.unpack(">Q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/common/__init__.py b/python/functionstream-api/src/fs_api/store/common/__init__.py deleted file mode 100644 index 4056a4e2..00000000 --- a/python/functionstream-api/src/fs_api/store/common/__init__.py +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import struct -from enum import IntEnum -from typing import Tuple - -from ..error import KvError - - -class StateKind(IntEnum): - VALUE = 0 - LIST = 1 - PRIORITY_QUEUE = 2 - MAP = 3 - AGGREGATING = 4 - REDUCING = 5 - - def prefix(self) -> bytes: - if self == StateKind.VALUE: - return b"__fssdk__/value/" - if self == StateKind.LIST: - return b"__fssdk__/list/" - if self == StateKind.PRIORITY_QUEUE: - return b"__fssdk__/priority_queue/" - if self == StateKind.MAP: - return b"" - if self == StateKind.AGGREGATING: - return b"__fssdk__/aggregating/" - if self == StateKind.REDUCING: - return b"__fssdk__/reducing/" - return b"" - - def group(self) -> bytes: - if self in (StateKind.VALUE, StateKind.AGGREGATING, StateKind.REDUCING): - return b"" - if self == StateKind.LIST: - return b"__fssdk__/list" - if self == StateKind.PRIORITY_QUEUE: - return b"__fssdk__/priority_queue" - if self == StateKind.MAP: - return b"__fssdk__/map" - return b"" - - -def validate_state_name(name: str) -> None: - if not isinstance(name, str) or not name.strip(): - raise KvError("state name must be a non-empty string") - - -def encode_int64_lex(value: int) -> bytes: - mapped = (value & 0xFFFFFFFFFFFFFFFF) ^ (1 << 63) - return struct.pack(">Q", mapped) - - -def encode_priority_key(priority: int, seq: int) -> bytes: - return encode_int64_lex(priority) + struct.pack(">Q", seq) - - -def decode_priority_key(data: bytes) -> Tuple[int, int]: - if len(data) != 16: - raise KvError("invalid priority queue key length") - mapped_priority = struct.unpack(">Q", data[:8])[0] - unsigned_priority = mapped_priority ^ (1 << 63) - if unsigned_priority >= (1 << 63): - priority = unsigned_priority - (1 << 64) - else: - priority = unsigned_priority - seq = struct.unpack(">Q", data[8:])[0] - return priority, seq - - -__all__ = [ - "StateKind", - "validate_state_name", - "encode_int64_lex", - "encode_priority_key", - "decode_priority_key", -] diff --git a/python/functionstream-api/src/fs_api/store/keyed/__init__.py b/python/functionstream-api/src/fs_api/store/keyed/__init__.py index f12cdfbb..31e274f5 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/__init__.py +++ b/python/functionstream-api/src/fs_api/store/keyed/__init__.py @@ -10,7 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .keyed_state_factory import KeyedStateFactory from .keyed_value_state import KeyedValueState, KeyedValueStateFactory from .keyed_list_state import KeyedListState, KeyedListStateFactory from .keyed_map_state import KeyedMapEntry, KeyedMapState, KeyedMapStateFactory @@ -19,7 +18,6 @@ from .keyed_reducing_state import KeyedReducingState, KeyedReducingStateFactory, ReduceFunc __all__ = [ - "KeyedStateFactory", "KeyedListStateFactory", "KeyedValueStateFactory", "KeyedMapStateFactory", diff --git a/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py b/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py index 81dc850f..106a2cc3 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py +++ b/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py @@ -14,14 +14,6 @@ from ..error import KvError -KEYED_VALUE_GROUP = b"__fssdk__/keyed/value" -KEYED_LIST_GROUP = b"__fssdk__/keyed/list" -KEYED_MAP_GROUP = b"__fssdk__/keyed/map" -KEYED_PQ_GROUP = b"__fssdk__/keyed/pq" -KEYED_AGGREGATING_GROUP = b"__fssdk__/keyed/aggregating" -KEYED_REDUCING_GROUP = b"__fssdk__/keyed/reducing" - - def ensure_ordered_key_codec(codec: Any, label: str) -> None: if not getattr(codec, "supports_ordered_keys", False): raise KvError(f"{label} key codec must support ordered key encoding") @@ -29,10 +21,4 @@ def ensure_ordered_key_codec(codec: Any, label: str) -> None: __all__ = [ "ensure_ordered_key_codec", - "KEYED_VALUE_GROUP", - "KEYED_LIST_GROUP", - "KEYED_MAP_GROUP", - "KEYED_PQ_GROUP", - "KEYED_AGGREGATING_GROUP", - "KEYED_REDUCING_GROUP", ] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py index c0e7173a..499d969e 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py @@ -17,7 +17,7 @@ from ..error import KvError from ..store import KvStore -from ._keyed_common import KEYED_AGGREGATING_GROUP, ensure_ordered_key_codec +from ._keyed_common import ensure_ordered_key_codec K = TypeVar("K") T_agg = TypeVar("T_agg") diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py index ee49d213..72d12d4f 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py @@ -18,7 +18,7 @@ from ..error import KvError from ..store import KvStore -from ._keyed_common import KEYED_LIST_GROUP, ensure_ordered_key_codec +from ._keyed_common import ensure_ordered_key_codec K = TypeVar("K") V = TypeVar("V") diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py index 0234e002..097e5551 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py @@ -11,14 +11,14 @@ # limitations under the License. from dataclasses import dataclass -from typing import Generic, List, Optional, TypeVar +from typing import Generic, Iterator, List, Optional, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError from ..store import KvStore -from ._keyed_common import KEYED_MAP_GROUP, ensure_ordered_key_codec +from ._keyed_common import ensure_ordered_key_codec K = TypeVar("K") MK = TypeVar("MK") @@ -131,7 +131,7 @@ def clear(self, key: K) -> None: ) self._store.delete_prefix(prefix_ck) - def all(self, key: K): + def all(self, key: K) -> Iterator[Tuple[MK, MV]]: it = self._store.scan_complex( self._key_group, self._key_codec.encode(key), @@ -144,50 +144,5 @@ def all(self, key: K): key_bytes, value_bytes = item yield self._map_key_codec.decode(key_bytes), self._map_value_codec.decode(value_bytes) - def len(self, key: K) -> int: - it = self._store.scan_complex( - self._key_group, - self._key_codec.encode(key), - self._namespace, - ) - n = 0 - while it.has_next(): - it.next() - n += 1 - return n - - def entries(self, key: K) -> List[KeyedMapEntry[MK, MV]]: - return [ - KeyedMapEntry(key=k, value=v) - for k, v in self.all(key) - ] - - def range( - self, key: K, start_inclusive: MK, end_exclusive: MK - ) -> List[KeyedMapEntry[MK, MV]]: - key_bytes = self._key_codec.encode(key) - start_bytes = self._map_key_codec.encode(start_inclusive) - end_bytes = self._map_key_codec.encode(end_exclusive) - user_keys = self._store.list_complex( - self._key_group, key_bytes, self._namespace, - start_bytes, end_bytes, - ) - out = [] - for uk in user_keys: - ck = ComplexKey( - key_group=self._key_group, - key=key_bytes, - namespace=self._namespace, - user_key=uk, - ) - raw = self._store.get(ck) - if raw is None: - raise KvError("map range key disappeared during scan") - out.append(KeyedMapEntry( - key=self._map_key_codec.decode(uk), - value=self._map_value_codec.decode(raw), - )) - return out - __all__ = ["KeyedMapEntry", "KeyedMapState", "KeyedMapStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py index 1bd3857a..70aba4c8 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py @@ -17,7 +17,7 @@ from ..error import KvError from ..store import KvStore -from ._keyed_common import KEYED_PQ_GROUP, ensure_ordered_key_codec +from ._keyed_common import ensure_ordered_key_codec K = TypeVar("K") V = TypeVar("V") diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py index c9c176c0..8813815c 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py @@ -17,7 +17,7 @@ from ..error import KvError from ..store import KvStore -from ._keyed_common import KEYED_REDUCING_GROUP, ensure_ordered_key_codec +from ._keyed_common import ensure_ordered_key_codec K = TypeVar("K") V = TypeVar("V") diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py deleted file mode 100644 index ebbfa1b8..00000000 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py +++ /dev/null @@ -1,113 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import TypeVar - -from ..codec import Codec -from ..error import KvError -from ..store import KvStore - -from .keyed_aggregating_state import AggregateFunc, KeyedAggregatingState -from .keyed_list_state import KeyedListState -from .keyed_map_state import KeyedMapState -from .keyed_priority_queue_state import KeyedPriorityQueueState -from .keyed_reducing_state import KeyedReducingState, ReduceFunc -from .keyed_value_state import KeyedValueState - -K = TypeVar("K") -V = TypeVar("V") -MK = TypeVar("MK") -MV = TypeVar("MV") -T_agg = TypeVar("T_agg") -ACC = TypeVar("ACC") -R = TypeVar("R") - - -class KeyedStateFactory: - def __init__(self, store: KvStore, namespace: bytes, key_group: bytes): - if store is None: - raise KvError("keyed state factory store must not be None") - if namespace is None: - raise KvError("keyed state factory namespace must not be None") - if key_group is None: - raise KvError("keyed state factory key_group must not be None") - self._store = store - self._namespace = namespace - self._key_group = key_group - self._kind = None - - def new_keyed_value(self, key_codec: Codec[K], value_codec: Codec[V]) -> KeyedValueState[K, V]: - self._claim_kind("value") - if value_codec is None: - raise KvError("keyed value state value_codec must not be None") - return KeyedValueState(self._store, self._namespace, key_codec, value_codec, self._key_group) - - def new_keyed_map( - self, key_codec: Codec[K], map_key_codec: Codec[MK], map_value_codec: Codec[MV] - ) -> KeyedMapState[K, MK, MV]: - self._claim_kind("map") - if map_key_codec is None or map_value_codec is None: - raise KvError("keyed map state map_key_codec and map_value_codec must not be None") - return KeyedMapState( - self._store, self._namespace, key_codec, map_key_codec, map_value_codec, self._key_group - ) - - def new_keyed_list(self, key_codec: Codec[K], value_codec: Codec[V]) -> KeyedListState[K, V]: - self._claim_kind("list") - if value_codec is None: - raise KvError("keyed list state value_codec must not be None") - return KeyedListState(self._store, self._namespace, key_codec, value_codec, self._key_group) - - def new_keyed_priority_queue( - self, key_codec: Codec[K], value_codec: Codec[V] - ) -> KeyedPriorityQueueState[K, V]: - self._claim_kind("priority_queue") - if value_codec is None: - raise KvError("keyed priority queue state value_codec must not be None") - return KeyedPriorityQueueState( - self._store, self._namespace, key_codec, value_codec, self._key_group - ) - - def new_keyed_aggregating( - self, - key_codec: Codec[K], - acc_codec: Codec[ACC], - agg_func: AggregateFunc[T_agg, ACC, R], - ) -> KeyedAggregatingState[K, T_agg, ACC, R]: - self._claim_kind("aggregating") - if acc_codec is None or agg_func is None: - raise KvError("keyed aggregating state acc_codec and agg_func must not be None") - return KeyedAggregatingState( - self._store, self._namespace, key_codec, acc_codec, agg_func, self._key_group - ) - - def new_keyed_reducing( - self, key_codec: Codec[K], value_codec: Codec[V], reduce_func: ReduceFunc[V] - ) -> KeyedReducingState[K, V]: - self._claim_kind("reducing") - if value_codec is None or reduce_func is None: - raise KvError("keyed reducing state value_codec and reduce_func must not be None") - return KeyedReducingState( - self._store, self._namespace, key_codec, value_codec, reduce_func, self._key_group - ) - - def _claim_kind(self, kind: str) -> None: - if self._kind is None: - self._kind = kind - return - if self._kind != kind: - raise KvError( - f"keyed state factory already bound to '{self._kind}', cannot create '{kind}'" - ) - - -__all__ = ["KeyedStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py index 334c4337..bf8f41d0 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py @@ -17,8 +17,6 @@ from ..error import KvError from ..store import KvStore -from ._keyed_common import KEYED_VALUE_GROUP, ensure_ordered_key_codec - K = TypeVar("K") V = TypeVar("V") diff --git a/python/functionstream-api/src/fs_api/store/structures/map_state.py b/python/functionstream-api/src/fs_api/store/structures/map_state.py index a1ceb8ee..4d3b9c59 100644 --- a/python/functionstream-api/src/fs_api/store/structures/map_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/map_state.py @@ -11,15 +11,11 @@ # limitations under the License. from dataclasses import dataclass -from typing import Any, Generator, Generic, List, Optional, Tuple, Type, TypeVar +from typing import Any, Generic, Iterator, Optional, Tuple, Type, TypeVar from ..codec import ( - BoolCodec, - BytesCodec, Codec, - OrderedFloat64Codec, - OrderedInt64Codec, - StringCodec, + default_codec_for, ) from ..complexkey import ComplexKey from ..error import KvError @@ -57,7 +53,7 @@ def with_auto_key_codec( key_type: Type[K], value_codec: Codec[V], ) -> "MapState[K, V]": - key_codec = infer_ordered_key_codec(key_type) + key_codec = default_codec_for(key_type) return cls(store, key_codec, value_codec) def put(self, key: K, value: V) -> None: @@ -79,7 +75,7 @@ def delete(self, key: K) -> None: def clear(self) -> None: self._store.delete_prefix(self._ck(b"")) - def all(self) -> Generator[Tuple[K, V], None, None]: + def all(self) -> Iterator[Tuple[K, V]]: it = self._store.scan_complex(self._key_group, self._key, self._namespace) while it.has_next(): item = it.next() @@ -88,49 +84,6 @@ def all(self) -> Generator[Tuple[K, V], None, None]: key_bytes, value_bytes = item yield self._key_codec.decode(key_bytes), self._value_codec.decode(value_bytes) - def len(self) -> int: - it = self._store.scan_complex(self._key_group, self._key, self._namespace) - size = 0 - while it.has_next(): - item = it.next() - if item is None: - break - size += 1 - return size - - def entries(self) -> List[MapEntry[K, V]]: - it = self._store.scan_complex(self._key_group, self._key, self._namespace) - out: List[MapEntry[K, V]] = [] - while it.has_next(): - item = it.next() - if item is None: - break - key_bytes, value_bytes = item - out.append( - MapEntry( - key=self._key_codec.decode(key_bytes), - value=self._value_codec.decode(value_bytes), - ) - ) - return out - - def range(self, start_inclusive: K, end_exclusive: K) -> List[MapEntry[K, V]]: - start_bytes = self._key_codec.encode(start_inclusive) - end_bytes = self._key_codec.encode(end_exclusive) - user_keys = self._store.list_complex(self._key_group, self._key, self._namespace, start_bytes, end_bytes) - out: List[MapEntry[K, V]] = [] - for user_key in user_keys: - raw = self._store.get(self._ck(user_key)) - if raw is None: - raise KvError("map range key disappeared during scan") - out.append( - MapEntry( - key=self._key_codec.decode(user_key), - value=self._value_codec.decode(raw), - ) - ) - return out - def _ck(self, user_key: bytes) -> ComplexKey: return ComplexKey( key_group=self._key_group, @@ -140,20 +93,6 @@ def _ck(self, user_key: bytes) -> ComplexKey: ) -def infer_ordered_key_codec(key_type: Type[Any]) -> Codec[Any]: - if key_type is str: - return StringCodec() - if key_type is bytes: - return BytesCodec() - if key_type is bool: - return BoolCodec() - if key_type is int: - return OrderedInt64Codec() - if key_type is float: - return OrderedFloat64Codec() - raise KvError("unsupported map key type for auto codec") - - def create_map_state_auto_key_codec( store: KvStore, key_type: Type[K], @@ -162,4 +101,9 @@ def create_map_state_auto_key_codec( return MapState.with_auto_key_codec(store, key_type, value_codec) -__all__ = ["MapEntry", "MapState", "infer_ordered_key_codec", "create_map_state_auto_key_codec"] +def infer_ordered_key_codec(key_type: Type[Any]) -> Codec[Any]: + """Return an ordered key codec for the given key type (uses default_codec_for).""" + return default_codec_for(key_type) + + +__all__ = ["MapEntry", "MapState", "create_map_state_auto_key_codec", "infer_ordered_key_codec"] diff --git a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py index ee86e62e..fe900983 100644 --- a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py +++ b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py @@ -30,7 +30,7 @@ KeyedReducingStateFactory, PickleCodec, BytesCodec, - OrderedInt64Codec, + IntCodec, default_codec_for, ) @@ -109,7 +109,7 @@ def getOrCreatePriorityQueueState(self, store_name: str, codec: Codec) -> Priori def getOrCreatePriorityQueueStateAutoCodec(self, store_name: str) -> PriorityQueueState: store = self.getOrCreateKVStore(store_name) - return PriorityQueueState(store, OrderedInt64Codec()) + return PriorityQueueState(store, IntCodec()) def getOrCreateAggregatingState( self, store_name: str, acc_codec: Codec, agg_func: object @@ -183,7 +183,7 @@ def getOrCreateKeyedPriorityQueueStateFactoryAutoCodec( self, store_name: str, namespace: bytes, key_group: bytes, item_type=None ) -> KeyedPriorityQueueStateFactory: store = self.getOrCreateKVStore(store_name) - codec = default_codec_for(item_type) if item_type is not None else OrderedInt64Codec() + codec = default_codec_for(item_type) if item_type is not None else IntCodec() return KeyedPriorityQueueStateFactory(store, namespace, key_group, codec) def getOrCreateKeyedAggregatingStateFactory( From 4fe95dd3feda5980a0997d880d0b57ea8a72765b Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 11 Mar 2026 13:26:54 +0800 Subject: [PATCH 6/9] update --- README-zh.md | 3 +- README.md | 3 +- docs/Go-SDK/go-sdk-guide-zh.md | 553 ++++++++++++++++++ docs/Go-SDK/go-sdk-guide.md | 553 ++++++++++++++++++ docs/{ => Python-SDK}/python-sdk-guide-zh.md | 0 docs/{ => Python-SDK}/python-sdk-guide.md | 0 docs/go-sdk-guide-zh.md | 324 ---------- docs/go-sdk-guide.md | 324 ---------- go-sdk/impl/state.go | 144 ----- go-sdk/state/keyed/keyed_aggregating_state.go | 24 +- go-sdk/state/keyed/keyed_list_state.go | 24 +- go-sdk/state/keyed/keyed_map_state.go | 28 +- .../state/keyed/keyed_priority_queue_state.go | 24 +- go-sdk/state/keyed/keyed_reducing_state.go | 24 +- go-sdk/state/keyed/keyed_value_state.go | 24 +- go-sdk/state/structures/aggregating.go | 33 +- go-sdk/state/structures/list.go | 24 +- go-sdk/state/structures/map.go | 41 +- go-sdk/state/structures/priority_queue.go | 26 +- go-sdk/state/structures/reducing.go | 33 +- go-sdk/state/structures/value.go | 24 +- 21 files changed, 1422 insertions(+), 811 deletions(-) create mode 100644 docs/Go-SDK/go-sdk-guide-zh.md create mode 100644 docs/Go-SDK/go-sdk-guide.md rename docs/{ => Python-SDK}/python-sdk-guide-zh.md (100%) rename docs/{ => Python-SDK}/python-sdk-guide.md (100%) delete mode 100644 docs/go-sdk-guide-zh.md delete mode 100644 docs/go-sdk-guide.md delete mode 100644 go-sdk/impl/state.go diff --git a/README-zh.md b/README-zh.md index 2f444484..b1d68eac 100644 --- a/README-zh.md +++ b/README-zh.md @@ -206,7 +206,8 @@ function-stream-/ | [Function 任务配置规范](docs/function-configuration-zh.md) | 任务定义规范 | | [SQL CLI 交互式管理指南](docs/sql-cli-guide-zh.md) | 交互式管理指南 | | [Function 管理与开发指南](docs/function-development-zh.md) | 管理与开发指南 | -| [Python SDK 开发与交互指南](docs/python-sdk-guide-zh.md) | Python SDK 指南 | +| [Go SDK 开发与交互指南](docs/Go-SDK/go-sdk-guide-zh.md) | Go SDK 指南 | +| [Python SDK 开发与交互指南](docs/Python-SDK/python-sdk-guide-zh.md) | Python SDK 指南 | ## 配置 diff --git a/README.md b/README.md index ed08e9fb..51a69de1 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,8 @@ We provide a robust shell script to manage the server process, capable of handli | [Function Configuration](docs/function-configuration.md) | Task Definition Specification | | [SQL CLI Guide](docs/sql-cli-guide.md) | Interactive Management Guide | | [Function Development](docs/function-development.md) | Management & Development Guide | -| [Python SDK Guide](docs/python-sdk-guide.md) | Python SDK Guide | +| [Go SDK Guide](docs/Go-SDK/go-sdk-guide.md) | Go SDK Guide | +| [Python SDK Guide](docs/Python-SDK/python-sdk-guide.md) | Python SDK Guide | ## Configuration diff --git a/docs/Go-SDK/go-sdk-guide-zh.md b/docs/Go-SDK/go-sdk-guide-zh.md new file mode 100644 index 00000000..8b2d6d52 --- /dev/null +++ b/docs/Go-SDK/go-sdk-guide-zh.md @@ -0,0 +1,553 @@ + + +# Go SDK 指南 + +Function Stream 为 Go 开发者提供基于 WebAssembly（WASI P2）的算子开发 SDK。使用 TinyGo 将 Go 代码编译为 WASM 组件，在服务端沙箱中运行，具备 KV 状态存储与数据发射能力。 + +--- + +## 一、SDK 核心组件 + +| 组件 | 定位 | 说明 | +|--------------|--------|--------------------------------------------------------------| +| **fssdk** | 主包 | 对外入口，提供 `Driver`、`Context`、`Store` 等类型及 `Run(driver)`。 | +| **api** | 接口定义 | 定义 `Driver`、`Context`、`Store`、`Iterator`、`ComplexKey` 及错误码。 | +| **impl** | 运行时实现 | 将 Driver 与 WASM 宿主（processor WIT）桥接，内部使用。 | +| **bindings** | WIT 绑定 | 由 wit-bindgen-go 根据 `wit/processor.wit` 生成的 Go 绑定，供 impl 调用。 | + +Go 算子**仅依赖 fssdk**：实现 `Driver`（或嵌入 `BaseDriver`），在 `init()` 中调用 `fssdk.Run(&YourProcessor{})` 即可。 + +--- + +## 二、Driver 接口与 BaseDriver + +### 2.1 Driver 接口 + +所有 Go 算子必须实现 `fssdk.Driver` 接口。运行时在对应时机调用以下方法： + +| 方法 | 触发时机 | 说明 | +|----------------------------------------------|-------------|-------------------------------------| +| `Init(ctx, config)` | 函数启动时执行一次 | 初始化状态、获取 Store、解析 config。 | +| `Process(ctx, sourceID, data)` | 每收到一条消息时 | 核心处理逻辑：计算、状态读写、`ctx.Emit()`。 | +| `ProcessWatermark(ctx, sourceID, watermark)` | 收到水位线事件时 | 处理基于时间的窗口或乱序重排，可转发 `EmitWatermark`。 | +| `TakeCheckpoint(ctx, checkpointID)` | 系统做状态备份时 | 可持久化额外内存状态，保证强一致性。 | +| `CheckHeartbeat(ctx)` | 定期健康检查 | 返回 `false` 会触发算子重启。 | +| `Close(ctx)` | 函数关闭时 | 释放资源、清空引用。 | +| `Exec(ctx, className, modules)` | 扩展能力（可选） | 动态加载模块等，默认可不实现。 | +| `Custom(ctx, payload)` | 自定义 RPC（可选） | 请求/响应自定义字节，默认返回 payload 副本。 | + +### 2.2 BaseDriver + +`fssdk.BaseDriver` 为上述所有方法提供空实现。嵌入后只需重写你关心的方法： + +```go +type MyProcessor struct { + fssdk.BaseDriver + store fssdk.Store +} + +func (p *MyProcessor) Init(ctx fssdk.Context, config map[string]string) error { + store, err := ctx.GetOrCreateStore("my-store") + if err != nil { + return err + } + p.store = store + return nil +} + +func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + // 业务逻辑，读写 p.store，最后 ctx.Emit(...) + return ctx.Emit(0, result) +} +``` + +### 2.3 注册入口 + +在 `init()` 中调用 `fssdk.Run`，传入你的 Driver 实例（通常为单例）： + +```go +func init() { + fssdk.Run(&MyProcessor{}) +} +``` + +--- + +## 三、Context 与 Store + +### 3.1 Context + +`fssdk.Context` 在 Init / Process / ProcessWatermark 等回调中传入，提供： + +| 方法 | 说明 | +|----------------------------------------------------------|---------------------------------------| +| `Emit(targetID uint32, data []byte) error` | 将数据发往指定输出通道。 | +| `EmitWatermark(targetID uint32, watermark uint64) error` | 发射水位线。 | +| `GetOrCreateStore(name string) (Store, error)` | 按名称获取或创建 KV Store（基于 RocksDB）。 | +| `Config() map[string]string` | 获取启动时下发的配置（对应 config.yaml 中的 init 等）。 | +| `Close() error` | 关闭 Context，一般由运行时管理。 | + +### 3.2 Store（KV 状态） + +`fssdk.Store` 提供键值存储与复杂键能力： + +**基础 API：** + +- `PutState(key, value []byte) error` +- `GetState(key []byte) (value []byte, found bool, err error)` +- `DeleteState(key []byte) error` +- `ListStates(startInclusive, endExclusive []byte) ([][]byte, error)`：按字节序范围列出键。 + +**ComplexKey（多维键/前缀扫描）：** + +- `Put(key ComplexKey, value []byte)` / `Get` / `Delete` / `Merge` / `DeletePrefix` +- `ListComplex(...)`：按 keyGroup、key、namespace 及范围列出 UserKey。 +- `ScanComplex(keyGroup, key, namespace []byte) (Iterator, error)`：返回迭代器，适用大范围扫描。 + +`ComplexKey` 结构包含 `KeyGroup`、`Key`、`Namespace`、`UserKey`，用于多维索引与前缀查询。 + +### 3.3 Iterator + +`Store.ScanComplex` 返回的 `fssdk.Iterator` 接口： + +- `HasNext() (bool, error)` +- `Next() (key, value []byte, ok bool, err error)` +- `Close() error`（用毕须调用以释放资源） + +--- + +## 四、生产级示例（词频统计） + +```go +package main + +import ( + "encoding/json" + fssdk "github.com/functionstream/function-stream/go-sdk" + "strconv" + "strings" +) + +func init() { + fssdk.Run(&CounterProcessor{}) +} + +type CounterProcessor struct { + fssdk.BaseDriver + store fssdk.Store + counterMap map[string]int64 + totalProcessed int64 + keyPrefix string +} + +func (p *CounterProcessor) Init(ctx fssdk.Context, config map[string]string) error { + store, err := ctx.GetOrCreateStore("counter-store") + if err != nil { + return err + } + p.store = store + p.counterMap = make(map[string]int64) + p.totalProcessed = 0 + p.keyPrefix = strings.TrimSpace(config["key_prefix"]) + return nil +} + +func (p *CounterProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + inputStr := strings.TrimSpace(string(data)) + if inputStr == "" { + return nil + } + p.totalProcessed++ + + fullKey := p.keyPrefix + inputStr + existing, found, err := p.store.GetState([]byte(fullKey)) + if err != nil { + return err + } + currentCount := int64(0) + if found { + if n, e := strconv.ParseInt(string(existing), 10, 64); e == nil { + currentCount = n + } + } + newCount := currentCount + 1 + p.counterMap[inputStr] = newCount + if err = p.store.PutState([]byte(fullKey), []byte(strconv.FormatInt(newCount, 10))); err != nil { + return err + } + + out := map[string]interface{}{ + "total_processed": p.totalProcessed, + "counter_map": p.counterMap, + } + jsonBytes, _ := json.Marshal(out) + return ctx.Emit(0, jsonBytes) +} + +func (p *CounterProcessor) ProcessWatermark(ctx fssdk.Context, sourceID uint32, watermark uint64) error { + return ctx.EmitWatermark(0, watermark) +} + +func (p *CounterProcessor) Close(ctx fssdk.Context) error { + p.store = nil + p.counterMap = nil + return nil +} + +func main() {} +``` + +--- + +## 五、构建与部署 + +### 5.1 环境要求 + +- **Go**：1.23+ +- **TinyGo**：0.40+（支持 WASI P2） +- **wit-bindgen-go**：`go install go.bytecodealliance.org/cmd/wit-bindgen-go@latest` +- **wkg**（可选）：用于拉取 WIT 依赖，`cargo install wkg --version 0.10.0` +- **wasm-tools**：用于组件校验等，`cargo install wasm-tools` + +### 5.2 生成绑定并构建 + +在项目根目录下： + +```bash +# 安装 Go SDK 工具链（wit-bindgen-go、wkg 等） +make -C go-sdk env + +# 生成 WIT 绑定（从 wit/processor.wit 及 deps） +make -C go-sdk bindings + +# 运行测试 +make -C go-sdk build +``` + +算子项目（如 `examples/go-processor`）中： + +```bash +# 使用 TinyGo 编译为 WASI P2 组件 +tinygo build -o build/processor.wasm -target=wasi . +``` + +具体参数见 `examples/go-processor/build.sh`。 + +### 5.3 注册与运行 + +将编译得到的 `processor.wasm` 与 `config.yaml` 通过 SQL CLI 注册为 function： + +```sql +create function with ( + 'function_path'='/path/to/build/processor.wasm', + 'config_path'='/path/to/config.yaml' +); +``` + +config.yaml 中需配置 `name`、`type: processor`、`input-groups`、`outputs`（如 Kafka）。详见 [Function 配置](../function-configuration-zh.md) 与 [examples/go-processor/README.md](../../examples/go-processor/README.md)。 + +--- + +## 六、错误码与异常处理 + +SDK 通过 `fssdk.SDKError` 返回错误，可用 `errors.As` 获取 `Code`： + +| 错误码 | 含义 | +|----------------------------|----------------| +| `ErrRuntimeInvalidDriver` | 传入的 Driver 无效。 | +| `ErrRuntimeNotInitialized` | 运行时未初始化即调用。 | +| `ErrRuntimeClosed` | 运行时已关闭。 | +| `ErrStoreInvalidName` | Store 名称不合法。 | +| `ErrStoreInternal` | Store 内部错误。 | +| `ErrStoreNotFound` | 未找到指定 Store。 | +| `ErrStoreIO` | Store 读写异常。 | +| `ErrResultUnexpected` | 宿主返回了意外结果。 | + +处理示例： + +```go +if err != nil { + var sdkErr *fssdk.SDKError + if errors.As(err, &sdkErr) { + switch sdkErr.Code { + case fssdk.ErrStoreNotFound: + // 按业务决定是否创建或返回 + default: + // 记录并向上返回 + } + } + return err +} +``` + +--- + +## 七、高级状态 API + +本节说明**高级状态 API**：在底层 `Store` 之上构建的**类型化状态抽象**，通过 **Codec** 序列化，并支持按主键的 **Keyed 状态**。适用于需要结构化状态（单值、列表、Map、优先队列、聚合、归约）且不希望手写字节编码与键布局的场景。 + +### 7.1 何时使用哪种 API + +| 使用场景 | 推荐 API | +|----------|-----------| +| 单一逻辑值（如计数器、配置 blob） | **ValueState[T]** 或单键的裸 `Store`。 | +| 仅追加序列（如事件日志） | **ListState[T]** 或自定义编码的 `Store.Merge`。 | +| 需按范围迭代的键值 Map | **MapState[K,V]**（键类型须为有序 Codec）。 | +| 优先队列（最小/最大、Top-K） | **PriorityQueueState[T]**（元素 Codec 须有序）。 | +| 运行中聚合（sum、count、自定义累加器） | 带 `AggregateFunc` 的 **AggregatingState[T,ACC,R]**。 | +| 运行中归约（二元合并） | 带 `ReduceFunc` 的 **ReducingState[V]**。 | +| **按 key 维度的状态**（如按用户、按分区键） | **Keyed*** 工厂：先获取工厂，再按 key 调用 `NewKeyedValue(primaryKey, ...)` 等。**供 Keyed 算子使用** — 当流按 key 分区（如经 keyBy 后）时，每个 key 拥有独立状态。 | +| 自定义键布局、批量扫描或非类型化存储 | 底层 **Store**（`Put`/`Get`/`ScanComplex`/`ComplexKey`）。 | + +**Keyed 与非 Keyed：** **Keyed 状态面向 Keyed 算子**：即流已按 key 分区（例如在 DAG 中经 keyBy 后）。此时运行时按 key 投递记录，每个 key 应有独立状态 — 通过 **工厂** 和 **keyGroup** 创建状态，再按 **主键**（即流上的 key）调用 `factory.NewKeyedValue(primaryKey, stateName)` 等。非 Keyed 状态（如 `ValueState`、`ListState`）在每个 store 下存一份逻辑实体，用于无 key 分区或单一全局状态的场景。 + +### 7.2 包与模块路径 + +| 包名 | 导入路径 | 职责 | +|------|----------|------| +| **structures** | `github.com/functionstream/function-stream/go-sdk/state/structures` | 非 Keyed 状态类型：`ValueState`、`ListState`、`MapState`、`PriorityQueueState`、`AggregatingState`、`ReducingState`。 | +| **keyed** | `github.com/functionstream/function-stream/go-sdk/state/keyed` | Keyed 状态**工厂**及其按 key 的状态类型（如 `KeyedListStateFactory`、`KeyedListState`）。**供 Keyed 算子使用** — 当流按 key 分区时，按流 key 创建状态。 | +| **codec** | `github.com/functionstream/function-stream/go-sdk/state/codec` | `Codec[T]` 接口、`DefaultCodecFor[T]()` 及内置 Codec（基本类型、`JSONCodec`）。 | + +所有状态构造函数均接收 `api.Context`（即 `fssdk.Context`）和 **store 名称**；内部通过 `ctx.GetOrCreateStore(storeName)` 获取 Store。同一 store 名称始终对应同一底层存储（默认实现为 RocksDB）。 + +### 7.3 Codec 约定与默认 Codec + +**Codec 接口**（`codec.Codec[T]`）： + +- `Encode(value T) ([]byte, error)` — 序列化为字节。 +- `Decode(data []byte) (T, error)` — 从字节反序列化。 +- `EncodedSize() int` — 固定长度时 `> 0`，变长时 `<= 0`（用于 List 优化）。 +- `IsOrderedKeyCodec() bool` — 为 `true` 时表示字节编码**全序**（字节字典序与值的确定顺序一致）。**MapState** 的键与 **PriorityQueueState** 的元素必须使用有序 Codec。 + +**DefaultCodecFor[T]()** 按类型返回默认 Codec： + +- **基本类型**（`int32`、`int64`、`uint32`、`uint64`、`float32`、`float64`、`string`、`bool`、`int`、`uint` 等）：内置定长或变长 Codec；用作 Map 键或 PQ 元素时**有序**。 +- **结构体、map、slice、数组**：使用 `JSONCodec[T]`（JSON 编码），**非有序**（`IsOrderedKeyCodec() == false`）。**不要**将其作为 MapState 键或 PriorityQueueState 元素类型与 AutoCodec 一起使用（创建可能成功，但依赖顺序的操作可能失败或 panic）。 +- **无类型约束的 interface**：返回错误（类型参数须为具体类型）。 + +**有序性要求：** `MapState[K,V]` 与 `PriorityQueueState[T]` 的**键**（或**元素**）类型必须使用 `IsOrderedKeyCodec() == true` 的 Codec。使用 Map/PQ 的 **AutoCodec** 构造函数时，请选择基本类型键/元素（如 `int64`、`string`），或显式提供有序 Codec。 + +### 7.4 创建状态：带 Codec 与 AutoCodec + +两类构造函数： + +1. **显式 Codec** — `NewXxxFromContext(ctx, storeName, codec, ...)` + 由调用方传入 `Codec[T]`（Map 需 key+value codec；Aggregating/Reducing 需 acc/value codec 与函数）。可完全控制编码与有序性。 + +2. **AutoCodec** — `NewXxxFromContextAutoCodec(ctx, storeName)` 或 `(ctx, storeName, aggFunc/reduceFunc)` + SDK 对值/累加器类型调用 `codec.DefaultCodecFor[T]()`。Map 与 PQ 的 key/元素类型须有**有序**默认 Codec（基本类型），否则运行时检查可能返回 `ErrStoreInternal`。 + +状态实例是**轻量**的；可在每次调用时创建（如在 `Process` 内调用 `NewValueStateFromContextAutoCodec[int64](ctx, "store")`），也可在 Driver 中缓存（如在 `Init` 中）。同一 store 名称对应同一底层 store，仅类型化视图不同。 + +### 7.5 非 Keyed 状态（structures）— 参考 + +| 状态 | 语义 | 方法（签名概要） | 是否要求有序 Codec | +|------|------|------------------|---------------------| +| **ValueState[T]** | 单一可替换值。 | `Update(value T) error`；`Value() (T, bool, error)`；`Clear() error` | 否 | +| **ListState[T]** | 仅追加列表；支持批量追加与整体替换。 | `Add(value T) error`；`AddAll(values []T) error`；`Get() ([]T, error)`；`Update(values []T) error`；`Clear() error` | 否 | +| **MapState[K,V]** | 键值 Map；通过 `All()` 按键范围迭代。 | `Put(key K, value V) error`；`Get(key K) (V, bool, error)`；`Delete(key K) error`；`Clear() error`；`All() iter.Seq2[K,V]` | **键 K：是** | +| **PriorityQueueState[T]** | 优先队列（按编码键顺序最小优先）。 | `Add(value T) error`；`Peek() (T, bool, error)`；`Poll() (T, bool, error)`；`Clear() error`；`All() iter.Seq[T]` | **元素 T：是** | +| **AggregatingState[T,ACC,R]** | 可合并累加器的运行中聚合。 | `Add(value T) error`；`Get() (R, bool, error)`；`Clear() error` | 否（ACC 任意） | +| **ReducingState[V]** | 二元合并的运行中归约。 | `Add(value V) error`；`Get() (V, bool, error)`；`Clear() error` | 否 | + +**构造函数概要（非 Keyed）：** + +| 状态 | 带 Codec | AutoCodec | +|------|-----------|-----------| +| ValueState[T] | `NewValueStateFromContext(ctx, storeName, valueCodec)` | `NewValueStateFromContextAutoCodec[T](ctx, storeName)` | +| ListState[T] | `NewListStateFromContext(ctx, storeName, itemCodec)` | `NewListStateFromContextAutoCodec[T](ctx, storeName)` | +| MapState[K,V] | `NewMapStateFromContext(ctx, storeName, keyCodec, valueCodec)` 或 `NewMapStateAutoKeyCodecFromContext(ctx, storeName, valueCodec)` | `NewMapStateFromContextAutoCodec[K,V](ctx, storeName)` | +| PriorityQueueState[T] | `NewPriorityQueueStateFromContext(ctx, storeName, itemCodec)` | `NewPriorityQueueStateFromContextAutoCodec[T](ctx, storeName)` | +| AggregatingState[T,ACC,R] | `NewAggregatingStateFromContext(ctx, storeName, accCodec, aggFunc)` | `NewAggregatingStateFromContextAutoCodec(ctx, storeName, aggFunc)` | +| ReducingState[V] | `NewReducingStateFromContext(ctx, storeName, valueCodec, reduceFunc)` | `NewReducingStateFromContextAutoCodec[V](ctx, storeName, reduceFunc)` | + +### 7.6 AggregateFunc 与 ReduceFunc + +**AggregatingState** 需要实现 **AggregateFunc[T, ACC, R]**（定义在 `structures`）： + +- `CreateAccumulator() ACC` — 空状态下的初始累加器。 +- `Add(value T, accumulator ACC) ACC` — 将一条输入折叠进累加器。 +- `GetResult(accumulator ACC) R` — 从累加器得到最终结果。 +- `Merge(a, b ACC) ACC` — 合并两个累加器（用于分布式/ checkpoint 合并）。 + +**ReducingState** 需要 **ReduceFunc[V]**（函数类型）：`func(value1, value2 V) (V, error)`。须满足结合律（最好可交换），以便多次应用得到确定的归约结果。 + +### 7.7 Keyed 状态 — 工厂与按 key 的实例 + +**Keyed 状态专为 Keyed 算子设计。** 当流已按 key 分区（例如在流水线中经 keyBy 后）时，每个 key 在隔离的状态下被处理。Keyed 状态 API 让你先获取一次**工厂**（通过 context、store 名称和 **keyGroup**），再按**主键**创建状态 — 主键即当前记录所在的流 key（如用户 ID、分区键）。在算子运行于 keyed 流上时使用 Keyed* 工厂；流未按 key 分区或需要单一全局状态时使用非 Keyed 状态。 + +Keyed 状态由 **keyGroup**（[]byte）和 **primary key**（[]byte）组织。先由 context、store 名称和 keyGroup 创建**工厂**，再通过工厂方法获取某主键下的状态。 + +#### 7.7.1 keyGroup、key（primaryKey）与 namespace + +Keyed 状态 API 对应底层 Store 的 **ComplexKey** 三个维度，含义如下： + +| 概念 | API 参数 | 含义 | +|------|----------|------| +| **keyGroup** | 创建工厂时的 `keyGroup` | **Keyed 的组**：标识该状态属于哪一个 keyed 分区/组。每个逻辑上的“keyed 组”或状态种类对应一个 keyGroup（例如一组用于 “counters”，另一组用于 “sessions”）。同一 keyed 组使用相同 keyGroup 字节；不同组使用不同 keyGroup。 | +| **key** | 工厂方法中的 `primaryKey`（如 `NewKeyedValue(primaryKey, ...)`、`NewKeyedList(primaryKey, namespace)`） | **对应 Key 的值**：当前记录在流上的 key，以字节形式序列化。即对流进行分区时使用的 key（如用户 ID、分区键）。每个不同的 key 值拥有独立状态。 | +| **namespace** | 工厂方法中的 `namespace`（[]byte） | **若存在窗口函数**，则传入**该窗口对应的 bytes**（例如序列化后的窗口边界或窗口 ID），使状态按 key *且* 按窗口隔离。**无窗口时**传入**空 bytes**（如 `nil` 或 `[]byte{}`）。 | + +**小结：** **keyGroup** = keyed 的组；**key**（primaryKey）= Key 的值（流 key）；**namespace** = 有窗口函数时为窗口的 bytes，否则为**空 bytes**。 + +**工厂构造函数概要（Keyed）：** + +| 工厂 | 带 Codec | AutoCodec | +|------|-----------|-----------| +| KeyedValueStateFactory[V] | `NewKeyedValueStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec)` | `NewKeyedValueStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup)` | +| KeyedListStateFactory[V] | `NewKeyedListStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec)` | `NewKeyedListStateFactoryAutoCodecFromContext[V](ctx, storeName, keyGroup)` | +| KeyedMapStateFactory[MK,MV] | `NewKeyedMapStateFactoryFromContext(ctx, storeName, keyGroup, keyCodec, valueCodec)` | `NewKeyedMapStateFactoryFromContextAutoCodec[MK,MV](ctx, storeName, keyGroup)` | +| KeyedPriorityQueueStateFactory[V] | `NewKeyedPriorityQueueStateFactoryFromContext(ctx, storeName, keyGroup, itemCodec)` | `NewKeyedPriorityQueueStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup)` | +| KeyedAggregatingStateFactory[T,ACC,R] | `NewKeyedAggregatingStateFactoryFromContext(ctx, storeName, keyGroup, accCodec, aggFunc)` | `NewKeyedAggregatingStateFactoryFromContextAutoCodec(ctx, storeName, keyGroup, aggFunc)` | +| KeyedReducingStateFactory[V] | `NewKeyedReducingStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec, reduceFunc)` | `NewKeyedReducingStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup, reduceFunc)` | + +**从工厂获取按 key 的状态：** + +- **KeyedValueStateFactory[V]**：`NewKeyedValue(primaryKey []byte, stateName string) (*KeyedValueState[V], error)` — 每个 (primaryKey, stateName) 对应一个 value 状态。 +- **KeyedListStateFactory[V]**：`NewKeyedList(primaryKey []byte, namespace []byte) (*KeyedListState[V], error)`。 +- **KeyedMapStateFactory[MK,MV]**：`NewKeyedMap(primaryKey []byte, mapName string) (*KeyedMapState[MK,MV], error)`。 +- **KeyedPriorityQueueStateFactory[V]**：`NewKeyedPriorityQueue(primaryKey []byte, namespace []byte) (*KeyedPriorityQueueState[V], error)`。 +- **KeyedAggregatingStateFactory**：`NewAggregatingState(primaryKey []byte, stateName string) (*KeyedAggregatingState[T,ACC,R], error)`。 +- **KeyedReducingStateFactory[V]**：`NewReducingState(primaryKey []byte, namespace []byte) (*KeyedReducingState[V], error)`。 + +其中 **primaryKey** 即 key（流 key 的值）；**namespace** 在有窗口函数时为窗口的 bytes，无窗口时为**空 bytes**。 + +**keyGroup** 用于划分键空间（例如按逻辑状态名或功能维度）。同一逻辑状态应保持 keyGroup 稳定；不同状态实体可共用同一 store 但使用不同 keyGroup。工厂方法中的 **primaryKey** 即 keyed 流中当前记录的 key — 传入你的 Keyed 算子收到的 key（如从 key 提取器或消息元数据）。 + +### 7.8 错误处理与最佳实践 + +- **状态 API 返回的错误**：创建与方法返回的错误与 `fssdk.SDKError` 兼容（如 `ErrStoreInternal`、`ErrStoreIO`）。Codec 编解码错误会被包装（如 `"encode value state failed"`）。生产环境应始终检查并处理错误。 +- **Store 命名**：按逻辑状态使用稳定、唯一的 store 名称（如 `"counters"`、`"user-sessions"`）。同一运行时内相同名称指向同一 store。 +- **状态缓存**：可在 `Init` 中创建一次状态实例并在 `Process` 中复用，也可每条消息创建。按消息创建是安全的，在不需要分摊创建成本时可使代码更简单。 +- **KeyGroup 设计**：Keyed 状态下，每个“逻辑表”使用一致的 keyGroup（如 `[]byte("orders")`）。primaryKey 在 Keyed 算子中即**流 key** — 使用标识当前记录的那个 key（如来自 key 提取器或消息 key）。**使用窗口函数时**，将窗口标识作为 **namespace**（如序列化后的窗口边界）传入，使状态按 key 且按窗口隔离。 +- **有序 Codec**：MapState 与 PriorityQueueState 使用 AutoCodec 时，键/元素使用基本类型。自定义结构体键时，实现 `IsOrderedKeyCodec() == true` 的 `Codec[K]` 并改用“带 codec”的构造函数。 + +### 7.9 示例：带 AutoCodec 的 ValueState（计数器） + +```go +import ( + fssdk "github.com/functionstream/function-stream/go-sdk" + "github.com/functionstream/function-stream/go-sdk/state/structures" +) + +func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + valState, err := structures.NewValueStateFromContextAutoCodec[int64](ctx, "my-store") + if err != nil { + return err + } + cur, _, _ := valState.Value() + if err := valState.Update(cur + 1); err != nil { + return err + } + // ... +} +``` + +### 7.10 示例：Keyed list 工厂（Keyed 算子） + +当算子运行在**按 key 分区的流**上时，使用 Keyed list 工厂，并为每条消息传入**流 key** 作为 primaryKey： + +```go +import ( + fssdk "github.com/functionstream/function-stream/go-sdk" + "github.com/functionstream/function-stream/go-sdk/state/keyed" +) + +type Order struct { Id string; Amount int64 } + +func (p *MyProcessor) Init(ctx fssdk.Context, config map[string]string) error { + keyGroup := []byte("orders") + factory, err := keyed.NewKeyedListStateFactoryAutoCodecFromContext[Order](ctx, "app-store", keyGroup) + if err != nil { + return err + } + p.listFactory = factory + return nil +} + +func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + userID := parseUserID(data) // []byte + list, err := p.listFactory.NewKeyedList(userID, "items") + if err != nil { + return err + } + if err := list.Add(Order{Id: "1", Amount: 100}); err != nil { + return err + } + items, err := list.Get() + if err != nil { + return err + } + // 使用 items... +} +``` + +### 7.11 示例：MapState 与 AggregatingState（求和） + +```go +// MapState：string -> int64 计数（二者均有有序默认 codec） +m, err := structures.NewMapStateFromContextAutoCodec[string, int64](ctx, "counts") +if err != nil { return err } +_ = m.Put("a", 1) +v, ok, _ := m.Get("a") + +// AggregatingState：int64 求和，AutoCodec（ACC = int64, R = int64） +type sumAgg struct{} +func (sumAgg) CreateAccumulator() int64 { return 0 } +func (sumAgg) Add(v int64, acc int64) int64 { return acc + v } +func (sumAgg) GetResult(acc int64) int64 { return acc } +func (sumAgg) Merge(a, b int64) int64 { return a + b } +agg, err := structures.NewAggregatingStateFromContextAutoCodec[int64, int64, int64](ctx, "sum-store", sumAgg{}) +if err != nil { return err } +_ = agg.Add(10) +total, _, _ := agg.Get() +``` + +--- + +## 八、目录结构参考 + +```text +go-sdk/ +├── Makefile # env / wit / bindings / build +├── fssdk.go # 对外入口与类型重导出 +├── go.mod / go.sum +├── api/ # 接口与错误码 +│ ├── driver.go # Driver、BaseDriver +│ ├── context.go # Context +│ ├── store.go # Store、Iterator、ComplexKey +│ └── errors.go # ErrorCode、SDKError +├── impl/ # 与 WASM 宿主桥接 +│ ├── runtime.go +│ ├── context.go +│ └── store.go +├── state/ # 高级状态 API +│ ├── structures/ # ValueState、ListState、MapState、PriorityQueue、Aggregating、Reducing +│ ├── keyed/ # Keyed 状态工厂（value、list、map、PQ、aggregating、reducing） +│ ├── codec/ # Codec[T]、DefaultCodecFor、内置与 JSON codec +│ └── common/ # 公共辅助（如 DupBytes） +├── wit/ # processor.wit 及依赖（可由 make wit 生成） +└── bindings/ # wit-bindgen-go 生成的 Go 代码（make bindings） +``` + +更多示例与 SQL 操作见 [examples/go-processor/README.md](../../examples/go-processor/README.md)、[SQL CLI 指南](../sql-cli-guide-zh.md)。 diff --git a/docs/Go-SDK/go-sdk-guide.md b/docs/Go-SDK/go-sdk-guide.md new file mode 100644 index 00000000..49592ef6 --- /dev/null +++ b/docs/Go-SDK/go-sdk-guide.md @@ -0,0 +1,553 @@ + + +# Go SDK Guide + +Function Stream provides a WebAssembly (WASI P2) based operator SDK for Go developers. Go code is compiled to a WASM component with TinyGo and runs in the server sandbox with KV state storage and data emission support. + +--- + +## 1. SDK Core Components + +| Component | Role | Description | +|--------------|------------------------|----------------------------------------------------------------------------------| +| **fssdk** | Main package | Public entry point; exposes `Driver`, `Context`, `Store`, and `Run(driver)`. | +| **api** | Interface definitions | Defines `Driver`, `Context`, `Store`, `Iterator`, `ComplexKey`, and error codes. | +| **impl** | Runtime implementation | Bridges the Driver to the WASM host (processor WIT); internal use. | +| **bindings** | WIT bindings | Go code generated by wit-bindgen-go from `wit/processor.wit`; used by impl. | + +Operators **only depend on fssdk**: implement `Driver` (or embed `BaseDriver`) and call `fssdk.Run(&YourProcessor{})` from `init()`. + +--- + +## 2. Driver Interface and BaseDriver + +### 2.1 Driver Interface + +All Go operators must implement the `fssdk.Driver` interface. The runtime invokes the following methods at the appropriate times: + +| Method | When invoked | Description | +|----------------------------------------------|--------------------------------|-----------------------------------------------------------------------------------| +| `Init(ctx, config)` | Once when the function starts | Initialize state, obtain Store, parse config. | +| `Process(ctx, sourceID, data)` | For each incoming message | Core logic: compute, state read/write, `ctx.Emit()`. | +| `ProcessWatermark(ctx, sourceID, watermark)` | On watermark events | Handle time-based windows or out-of-order logic; may forward via `EmitWatermark`. | +| `TakeCheckpoint(ctx, checkpointID)` | When the system backs up state | Persist extra in-memory state for strong consistency. | +| `CheckHeartbeat(ctx)` | Periodic health check | Return `false` to trigger operator restart. | +| `Close(ctx)` | When the function shuts down | Release resources and clear references. | +| `Exec(ctx, className, modules)` | Optional extension | Dynamic module loading, etc.; default no-op. | +| `Custom(ctx, payload)` | Optional custom RPC | Request/response custom bytes; default returns a copy of payload. | + +### 2.2 BaseDriver + +`fssdk.BaseDriver` provides no-op implementations for all of the above. Embed it and override only the methods you need: + +```go +type MyProcessor struct { + fssdk.BaseDriver + store fssdk.Store +} + +func (p *MyProcessor) Init(ctx fssdk.Context, config map[string]string) error { + store, err := ctx.GetOrCreateStore("my-store") + if err != nil { + return err + } + p.store = store + return nil +} + +func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + // Your logic; read/write p.store, then ctx.Emit(...) + return ctx.Emit(0, result) +} +``` + +### 2.3 Registration Entry Point + +Call `fssdk.Run` from `init()` with your Driver instance (typically a singleton): + +```go +func init() { + fssdk.Run(&MyProcessor{}) +} +``` + +--- + +## 3. Context and Store + +### 3.1 Context + +`fssdk.Context` is passed into Init, Process, ProcessWatermark, etc. It provides: + +| Method | Description | +|----------------------------------------------------------|-------------------------------------------------------------| +| `Emit(targetID uint32, data []byte) error` | Send data to the given output channel. | +| `EmitWatermark(targetID uint32, watermark uint64) error` | Emit a watermark. | +| `GetOrCreateStore(name string) (Store, error)` | Get or create a KV Store by name (RocksDB-backed). | +| `Config() map[string]string` | Startup configuration (e.g. from config.yaml init section). | +| `Close() error` | Close the context; usually managed by the runtime. | + +### 3.2 Store (KV State) + +`fssdk.Store` provides key-value and complex-key operations: + +**Basic API:** + +- `PutState(key, value []byte) error` +- `GetState(key []byte) (value []byte, found bool, err error)` +- `DeleteState(key []byte) error` +- `ListStates(startInclusive, endExclusive []byte) ([][]byte, error)` — list keys in byte order range. + +**ComplexKey (multi-dimensional keys / prefix scan):** + +- `Put(key ComplexKey, value []byte)` / `Get` / `Delete` / `Merge` / `DeletePrefix` +- `ListComplex(...)` — list UserKeys by keyGroup, key, namespace and range. +- `ScanComplex(keyGroup, key, namespace []byte) (Iterator, error)` — returns an iterator for large scans. + +`ComplexKey` holds `KeyGroup`, `Key`, `Namespace`, and `UserKey` for multi-dimensional indexing and prefix queries. + +### 3.3 Iterator + +`Store.ScanComplex` returns an `fssdk.Iterator`: + +- `HasNext() (bool, error)` +- `Next() (key, value []byte, ok bool, err error)` +- `Close() error` — must be called when done to release resources. + +--- + +## 4. Production Example (Word Count) + +```go +package main + +import ( + "encoding/json" + fssdk "github.com/functionstream/function-stream/go-sdk" + "strconv" + "strings" +) + +func init() { + fssdk.Run(&CounterProcessor{}) +} + +type CounterProcessor struct { + fssdk.BaseDriver + store fssdk.Store + counterMap map[string]int64 + totalProcessed int64 + keyPrefix string +} + +func (p *CounterProcessor) Init(ctx fssdk.Context, config map[string]string) error { + store, err := ctx.GetOrCreateStore("counter-store") + if err != nil { + return err + } + p.store = store + p.counterMap = make(map[string]int64) + p.totalProcessed = 0 + p.keyPrefix = strings.TrimSpace(config["key_prefix"]) + return nil +} + +func (p *CounterProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + inputStr := strings.TrimSpace(string(data)) + if inputStr == "" { + return nil + } + p.totalProcessed++ + + fullKey := p.keyPrefix + inputStr + existing, found, err := p.store.GetState([]byte(fullKey)) + if err != nil { + return err + } + currentCount := int64(0) + if found { + if n, e := strconv.ParseInt(string(existing), 10, 64); e == nil { + currentCount = n + } + } + newCount := currentCount + 1 + p.counterMap[inputStr] = newCount + if err = p.store.PutState([]byte(fullKey), []byte(strconv.FormatInt(newCount, 10))); err != nil { + return err + } + + out := map[string]interface{}{ + "total_processed": p.totalProcessed, + "counter_map": p.counterMap, + } + jsonBytes, _ := json.Marshal(out) + return ctx.Emit(0, jsonBytes) +} + +func (p *CounterProcessor) ProcessWatermark(ctx fssdk.Context, sourceID uint32, watermark uint64) error { + return ctx.EmitWatermark(0, watermark) +} + +func (p *CounterProcessor) Close(ctx fssdk.Context) error { + p.store = nil + p.counterMap = nil + return nil +} + +func main() {} +``` + +--- + +## 5. Build and Deployment + +### 5.1 Prerequisites + +- **Go**: 1.23+ +- **TinyGo**: 0.40+ (WASI P2 support) +- **wit-bindgen-go**: `go install go.bytecodealliance.org/cmd/wit-bindgen-go@latest` +- **wkg** (optional): for fetching WIT deps — `cargo install wkg --version 0.10.0` +- **wasm-tools**: for component validation — `cargo install wasm-tools` + +### 5.2 Generate Bindings and Build + +From the project root: + +```bash +# Install Go SDK toolchain (wit-bindgen-go, wkg, etc.) +make -C go-sdk env + +# Generate WIT bindings (from wit/processor.wit and deps) +make -C go-sdk bindings + +# Run tests +make -C go-sdk build +``` + +In your operator project (e.g. `examples/go-processor`): + +```bash +# Build as WASI P2 component with TinyGo +tinygo build -o build/processor.wasm -target=wasi . +``` + +See `examples/go-processor/build.sh` for the exact command. + +### 5.3 Register and Run + +Register the built `processor.wasm` and `config.yaml` as a function via the SQL CLI: + +```sql +create function with ( + 'function_path'='/path/to/build/processor.wasm', + 'config_path'='/path/to/config.yaml' +); +``` + +Configure `name`, `type: processor`, `input-groups`, and `outputs` (e.g. Kafka) in config.yaml. See [Function Configuration](../function-configuration.md) and [examples/go-processor/README.md](../../examples/go-processor/README.md). + +--- + +## 6. Error Codes and Handling + +The SDK returns errors as `fssdk.SDKError`; use `errors.As` to get the `Code`: + +| Code | Meaning | +|----------------------------|-------------------------------| +| `ErrRuntimeInvalidDriver` | Invalid Driver passed to Run. | +| `ErrRuntimeNotInitialized` | Runtime not initialized. | +| `ErrRuntimeClosed` | Runtime already closed. | +| `ErrStoreInvalidName` | Invalid Store name. | +| `ErrStoreInternal` | Store internal error. | +| `ErrStoreNotFound` | Store not found. | +| `ErrStoreIO` | Store I/O error. | +| `ErrResultUnexpected` | Unexpected result from host. | + +Example handling: + +```go +if err != nil { + var sdkErr *fssdk.SDKError + if errors.As(err, &sdkErr) { + switch sdkErr.Code { + case fssdk.ErrStoreNotFound: + // Handle: create or return + default: + // Log and propagate + } + } + return err +} +``` + +--- + +## 7. Advanced State API + +This section describes the **high-level state API**: typed state abstractions built on top of the low-level `Store`, with serialization via **codecs** and optional **keyed state** per primary key. Use it when you want structured state (single value, list, map, priority queue, aggregation, reduction) without manual byte encoding or key layout. + +### 7.1 When to Use Which API + +| Use case | Recommended API | +|----------|------------------| +| Single logical value (e.g. counter, config blob) | **ValueState[T]** or raw `Store` with one key. | +| Append-only sequence (e.g. event log) | **ListState[T]** or `Store.Merge` with custom encoding. | +| Key-value map with range/iteration | **MapState[K,V]** (key type must have ordered codec). | +| Priority queue (min/max, top-K) | **PriorityQueueState[T]** (item codec must be ordered). | +| Running aggregate (sum, count, custom accumulator) | **AggregatingState[T,ACC,R]** with `AggregateFunc`. | +| Running reduce (binary combine) | **ReducingState[V]** with `ReduceFunc`. | +| **Per-key** state (e.g. per user, per partition key) | **Keyed*** factories: get a factory once, then `NewKeyedValue(primaryKey, ...)` etc. per key. **Use in keyed operators** — when the stream is partitioned by a key (e.g. after keyBy), each key gets isolated state. | +| Custom key layout, bulk scan, or non-typed storage | Low-level **Store** (`Put`/`Get`/`ScanComplex`/`ComplexKey`). | + +**Keyed vs non-keyed:** **Keyed state is intended for keyed operators**: streams that are partitioned by a key (e.g. after a keyBy in the pipeline). In that case, the runtime delivers records per key, and each key should have its own independent state — use a **factory** and **keyGroup**, then create state per **primary key** (the stream key) via `factory.NewKeyedValue(primaryKey, stateName)` etc. Non-keyed state (e.g. `ValueState`, `ListState`) stores one logical entity per store and is used when there is no key partitioning or you maintain a single global state. + +### 7.2 Packages and Module Paths + +| Package | Import path | Responsibility | +|---------|-------------|----------------| +| **structures** | `github.com/functionstream/function-stream/go-sdk/state/structures` | Non-keyed state types: `ValueState`, `ListState`, `MapState`, `PriorityQueueState`, `AggregatingState`, `ReducingState`. | +| **keyed** | `github.com/functionstream/function-stream/go-sdk/state/keyed` | Keyed state **factories** and their per-key state types (e.g. `KeyedListStateFactory`, `KeyedListState`). **Use in keyed operators** — when the stream is partitioned by a key, create state per stream key. | +| **codec** | `github.com/functionstream/function-stream/go-sdk/state/codec` | `Codec[T]` interface, `DefaultCodecFor[T]()`, and built-in codecs (primitives, `JSONCodec`). | + +All state constructors take `api.Context` (i.e. `fssdk.Context`) and a **store name**; the store is obtained via `ctx.GetOrCreateStore(storeName)` internally. The same store name always refers to the same backing store (RocksDB-backed in the default implementation). + +### 7.3 Codec Contract and Default Codecs + +**Codec interface** (`codec.Codec[T]`): + +- `Encode(value T) ([]byte, error)` — serialize to bytes. +- `Decode(data []byte) (T, error)` — deserialize from bytes. +- `EncodedSize() int` — fixed size if `> 0`, variable size if `<= 0` (used for list optimization). +- `IsOrderedKeyCodec() bool` — if `true`, the byte encoding is **totally ordered** (lexicographic order of bytes corresponds to a well-defined order of values). Required for **MapState** key and **PriorityQueueState** item. + +**DefaultCodecFor[T]()** returns a default codec for type `T`: + +- **Primitives** (`int32`, `int64`, `uint32`, `uint64`, `float32`, `float64`, `string`, `bool`, `int`, `uint`, etc.): built-in fixed- or variable-size codecs; **ordered** for types used as map keys or PQ elements. +- **Struct, map, slice, array**: `JSONCodec[T]` — JSON encoding; **not ordered** (`IsOrderedKeyCodec() == false`). Do **not** use as MapState key or PriorityQueueState element type with AutoCodec (creation will succeed but operations that rely on ordering may fail or panic). +- **Interface type without constraint**: returns error (type parameter must be concrete). + +**Ordering requirement:** For `MapState[K,V]` and `PriorityQueueState[T]`, the **key** (respectively **element**) type must use a codec with `IsOrderedKeyCodec() == true`. When using **AutoCodec** constructors for Map or PQ, choose primitive key/element types (e.g. `int64`, `string`) or provide an explicit ordered codec. + +### 7.4 Creating State: With Codec vs AutoCodec + +Two constructor families: + +1. **Explicit codec** — `NewXxxFromContext(ctx, storeName, codec, ...)` + You pass a `Codec[T]` (and for Map: key + value codecs; for Aggregating/Reducing: acc/value codec + function). Full control over encoding and ordering. + +2. **AutoCodec** — `NewXxxFromContextAutoCodec(ctx, storeName)` or `(ctx, storeName, aggFunc/reduceFunc)` + The SDK calls `codec.DefaultCodecFor[T]()` for the value/accumulator type. For Map and PQ, key/element type `T` must have an **ordered** default (primitives); otherwise runtime checks may return `ErrStoreInternal`. + +State instances are **lightweight**; you may create them per invocation (e.g. `NewValueStateFromContextAutoCodec[int64](ctx, "store")` inside `Process`) or cache in the Driver (e.g. in `Init`). Same store name yields the same underlying store; only the typed view differs. + +### 7.5 Non-Keyed State (structures) — Reference + +| State | Semantics | Methods (signature summary) | Ordered codec? | +|-------|------------|-----------------------------|----------------| +| **ValueState[T]** | Single replaceable value. | `Update(value T) error`; `Value() (T, bool, error)`; `Clear() error` | No | +| **ListState[T]** | Append-only list; supports batch add and full replace. | `Add(value T) error`; `AddAll(values []T) error`; `Get() ([]T, error)`; `Update(values []T) error`; `Clear() error` | No | +| **MapState[K,V]** | Key-value map; key range iteration via `All()`. | `Put(key K, value V) error`; `Get(key K) (V, bool, error)`; `Delete(key K) error`; `Clear() error`; `All() iter.Seq2[K,V]` | **Key K: yes** | +| **PriorityQueueState[T]** | Priority queue (min-first by encoded key order). | `Add(value T) error`; `Peek() (T, bool, error)`; `Poll() (T, bool, error)`; `Clear() error`; `All() iter.Seq[T]` | **Item T: yes** | +| **AggregatingState[T,ACC,R]** | Running aggregation with mergeable accumulator. | `Add(value T) error`; `Get() (R, bool, error)`; `Clear() error` | No (ACC codec any) | +| **ReducingState[V]** | Running reduce with binary combine. | `Add(value V) error`; `Get() (V, bool, error)`; `Clear() error` | No | + +**Constructor summary (non-keyed):** + +| State | With codec | AutoCodec | +|-------|------------|-----------| +| ValueState[T] | `NewValueStateFromContext(ctx, storeName, valueCodec)` | `NewValueStateFromContextAutoCodec[T](ctx, storeName)` | +| ListState[T] | `NewListStateFromContext(ctx, storeName, itemCodec)` | `NewListStateFromContextAutoCodec[T](ctx, storeName)` | +| MapState[K,V] | `NewMapStateFromContext(ctx, storeName, keyCodec, valueCodec)` or `NewMapStateAutoKeyCodecFromContext(ctx, storeName, valueCodec)` | `NewMapStateFromContextAutoCodec[K,V](ctx, storeName)` | +| PriorityQueueState[T] | `NewPriorityQueueStateFromContext(ctx, storeName, itemCodec)` | `NewPriorityQueueStateFromContextAutoCodec[T](ctx, storeName)` | +| AggregatingState[T,ACC,R] | `NewAggregatingStateFromContext(ctx, storeName, accCodec, aggFunc)` | `NewAggregatingStateFromContextAutoCodec(ctx, storeName, aggFunc)` | +| ReducingState[V] | `NewReducingStateFromContext(ctx, storeName, valueCodec, reduceFunc)` | `NewReducingStateFromContextAutoCodec[V](ctx, storeName, reduceFunc)` | + +### 7.6 AggregateFunc and ReduceFunc + +**AggregatingState** requires an **AggregateFunc[T, ACC, R]** (defined in `structures`): + +- `CreateAccumulator() ACC` — initial accumulator for empty state. +- `Add(value T, accumulator ACC) ACC` — fold one input into the accumulator. +- `GetResult(accumulator ACC) R` — produce the final result from the accumulator. +- `Merge(a, b ACC) ACC` — combine two accumulators (e.g. for merge in distributed/checkpointed execution). + +**ReducingState** requires a **ReduceFunc[V]** (function type): `func(value1, value2 V) (V, error)`. It must be associative (and ideally commutative) so that repeated application yields a well-defined reduced value. + +### 7.7 Keyed State — Factories and Per-Key Instances + +**Keyed state is for keyed operators.** When the stream is partitioned by a key (e.g. after keyBy in the pipeline), each key is processed with isolated state. The Keyed state API lets you obtain a **factory** once (from context, store name, and **keyGroup**), then create state **per primary key** — the primary key is the stream key (e.g. user ID, partition key) for the current record. Use Keyed* factories when your operator runs on a keyed stream; use non-keyed state when the stream is not keyed or you need a single global state. + +Keyed state is organized by **keyGroup** ([]byte) and **primary key** ([]byte). You first create a **factory** from context, store name, and keyGroup; then call factory methods to obtain state for a given primary key. + +#### 7.7.1 keyGroup, key (primaryKey), and namespace + +The Keyed state API maps onto the store’s **ComplexKey** with three dimensions. Use them as follows: + +| Term | API parameter | Meaning | +|------|----------------|---------| +| **keyGroup** | `keyGroup` when creating the factory | The **keyed group**: identifies which keyed partition/group this state belongs to. One keyGroup per logical “keyed group” or state kind (e.g. one group for “counters”, another for “sessions”). Same keyed group ⇒ same keyGroup bytes; different groups ⇒ different keyGroups. | +| **key** | `primaryKey` in factory methods (`NewKeyedValue(primaryKey, ...)`, `NewKeyedList(primaryKey, namespace)`, etc.) | The **value of the key**: the stream key for the current record, serialized as bytes. This is the key that partitioned the stream (e.g. user ID, partition key). Each distinct key value gets isolated state. | +| **namespace** | `namespace` ([]byte) in factory methods | **If a window function is present**, use the **window identifier as bytes** (e.g. serialized window bounds or window ID) so that state is scoped per key *and* per window. **Without windows**, pass **empty bytes** (e.g. `nil` or `[]byte{}`). | + +**Summary:** **keyGroup** = keyed group; **key** (primaryKey) = the key’s value (stream key); **namespace** = window bytes when using window functions, otherwise **empty bytes**. + +**Factory constructor summary (keyed):** + +| Factory | With codec | AutoCodec | +|---------|------------|-----------| +| KeyedValueStateFactory[V] | `NewKeyedValueStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec)` | `NewKeyedValueStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup)` | +| KeyedListStateFactory[V] | `NewKeyedListStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec)` | `NewKeyedListStateFactoryAutoCodecFromContext[V](ctx, storeName, keyGroup)` | +| KeyedMapStateFactory[MK,MV] | `NewKeyedMapStateFactoryFromContext(ctx, storeName, keyGroup, keyCodec, valueCodec)` | `NewKeyedMapStateFactoryFromContextAutoCodec[MK,MV](ctx, storeName, keyGroup)` | +| KeyedPriorityQueueStateFactory[V] | `NewKeyedPriorityQueueStateFactoryFromContext(ctx, storeName, keyGroup, itemCodec)` | `NewKeyedPriorityQueueStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup)` | +| KeyedAggregatingStateFactory[T,ACC,R] | `NewKeyedAggregatingStateFactoryFromContext(ctx, storeName, keyGroup, accCodec, aggFunc)` | `NewKeyedAggregatingStateFactoryFromContextAutoCodec(ctx, storeName, keyGroup, aggFunc)` | +| KeyedReducingStateFactory[V] | `NewKeyedReducingStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec, reduceFunc)` | `NewKeyedReducingStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup, reduceFunc)` | + +**Obtaining per-key state from a factory:** + +- **KeyedValueStateFactory[V]**: `NewKeyedValue(primaryKey []byte, stateName string) (*KeyedValueState[V], error)` — one value state per (primaryKey, stateName). +- **KeyedListStateFactory[V]**: `NewKeyedList(primaryKey []byte, namespace []byte) (*KeyedListState[V], error)`. +- **KeyedMapStateFactory[MK,MV]**: `NewKeyedMap(primaryKey []byte, mapName string) (*KeyedMapState[MK,MV], error)`. +- **KeyedPriorityQueueStateFactory[V]**: `NewKeyedPriorityQueue(primaryKey []byte, namespace []byte) (*KeyedPriorityQueueState[V], error)`. +- **KeyedAggregatingStateFactory**: `NewAggregatingState(primaryKey []byte, stateName string) (*KeyedAggregatingState[T,ACC,R], error)`. +- **KeyedReducingStateFactory[V]**: `NewReducingState(primaryKey []byte, namespace []byte) (*KeyedReducingState[V], error)`. + +Here **primaryKey** is the key (stream key value); **namespace** is the window bytes when using window functions, or **empty bytes** when not using windows. + +**keyGroup** partitions the key space (e.g. one keyGroup per logical state name or feature). Keep it stable for a given logical state; use different keyGroups for different state entities that share the same store. **primaryKey** in factory methods is the key of the current record in a keyed stream — pass the key your keyed operator received (e.g. from the key extractor or message metadata). + +### 7.8 Error Handling and Best Practices + +- **Errors from state API**: Creation and methods return errors compatible with `fssdk.SDKError` (e.g. `ErrStoreInternal`, `ErrStoreIO`). Codec encode/decode errors are wrapped (e.g. `"encode value state failed"`). Always check and handle errors in production. +- **Store naming**: Use stable, unique store names per logical state (e.g. `"counters"`, `"user-sessions"`). Same name in the same runtime refers to the same store. +- **Caching state**: You may create a state instance once in `Init` and reuse it in `Process`, or create it per message. Creating per message is safe and keeps code simple when you do not need to amortize creation cost. +- **KeyGroup design**: For keyed state, use a consistent keyGroup per “logical table” (e.g. `[]byte("orders")`). primaryKey is the **stream key** in keyed operators — use the key that identifies the current record (e.g. from key extractor or message key). When using **window functions**, pass the window identifier as **namespace** (e.g. serialized window bounds) so state is per key and per window. +- **Ordered codec**: For MapState and PriorityQueueState with AutoCodec, use primitive key/element types. For custom struct keys, implement a `Codec[K]` with `IsOrderedKeyCodec() == true` and use the “with codec” constructor. + +### 7.9 Example: ValueState with AutoCodec (counter) + +```go +import ( + fssdk "github.com/functionstream/function-stream/go-sdk" + "github.com/functionstream/function-stream/go-sdk/state/structures" +) + +func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + valState, err := structures.NewValueStateFromContextAutoCodec[int64](ctx, "my-store") + if err != nil { + return err + } + cur, _, _ := valState.Value() + if err := valState.Update(cur + 1); err != nil { + return err + } + // ... +} +``` + +### 7.10 Example: Keyed list factory (keyed operator) + +When your operator runs on a **keyed stream** (partitioned by key), use a Keyed list factory and pass the **stream key** as primaryKey for each message: + +```go +import ( + fssdk "github.com/functionstream/function-stream/go-sdk" + "github.com/functionstream/function-stream/go-sdk/state/keyed" +) + +type Order struct { Id string; Amount int64 } + +func (p *MyProcessor) Init(ctx fssdk.Context, config map[string]string) error { + keyGroup := []byte("orders") + factory, err := keyed.NewKeyedListStateFactoryAutoCodecFromContext[Order](ctx, "app-store", keyGroup) + if err != nil { + return err + } + p.listFactory = factory + return nil +} + +func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + userID := parseUserID(data) // []byte + list, err := p.listFactory.NewKeyedList(userID, "items") + if err != nil { + return err + } + if err := list.Add(Order{Id: "1", Amount: 100}); err != nil { + return err + } + items, err := list.Get() + if err != nil { + return err + } + // use items... +} +``` + +### 7.11 Example: MapState and AggregatingState (sum) + +```go +// MapState: string -> int64 count (both have ordered default codecs) +m, err := structures.NewMapStateFromContextAutoCodec[string, int64](ctx, "counts") +if err != nil { return err } +_ = m.Put("a", 1) +v, ok, _ := m.Get("a") + +// AggregatingState: sum of int64 with AutoCodec (ACC = int64, R = int64) +type sumAgg struct{} +func (sumAgg) CreateAccumulator() int64 { return 0 } +func (sumAgg) Add(v int64, acc int64) int64 { return acc + v } +func (sumAgg) GetResult(acc int64) int64 { return acc } +func (sumAgg) Merge(a, b int64) int64 { return a + b } +agg, err := structures.NewAggregatingStateFromContextAutoCodec[int64, int64, int64](ctx, "sum-store", sumAgg{}) +if err != nil { return err } +_ = agg.Add(10) +total, _, _ := agg.Get() +``` + +--- + +## 8. Directory Layout + +```text +go-sdk/ +├── Makefile # env / wit / bindings / build +├── fssdk.go # Entry point and type re-exports +├── go.mod / go.sum +├── api/ # Interfaces and error codes +│ ├── driver.go # Driver, BaseDriver +│ ├── context.go # Context +│ ├── store.go # Store, Iterator, ComplexKey +│ └── errors.go # ErrorCode, SDKError +├── impl/ # Bridge to WASM host +│ ├── runtime.go +│ ├── context.go +│ └── store.go +├── state/ # Advanced state API +│ ├── structures/ # ValueState, ListState, MapState, PriorityQueue, Aggregating, Reducing +│ ├── keyed/ # Keyed state factories (value, list, map, PQ, aggregating, reducing) +│ ├── codec/ # Codec[T], DefaultCodecFor, built-in and JSON codecs +│ └── common/ # Shared helpers (e.g. DupBytes) +├── wit/ # processor.wit and deps (make wit) +└── bindings/ # Generated by wit-bindgen-go (make bindings) +``` + +For more examples and SQL operations, see [examples/go-processor/README.md](../../examples/go-processor/README.md) and the [SQL CLI Guide](../sql-cli-guide.md). diff --git a/docs/python-sdk-guide-zh.md b/docs/Python-SDK/python-sdk-guide-zh.md similarity index 100% rename from docs/python-sdk-guide-zh.md rename to docs/Python-SDK/python-sdk-guide-zh.md diff --git a/docs/python-sdk-guide.md b/docs/Python-SDK/python-sdk-guide.md similarity index 100% rename from docs/python-sdk-guide.md rename to docs/Python-SDK/python-sdk-guide.md diff --git a/docs/go-sdk-guide-zh.md b/docs/go-sdk-guide-zh.md deleted file mode 100644 index 749b3ef6..00000000 --- a/docs/go-sdk-guide-zh.md +++ /dev/null @@ -1,324 +0,0 @@ - - -# Go SDK 指南 - -Function Stream 为 Go 开发者提供基于 WebAssembly（WASI P2）的算子开发 SDK。使用 TinyGo 将 Go 代码编译为 WASM 组件，在服务端沙箱中运行，具备 KV 状态存储与数据发射能力。 - ---- - -## 一、SDK 核心组件 - -| 组件 | 定位 | 说明 | -|--------------|--------|--------------------------------------------------------------| -| **fssdk** | 主包 | 对外入口，提供 `Driver`、`Context`、`Store` 等类型及 `Run(driver)`。 | -| **api** | 接口定义 | 定义 `Driver`、`Context`、`Store`、`Iterator`、`ComplexKey` 及错误码。 | -| **impl** | 运行时实现 | 将 Driver 与 WASM 宿主（processor WIT）桥接，内部使用。 | -| **bindings** | WIT 绑定 | 由 wit-bindgen-go 根据 `wit/processor.wit` 生成的 Go 绑定，供 impl 调用。 | - -Go 算子**仅依赖 fssdk**：实现 `Driver`（或嵌入 `BaseDriver`），在 `init()` 中调用 `fssdk.Run(&YourProcessor{})` 即可。 - ---- - -## 二、Driver 接口与 BaseDriver - -### 2.1 Driver 接口 - -所有 Go 算子必须实现 `fssdk.Driver` 接口。运行时在对应时机调用以下方法： - -| 方法 | 触发时机 | 说明 | -|----------------------------------------------|-------------|-------------------------------------| -| `Init(ctx, config)` | 函数启动时执行一次 | 初始化状态、获取 Store、解析 config。 | -| `Process(ctx, sourceID, data)` | 每收到一条消息时 | 核心处理逻辑：计算、状态读写、`ctx.Emit()`。 | -| `ProcessWatermark(ctx, sourceID, watermark)` | 收到水位线事件时 | 处理基于时间的窗口或乱序重排，可转发 `EmitWatermark`。 | -| `TakeCheckpoint(ctx, checkpointID)` | 系统做状态备份时 | 可持久化额外内存状态，保证强一致性。 | -| `CheckHeartbeat(ctx)` | 定期健康检查 | 返回 `false` 会触发算子重启。 | -| `Close(ctx)` | 函数关闭时 | 释放资源、清空引用。 | -| `Exec(ctx, className, modules)` | 扩展能力（可选） | 动态加载模块等，默认可不实现。 | -| `Custom(ctx, payload)` | 自定义 RPC（可选） | 请求/响应自定义字节，默认返回 payload 副本。 | - -### 2.2 BaseDriver - -`fssdk.BaseDriver` 为上述所有方法提供空实现。嵌入后只需重写你关心的方法： - -```go -type MyProcessor struct { - fssdk.BaseDriver - store fssdk.Store -} - -func (p *MyProcessor) Init(ctx fssdk.Context, config map[string]string) error { - store, err := ctx.GetOrCreateStore("my-store") - if err != nil { - return err - } - p.store = store - return nil -} - -func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { - // 业务逻辑，读写 p.store，最后 ctx.Emit(...) - return ctx.Emit(0, result) -} -``` - -### 2.3 注册入口 - -在 `init()` 中调用 `fssdk.Run`，传入你的 Driver 实例（通常为单例）： - -```go -func init() { - fssdk.Run(&MyProcessor{}) -} -``` - ---- - -## 三、Context 与 Store - -### 3.1 Context - -`fssdk.Context` 在 Init / Process / ProcessWatermark 等回调中传入，提供： - -| 方法 | 说明 | -|----------------------------------------------------------|---------------------------------------| -| `Emit(targetID uint32, data []byte) error` | 将数据发往指定输出通道。 | -| `EmitWatermark(targetID uint32, watermark uint64) error` | 发射水位线。 | -| `GetOrCreateStore(name string) (Store, error)` | 按名称获取或创建 KV Store（基于 RocksDB）。 | -| `Config() map[string]string` | 获取启动时下发的配置（对应 config.yaml 中的 init 等）。 | -| `Close() error` | 关闭 Context，一般由运行时管理。 | - -### 3.2 Store（KV 状态） - -`fssdk.Store` 提供键值存储与复杂键能力： - -**基础 API：** - -- `PutState(key, value []byte) error` -- `GetState(key []byte) (value []byte, found bool, err error)` -- `DeleteState(key []byte) error` -- `ListStates(startInclusive, endExclusive []byte) ([][]byte, error)`：按字节序范围列出键。 - -**ComplexKey（多维键/前缀扫描）：** - -- `Put(key ComplexKey, value []byte)` / `Get` / `Delete` / `Merge` / `DeletePrefix` -- `ListComplex(...)`：按 keyGroup、key、namespace 及范围列出 UserKey。 -- `ScanComplex(keyGroup, key, namespace []byte) (Iterator, error)`：返回迭代器，适用大范围扫描。 - -`ComplexKey` 结构包含 `KeyGroup`、`Key`、`Namespace`、`UserKey`，用于多维索引与前缀查询。 - -### 3.3 Iterator - -`Store.ScanComplex` 返回的 `fssdk.Iterator` 接口： - -- `HasNext() (bool, error)` -- `Next() (key, value []byte, ok bool, err error)` -- `Close() error`（用毕须调用以释放资源） - ---- - -## 四、生产级示例（词频统计） - -```go -package main - -import ( - "encoding/json" - fssdk "github.com/functionstream/function-stream/go-sdk" - "strconv" - "strings" -) - -func init() { - fssdk.Run(&CounterProcessor{}) -} - -type CounterProcessor struct { - fssdk.BaseDriver - store fssdk.Store - counterMap map[string]int64 - totalProcessed int64 - keyPrefix string -} - -func (p *CounterProcessor) Init(ctx fssdk.Context, config map[string]string) error { - store, err := ctx.GetOrCreateStore("counter-store") - if err != nil { - return err - } - p.store = store - p.counterMap = make(map[string]int64) - p.totalProcessed = 0 - p.keyPrefix = strings.TrimSpace(config["key_prefix"]) - return nil -} - -func (p *CounterProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { - inputStr := strings.TrimSpace(string(data)) - if inputStr == "" { - return nil - } - p.totalProcessed++ - - fullKey := p.keyPrefix + inputStr - existing, found, err := p.store.GetState([]byte(fullKey)) - if err != nil { - return err - } - currentCount := int64(0) - if found { - if n, e := strconv.ParseInt(string(existing), 10, 64); e == nil { - currentCount = n - } - } - newCount := currentCount + 1 - p.counterMap[inputStr] = newCount - if err = p.store.PutState([]byte(fullKey), []byte(strconv.FormatInt(newCount, 10))); err != nil { - return err - } - - out := map[string]interface{}{ - "total_processed": p.totalProcessed, - "counter_map": p.counterMap, - } - jsonBytes, _ := json.Marshal(out) - return ctx.Emit(0, jsonBytes) -} - -func (p *CounterProcessor) ProcessWatermark(ctx fssdk.Context, sourceID uint32, watermark uint64) error { - return ctx.EmitWatermark(0, watermark) -} - -func (p *CounterProcessor) Close(ctx fssdk.Context) error { - p.store = nil - p.counterMap = nil - return nil -} - -func main() {} -``` - ---- - -## 五、构建与部署 - -### 5.1 环境要求 - -- **Go**：1.23+ -- **TinyGo**：0.40+（支持 WASI P2） -- **wit-bindgen-go**：`go install go.bytecodealliance.org/cmd/wit-bindgen-go@latest` -- **wkg**（可选）：用于拉取 WIT 依赖，`cargo install wkg --version 0.10.0` -- **wasm-tools**：用于组件校验等，`cargo install wasm-tools` - -### 5.2 生成绑定并构建 - -在项目根目录下： - -```bash -# 安装 Go SDK 工具链（wit-bindgen-go、wkg 等） -make -C go-sdk env - -# 生成 WIT 绑定（从 wit/processor.wit 及 deps） -make -C go-sdk bindings - -# 运行测试 -make -C go-sdk build -``` - -算子项目（如 `examples/go-processor`）中： - -```bash -# 使用 TinyGo 编译为 WASI P2 组件 -tinygo build -o build/processor.wasm -target=wasi . -``` - -具体参数见 `examples/go-processor/build.sh`。 - -### 5.3 注册与运行 - -将编译得到的 `processor.wasm` 与 `config.yaml` 通过 SQL CLI 注册为 function： - -```sql -create function with ( - 'function_path'='/path/to/build/processor.wasm', - 'config_path'='/path/to/config.yaml' -); -``` - -config.yaml 中需配置 `name`、`type: processor`、`input-groups`、`outputs`（如 Kafka）。详见 [Function 配置](function-configuration-zh.md) 与 [examples/go-processor/README.md](../examples/go-processor/README.md)。 - ---- - -## 六、错误码与异常处理 - -SDK 通过 `fssdk.SDKError` 返回错误，可用 `errors.As` 获取 `Code`： - -| 错误码 | 含义 | -|----------------------------|----------------| -| `ErrRuntimeInvalidDriver` | 传入的 Driver 无效。 | -| `ErrRuntimeNotInitialized` | 运行时未初始化即调用。 | -| `ErrRuntimeClosed` | 运行时已关闭。 | -| `ErrStoreInvalidName` | Store 名称不合法。 | -| `ErrStoreInternal` | Store 内部错误。 | -| `ErrStoreNotFound` | 未找到指定 Store。 | -| `ErrStoreIO` | Store 读写异常。 | -| `ErrResultUnexpected` | 宿主返回了意外结果。 | - -处理示例： - -```go -if err != nil { - var sdkErr *fssdk.SDKError - if errors.As(err, &sdkErr) { - switch sdkErr.Code { - case fssdk.ErrStoreNotFound: - // 按业务决定是否创建或返回 - default: - // 记录并向上返回 - } - } - return err -} -``` - ---- - -## 七、目录结构参考 - -```text -go-sdk/ -├── Makefile # env / wit / bindings / build -├── fssdk.go # 对外入口与类型重导出 -├── go.mod / go.sum -├── api/ # 接口与错误码 -│ ├── driver.go # Driver、BaseDriver -│ ├── context.go # Context -│ ├── store.go # Store、Iterator、ComplexKey -│ └── errors.go # ErrorCode、SDKError -├── impl/ # 与 WASM 宿主桥接 -│ ├── runtime.go -│ ├── context.go -│ └── store.go -├── wit/ # processor.wit 及依赖（可由 make wit 生成） -└── bindings/ # wit-bindgen-go 生成的 Go 代码（make bindings） -``` - -更多示例与 SQL 操作见 [examples/go-processor/README.md](../examples/go-processor/README.md)、[SQL CLI 指南](sql-cli-guide-zh.md)。 diff --git a/docs/go-sdk-guide.md b/docs/go-sdk-guide.md deleted file mode 100644 index 641e1017..00000000 --- a/docs/go-sdk-guide.md +++ /dev/null @@ -1,324 +0,0 @@ - - -# Go SDK Guide - -Function Stream provides a WebAssembly (WASI P2) based operator SDK for Go developers. Go code is compiled to a WASM component with TinyGo and runs in the server sandbox with KV state storage and data emission support. - ---- - -## 1. SDK Core Components - -| Component | Role | Description | -|--------------|------------------------|----------------------------------------------------------------------------------| -| **fssdk** | Main package | Public entry point; exposes `Driver`, `Context`, `Store`, and `Run(driver)`. | -| **api** | Interface definitions | Defines `Driver`, `Context`, `Store`, `Iterator`, `ComplexKey`, and error codes. | -| **impl** | Runtime implementation | Bridges the Driver to the WASM host (processor WIT); internal use. | -| **bindings** | WIT bindings | Go code generated by wit-bindgen-go from `wit/processor.wit`; used by impl. | - -Operators **only depend on fssdk**: implement `Driver` (or embed `BaseDriver`) and call `fssdk.Run(&YourProcessor{})` from `init()`. - ---- - -## 2. Driver Interface and BaseDriver - -### 2.1 Driver Interface - -All Go operators must implement the `fssdk.Driver` interface. The runtime invokes the following methods at the appropriate times: - -| Method | When invoked | Description | -|----------------------------------------------|--------------------------------|-----------------------------------------------------------------------------------| -| `Init(ctx, config)` | Once when the function starts | Initialize state, obtain Store, parse config. | -| `Process(ctx, sourceID, data)` | For each incoming message | Core logic: compute, state read/write, `ctx.Emit()`. | -| `ProcessWatermark(ctx, sourceID, watermark)` | On watermark events | Handle time-based windows or out-of-order logic; may forward via `EmitWatermark`. | -| `TakeCheckpoint(ctx, checkpointID)` | When the system backs up state | Persist extra in-memory state for strong consistency. | -| `CheckHeartbeat(ctx)` | Periodic health check | Return `false` to trigger operator restart. | -| `Close(ctx)` | When the function shuts down | Release resources and clear references. | -| `Exec(ctx, className, modules)` | Optional extension | Dynamic module loading, etc.; default no-op. | -| `Custom(ctx, payload)` | Optional custom RPC | Request/response custom bytes; default returns a copy of payload. | - -### 2.2 BaseDriver - -`fssdk.BaseDriver` provides no-op implementations for all of the above. Embed it and override only the methods you need: - -```go -type MyProcessor struct { - fssdk.BaseDriver - store fssdk.Store -} - -func (p *MyProcessor) Init(ctx fssdk.Context, config map[string]string) error { - store, err := ctx.GetOrCreateStore("my-store") - if err != nil { - return err - } - p.store = store - return nil -} - -func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { - // Your logic; read/write p.store, then ctx.Emit(...) - return ctx.Emit(0, result) -} -``` - -### 2.3 Registration Entry Point - -Call `fssdk.Run` from `init()` with your Driver instance (typically a singleton): - -```go -func init() { - fssdk.Run(&MyProcessor{}) -} -``` - ---- - -## 3. Context and Store - -### 3.1 Context - -`fssdk.Context` is passed into Init, Process, ProcessWatermark, etc. It provides: - -| Method | Description | -|----------------------------------------------------------|-------------------------------------------------------------| -| `Emit(targetID uint32, data []byte) error` | Send data to the given output channel. | -| `EmitWatermark(targetID uint32, watermark uint64) error` | Emit a watermark. | -| `GetOrCreateStore(name string) (Store, error)` | Get or create a KV Store by name (RocksDB-backed). | -| `Config() map[string]string` | Startup configuration (e.g. from config.yaml init section). | -| `Close() error` | Close the context; usually managed by the runtime. | - -### 3.2 Store (KV State) - -`fssdk.Store` provides key-value and complex-key operations: - -**Basic API:** - -- `PutState(key, value []byte) error` -- `GetState(key []byte) (value []byte, found bool, err error)` -- `DeleteState(key []byte) error` -- `ListStates(startInclusive, endExclusive []byte) ([][]byte, error)` — list keys in byte order range. - -**ComplexKey (multi-dimensional keys / prefix scan):** - -- `Put(key ComplexKey, value []byte)` / `Get` / `Delete` / `Merge` / `DeletePrefix` -- `ListComplex(...)` — list UserKeys by keyGroup, key, namespace and range. -- `ScanComplex(keyGroup, key, namespace []byte) (Iterator, error)` — returns an iterator for large scans. - -`ComplexKey` holds `KeyGroup`, `Key`, `Namespace`, and `UserKey` for multi-dimensional indexing and prefix queries. - -### 3.3 Iterator - -`Store.ScanComplex` returns an `fssdk.Iterator`: - -- `HasNext() (bool, error)` -- `Next() (key, value []byte, ok bool, err error)` -- `Close() error` — must be called when done to release resources. - ---- - -## 4. Production Example (Word Count) - -```go -package main - -import ( - "encoding/json" - fssdk "github.com/functionstream/function-stream/go-sdk" - "strconv" - "strings" -) - -func init() { - fssdk.Run(&CounterProcessor{}) -} - -type CounterProcessor struct { - fssdk.BaseDriver - store fssdk.Store - counterMap map[string]int64 - totalProcessed int64 - keyPrefix string -} - -func (p *CounterProcessor) Init(ctx fssdk.Context, config map[string]string) error { - store, err := ctx.GetOrCreateStore("counter-store") - if err != nil { - return err - } - p.store = store - p.counterMap = make(map[string]int64) - p.totalProcessed = 0 - p.keyPrefix = strings.TrimSpace(config["key_prefix"]) - return nil -} - -func (p *CounterProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { - inputStr := strings.TrimSpace(string(data)) - if inputStr == "" { - return nil - } - p.totalProcessed++ - - fullKey := p.keyPrefix + inputStr - existing, found, err := p.store.GetState([]byte(fullKey)) - if err != nil { - return err - } - currentCount := int64(0) - if found { - if n, e := strconv.ParseInt(string(existing), 10, 64); e == nil { - currentCount = n - } - } - newCount := currentCount + 1 - p.counterMap[inputStr] = newCount - if err = p.store.PutState([]byte(fullKey), []byte(strconv.FormatInt(newCount, 10))); err != nil { - return err - } - - out := map[string]interface{}{ - "total_processed": p.totalProcessed, - "counter_map": p.counterMap, - } - jsonBytes, _ := json.Marshal(out) - return ctx.Emit(0, jsonBytes) -} - -func (p *CounterProcessor) ProcessWatermark(ctx fssdk.Context, sourceID uint32, watermark uint64) error { - return ctx.EmitWatermark(0, watermark) -} - -func (p *CounterProcessor) Close(ctx fssdk.Context) error { - p.store = nil - p.counterMap = nil - return nil -} - -func main() {} -``` - ---- - -## 5. Build and Deployment - -### 5.1 Prerequisites - -- **Go**: 1.23+ -- **TinyGo**: 0.40+ (WASI P2 support) -- **wit-bindgen-go**: `go install go.bytecodealliance.org/cmd/wit-bindgen-go@latest` -- **wkg** (optional): for fetching WIT deps — `cargo install wkg --version 0.10.0` -- **wasm-tools**: for component validation — `cargo install wasm-tools` - -### 5.2 Generate Bindings and Build - -From the project root: - -```bash -# Install Go SDK toolchain (wit-bindgen-go, wkg, etc.) -make -C go-sdk env - -# Generate WIT bindings (from wit/processor.wit and deps) -make -C go-sdk bindings - -# Run tests -make -C go-sdk build -``` - -In your operator project (e.g. `examples/go-processor`): - -```bash -# Build as WASI P2 component with TinyGo -tinygo build -o build/processor.wasm -target=wasi . -``` - -See `examples/go-processor/build.sh` for the exact command. - -### 5.3 Register and Run - -Register the built `processor.wasm` and `config.yaml` as a function via the SQL CLI: - -```sql -create function with ( - 'function_path'='/path/to/build/processor.wasm', - 'config_path'='/path/to/config.yaml' -); -``` - -Configure `name`, `type: processor`, `input-groups`, and `outputs` (e.g. Kafka) in config.yaml. See [Function Configuration](function-configuration.md) and [examples/go-processor/README.md](../examples/go-processor/README.md). - ---- - -## 6. Error Codes and Handling - -The SDK returns errors as `fssdk.SDKError`; use `errors.As` to get the `Code`: - -| Code | Meaning | -|----------------------------|-------------------------------| -| `ErrRuntimeInvalidDriver` | Invalid Driver passed to Run. | -| `ErrRuntimeNotInitialized` | Runtime not initialized. | -| `ErrRuntimeClosed` | Runtime already closed. | -| `ErrStoreInvalidName` | Invalid Store name. | -| `ErrStoreInternal` | Store internal error. | -| `ErrStoreNotFound` | Store not found. | -| `ErrStoreIO` | Store I/O error. | -| `ErrResultUnexpected` | Unexpected result from host. | - -Example handling: - -```go -if err != nil { - var sdkErr *fssdk.SDKError - if errors.As(err, &sdkErr) { - switch sdkErr.Code { - case fssdk.ErrStoreNotFound: - // Handle: create or return - default: - // Log and propagate - } - } - return err -} -``` - ---- - -## 7. Directory Layout - -```text -go-sdk/ -├── Makefile # env / wit / bindings / build -├── fssdk.go # Entry point and type re-exports -├── go.mod / go.sum -├── api/ # Interfaces and error codes -│ ├── driver.go # Driver, BaseDriver -│ ├── context.go # Context -│ ├── store.go # Store, Iterator, ComplexKey -│ └── errors.go # ErrorCode, SDKError -├── impl/ # Bridge to WASM host -│ ├── runtime.go -│ ├── context.go -│ └── store.go -├── wit/ # processor.wit and deps (make wit) -└── bindings/ # Generated by wit-bindgen-go (make bindings) -``` - -For more examples and SQL operations, see [examples/go-processor/README.md](../examples/go-processor/README.md) and the [SQL CLI Guide](sql-cli-guide.md). diff --git a/go-sdk/impl/state.go b/go-sdk/impl/state.go deleted file mode 100644 index 530a03db..00000000 --- a/go-sdk/impl/state.go +++ /dev/null @@ -1,144 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package impl - -import ( - "github.com/functionstream/function-stream/go-sdk/api" - "github.com/functionstream/function-stream/go-sdk/state/codec" - "github.com/functionstream/function-stream/go-sdk/state/keyed" - "github.com/functionstream/function-stream/go-sdk/state/structures" -) - -func getStoreFromContext(ctx api.Context, storeName string) (*storeImpl, error) { - store, err := ctx.GetOrCreateStore(storeName) - if err != nil { - return nil, err - } - s, ok := store.(*storeImpl) - if !ok { - return nil, api.NewError(api.ErrStoreInternal, "store %q is not the default implementation", storeName) - } - return s, nil -} - -func NewValueState[T any](ctx api.Context, storeName string, valueCodec codec.Codec[T]) (*structures.ValueState[T], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return structures.NewValueState(s, valueCodec) -} - -func NewListState[T any](ctx api.Context, storeName string, itemCodec codec.Codec[T]) (*structures.ListState[T], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return structures.NewListState(s, itemCodec) -} - -func NewMapState[K any, V any](ctx api.Context, storeName string, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*structures.MapState[K, V], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return structures.NewMapState(s, keyCodec, valueCodec) -} - -func NewMapStateAutoKeyCodec[K any, V any](ctx api.Context, storeName string, valueCodec codec.Codec[V]) (*structures.MapState[K, V], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return structures.NewMapStateAutoKeyCodec[K, V](s, valueCodec) -} - -func NewPriorityQueueState[T any](ctx api.Context, storeName string, itemCodec codec.Codec[T]) (*structures.PriorityQueueState[T], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return structures.NewPriorityQueueState(s, itemCodec) -} - -func NewAggregatingState[T any, ACC any, R any](ctx api.Context, storeName string, accCodec codec.Codec[ACC], aggFunc structures.AggregateFunc[T, ACC, R]) (*structures.AggregatingState[T, ACC, R], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return structures.NewAggregatingState(s, accCodec, aggFunc) -} - -func NewReducingState[V any](ctx api.Context, storeName string, valueCodec codec.Codec[V], reduceFunc structures.ReduceFunc[V]) (*structures.ReducingState[V], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return structures.NewReducingState(s, valueCodec, reduceFunc) -} - -func NewKeyedListStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedListStateFactory[V], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return keyed.NewKeyedListStateFactory(s, keyGroup, valueCodec) -} - -func NewKeyedListStateFactoryAutoCodec[V any](ctx api.Context, storeName string, keyGroup []byte) (*keyed.KeyedListStateFactory[V], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return keyed.NewKeyedListStateFactoryAutoCodec[V](s, keyGroup) -} - -func NewKeyedValueStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedValueStateFactory[V], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return keyed.NewKeyedValueStateFactory(s, keyGroup, valueCodec) -} - -func NewKeyedMapStateFactory[MK any, MV any](ctx api.Context, storeName string, keyGroup []byte, keyCodec codec.Codec[MK], valueCodec codec.Codec[MV]) (*keyed.KeyedMapStateFactory[MK, MV], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return keyed.NewKeyedMapStateFactory(s, keyGroup, keyCodec, valueCodec) -} - -func NewKeyedPriorityQueueStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, itemCodec codec.Codec[V]) (*keyed.KeyedPriorityQueueStateFactory[V], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return keyed.NewKeyedPriorityQueueStateFactory(s, keyGroup, itemCodec) -} - -func NewKeyedAggregatingStateFactory[T any, ACC any, R any](ctx api.Context, storeName string, keyGroup []byte, accCodec codec.Codec[ACC], aggFunc keyed.AggregateFunc[T, ACC, R]) (*keyed.KeyedAggregatingStateFactory[T, ACC, R], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return keyed.NewKeyedAggregatingStateFactory(s, keyGroup, accCodec, aggFunc) -} - -func NewKeyedReducingStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V], reduceFunc keyed.ReduceFunc[V]) (*keyed.KeyedReducingStateFactory[V], error) { - s, err := getStoreFromContext(ctx, storeName) - if err != nil { - return nil, err - } - return keyed.NewKeyedReducingStateFactory(s, keyGroup, valueCodec, reduceFunc) -} diff --git a/go-sdk/state/keyed/keyed_aggregating_state.go b/go-sdk/state/keyed/keyed_aggregating_state.go index 748ed2b6..9014a3ac 100644 --- a/go-sdk/state/keyed/keyed_aggregating_state.go +++ b/go-sdk/state/keyed/keyed_aggregating_state.go @@ -34,7 +34,29 @@ type KeyedAggregatingStateFactory[T any, ACC any, R any] struct { aggFunc AggregateFunc[T, ACC, R] } -func NewKeyedAggregatingStateFactory[T any, ACC any, R any]( +// NewKeyedAggregatingStateFactoryFromContext creates a KeyedAggregatingStateFactory using the store from ctx.GetOrCreateStore(storeName). +func NewKeyedAggregatingStateFactoryFromContext[T any, ACC any, R any](ctx api.Context, storeName string, keyGroup []byte, accCodec codec.Codec[ACC], aggFunc AggregateFunc[T, ACC, R]) (*KeyedAggregatingStateFactory[T, ACC, R], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newKeyedAggregatingStateFactory(store, keyGroup, accCodec, aggFunc) +} + +// NewKeyedAggregatingStateFactoryFromContextAutoCodec creates a KeyedAggregatingStateFactory with default accumulator codec from ctx.GetOrCreateStore(storeName). +func NewKeyedAggregatingStateFactoryFromContextAutoCodec[T any, ACC any, R any](ctx api.Context, storeName string, keyGroup []byte, aggFunc AggregateFunc[T, ACC, R]) (*KeyedAggregatingStateFactory[T, ACC, R], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + accCodec, err := codec.DefaultCodecFor[ACC]() + if err != nil { + return nil, err + } + return newKeyedAggregatingStateFactory(store, keyGroup, accCodec, aggFunc) +} + +func newKeyedAggregatingStateFactory[T any, ACC any, R any]( store common.Store, keyGroup []byte, accCodec codec.Codec[ACC], diff --git a/go-sdk/state/keyed/keyed_list_state.go b/go-sdk/state/keyed/keyed_list_state.go index b33b44f0..e370fa6c 100644 --- a/go-sdk/state/keyed/keyed_list_state.go +++ b/go-sdk/state/keyed/keyed_list_state.go @@ -29,7 +29,25 @@ type KeyedListStateFactory[V any] struct { isFixed bool } -func NewKeyedListStateFactory[V any](store common.Store,keyGroup []byte, valueCodec codec.Codec[V]) (*KeyedListStateFactory[V], error) { +// NewKeyedListStateFactoryFromContext creates a KeyedListStateFactory using the store from ctx.GetOrCreateStore(storeName). +func NewKeyedListStateFactoryFromContext[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V]) (*KeyedListStateFactory[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newKeyedListStateFactory(store, keyGroup, valueCodec) +} + +// NewKeyedListStateFactoryAutoCodecFromContext creates a KeyedListStateFactory with default value codec using the store from context. +func NewKeyedListStateFactoryAutoCodecFromContext[V any](ctx api.Context, storeName string, keyGroup []byte) (*KeyedListStateFactory[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newKeyedListStateFactoryAutoCodec[V](store, keyGroup) +} + +func newKeyedListStateFactory[V any](store common.Store, keyGroup []byte, valueCodec codec.Codec[V]) (*KeyedListStateFactory[V], error) { if store == nil { return nil, api.NewError(api.ErrStoreInternal, "keyed list state factory store must not be nil") } @@ -49,12 +67,12 @@ func NewKeyedListStateFactory[V any](store common.Store,keyGroup []byte, valueCo }, nil } -func NewKeyedListStateFactoryAutoCodec[V any](store common.Store, keyGroup []byte) (*KeyedListStateFactory[V], error) { +func newKeyedListStateFactoryAutoCodec[V any](store common.Store, keyGroup []byte) (*KeyedListStateFactory[V], error) { valueCodec, err := codec.DefaultCodecFor[V]() if err != nil { return nil, err } - return NewKeyedListStateFactory[V](store, keyGroup, valueCodec) + return newKeyedListStateFactory[V](store, keyGroup, valueCodec) } type KeyedListState[V any] struct { diff --git a/go-sdk/state/keyed/keyed_map_state.go b/go-sdk/state/keyed/keyed_map_state.go index 937cffe8..fbc92132 100644 --- a/go-sdk/state/keyed/keyed_map_state.go +++ b/go-sdk/state/keyed/keyed_map_state.go @@ -28,7 +28,33 @@ type KeyedMapStateFactory[MK any, MV any] struct { mapValueCodec codec.Codec[MV] } -func NewKeyedMapStateFactory[MK any, MV any]( +// NewKeyedMapStateFactoryFromContext creates a KeyedMapStateFactory using the store from ctx.GetOrCreateStore(storeName). +func NewKeyedMapStateFactoryFromContext[MK any, MV any](ctx api.Context, storeName string, keyGroup []byte, keyCodec codec.Codec[MK], valueCodec codec.Codec[MV]) (*KeyedMapStateFactory[MK, MV], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newKeyedMapStateFactory(store, keyGroup, keyCodec, valueCodec) +} + +// NewKeyedMapStateFactoryFromContextAutoCodec creates a KeyedMapStateFactory with default map-key and map-value codecs. MK must have an ordered default codec. +func NewKeyedMapStateFactoryFromContextAutoCodec[MK any, MV any](ctx api.Context, storeName string, keyGroup []byte) (*KeyedMapStateFactory[MK, MV], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + mapKeyCodec, err := codec.DefaultCodecFor[MK]() + if err != nil { + return nil, err + } + mapValueCodec, err := codec.DefaultCodecFor[MV]() + if err != nil { + return nil, err + } + return newKeyedMapStateFactory(store, keyGroup, mapKeyCodec, mapValueCodec) +} + +func newKeyedMapStateFactory[MK any, MV any]( store common.Store, keyGroup []byte, mapKeyCodec codec.Codec[MK], diff --git a/go-sdk/state/keyed/keyed_priority_queue_state.go b/go-sdk/state/keyed/keyed_priority_queue_state.go index 49273755..f18f8112 100644 --- a/go-sdk/state/keyed/keyed_priority_queue_state.go +++ b/go-sdk/state/keyed/keyed_priority_queue_state.go @@ -27,7 +27,29 @@ type KeyedPriorityQueueStateFactory[V any] struct { valueCodec codec.Codec[V] } -func NewKeyedPriorityQueueStateFactory[V any]( +// NewKeyedPriorityQueueStateFactoryFromContext creates a KeyedPriorityQueueStateFactory using the store from ctx.GetOrCreateStore(storeName). +func NewKeyedPriorityQueueStateFactoryFromContext[V any](ctx api.Context, storeName string, keyGroup []byte, itemCodec codec.Codec[V]) (*KeyedPriorityQueueStateFactory[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newKeyedPriorityQueueStateFactory(store, keyGroup, itemCodec) +} + +// NewKeyedPriorityQueueStateFactoryFromContextAutoCodec creates a KeyedPriorityQueueStateFactory with default value codec. V must have an ordered default codec. +func NewKeyedPriorityQueueStateFactoryFromContextAutoCodec[V any](ctx api.Context, storeName string, keyGroup []byte) (*KeyedPriorityQueueStateFactory[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + valueCodec, err := codec.DefaultCodecFor[V]() + if err != nil { + return nil, err + } + return newKeyedPriorityQueueStateFactory(store, keyGroup, valueCodec) +} + +func newKeyedPriorityQueueStateFactory[V any]( store common.Store, keyGroup []byte, valueCodec codec.Codec[V], diff --git a/go-sdk/state/keyed/keyed_reducing_state.go b/go-sdk/state/keyed/keyed_reducing_state.go index 45a854bf..64ee2aee 100644 --- a/go-sdk/state/keyed/keyed_reducing_state.go +++ b/go-sdk/state/keyed/keyed_reducing_state.go @@ -29,7 +29,29 @@ type KeyedReducingStateFactory[V any] struct { reduceFunc ReduceFunc[V] } -func NewKeyedReducingStateFactory[V any]( +// NewKeyedReducingStateFactoryFromContext creates a KeyedReducingStateFactory using the store from ctx.GetOrCreateStore(storeName). +func NewKeyedReducingStateFactoryFromContext[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V], reduceFunc ReduceFunc[V]) (*KeyedReducingStateFactory[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newKeyedReducingStateFactory(store, keyGroup, valueCodec, reduceFunc) +} + +// NewKeyedReducingStateFactoryFromContextAutoCodec creates a KeyedReducingStateFactory with default value codec from ctx.GetOrCreateStore(storeName). +func NewKeyedReducingStateFactoryFromContextAutoCodec[V any](ctx api.Context, storeName string, keyGroup []byte, reduceFunc ReduceFunc[V]) (*KeyedReducingStateFactory[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + valueCodec, err := codec.DefaultCodecFor[V]() + if err != nil { + return nil, err + } + return newKeyedReducingStateFactory(store, keyGroup, valueCodec, reduceFunc) +} + +func newKeyedReducingStateFactory[V any]( store common.Store, keyGroup []byte, valueCodec codec.Codec[V], diff --git a/go-sdk/state/keyed/keyed_value_state.go b/go-sdk/state/keyed/keyed_value_state.go index b8bb9987..1ff1d5db 100644 --- a/go-sdk/state/keyed/keyed_value_state.go +++ b/go-sdk/state/keyed/keyed_value_state.go @@ -26,7 +26,29 @@ type KeyedValueStateFactory[V any] struct { valueCodec codec.Codec[V] } -func NewKeyedValueStateFactory[V any]( +// NewKeyedValueStateFactoryFromContext creates a KeyedValueStateFactory using the store from ctx.GetOrCreateStore(storeName). +func NewKeyedValueStateFactoryFromContext[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V]) (*KeyedValueStateFactory[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newKeyedValueStateFactory(store, keyGroup, valueCodec) +} + +// NewKeyedValueStateFactoryFromContextAutoCodec creates a KeyedValueStateFactory with default value codec from ctx.GetOrCreateStore(storeName). +func NewKeyedValueStateFactoryFromContextAutoCodec[V any](ctx api.Context, storeName string, keyGroup []byte) (*KeyedValueStateFactory[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + valueCodec, err := codec.DefaultCodecFor[V]() + if err != nil { + return nil, err + } + return newKeyedValueStateFactory(store, keyGroup, valueCodec) +} + +func newKeyedValueStateFactory[V any]( store common.Store, keyGroup []byte, valueCodec codec.Codec[V], diff --git a/go-sdk/state/structures/aggregating.go b/go-sdk/state/structures/aggregating.go index ab6f3270..b2d30a98 100644 --- a/go-sdk/state/structures/aggregating.go +++ b/go-sdk/state/structures/aggregating.go @@ -34,7 +34,38 @@ type AggregatingState[T any, ACC any, R any] struct { aggFunc AggregateFunc[T, ACC, R] } -func NewAggregatingState[T any, ACC any, R any]( +// NewAggregatingStateFromContext creates an AggregatingState using the store from ctx.GetOrCreateStore(storeName). +func NewAggregatingStateFromContext[T any, ACC any, R any]( + ctx api.Context, + storeName string, + accCodec codec.Codec[ACC], + aggFunc AggregateFunc[T, ACC, R], +) (*AggregatingState[T, ACC, R], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newAggregatingState(store, accCodec, aggFunc) +} + +// NewAggregatingStateFromContextAutoCodec creates an AggregatingState with default accumulator codec from ctx.GetOrCreateStore(storeName). +func NewAggregatingStateFromContextAutoCodec[T any, ACC any, R any]( + ctx api.Context, + storeName string, + aggFunc AggregateFunc[T, ACC, R], +) (*AggregatingState[T, ACC, R], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + accCodec, err := codec.DefaultCodecFor[ACC]() + if err != nil { + return nil, err + } + return newAggregatingState(store, accCodec, aggFunc) +} + +func newAggregatingState[T any, ACC any, R any]( store common.Store, accCodec codec.Codec[ACC], aggFunc AggregateFunc[T, ACC, R], diff --git a/go-sdk/state/structures/list.go b/go-sdk/state/structures/list.go index 83d2c845..b2630ee9 100644 --- a/go-sdk/state/structures/list.go +++ b/go-sdk/state/structures/list.go @@ -31,7 +31,29 @@ type ListState[T any] struct { serializeBatch func([]T) ([]byte, error) } -func NewListState[T any](store common.Store, itemCodec codec.Codec[T]) (*ListState[T], error) { +// NewListStateFromContext creates a ListState using the store from ctx.GetOrCreateStore(storeName). +func NewListStateFromContext[T any](ctx api.Context, storeName string, itemCodec codec.Codec[T]) (*ListState[T], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newListState(store, itemCodec) +} + +// NewListStateFromContextAutoCodec creates a ListState with default codec for T from ctx.GetOrCreateStore(storeName). +func NewListStateFromContextAutoCodec[T any](ctx api.Context, storeName string) (*ListState[T], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + itemCodec, err := codec.DefaultCodecFor[T]() + if err != nil { + return nil, err + } + return newListState(store, itemCodec) +} + +func newListState[T any](store common.Store, itemCodec codec.Codec[T]) (*ListState[T], error) { if store == nil { return nil, api.NewError(api.ErrStoreInternal, "list state store must not be nil") } diff --git a/go-sdk/state/structures/map.go b/go-sdk/state/structures/map.go index ff19bacc..a5adcecc 100644 --- a/go-sdk/state/structures/map.go +++ b/go-sdk/state/structures/map.go @@ -35,7 +35,42 @@ type MapState[K any, V any] struct { valueCodec codec.Codec[V] } -func NewMapState[K any, V any](store common.Store, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*MapState[K, V], error) { +// NewMapStateFromContext creates a MapState using the store from ctx.GetOrCreateStore(storeName). +func NewMapStateFromContext[K any, V any](ctx api.Context, storeName string, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*MapState[K, V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newMapState(store, keyCodec, valueCodec) +} + +// NewMapStateAutoKeyCodecFromContext creates a MapState with default key codec using the store from context. +func NewMapStateAutoKeyCodecFromContext[K any, V any](ctx api.Context, storeName string, valueCodec codec.Codec[V]) (*MapState[K, V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newMapStateAutoKeyCodec[K, V](store, valueCodec) +} + +// NewMapStateFromContextAutoCodec creates a MapState with default key and value codecs from ctx.GetOrCreateStore(storeName). Key type K must have an ordered default codec. +func NewMapStateFromContextAutoCodec[K any, V any](ctx api.Context, storeName string) (*MapState[K, V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + keyCodec, err := codec.DefaultCodecFor[K]() + if err != nil { + return nil, err + } + valueCodec, err := codec.DefaultCodecFor[V]() + if err != nil { + return nil, err + } + return newMapState(store, keyCodec, valueCodec) +} + +func newMapState[K any, V any](store common.Store, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*MapState[K, V], error) { if store == nil { return nil, api.NewError(api.ErrStoreInternal, "map state store must not be nil") } @@ -51,12 +86,12 @@ func NewMapState[K any, V any](store common.Store, keyCodec codec.Codec[K], valu return &MapState[K, V]{store: store, keyGroup: []byte{}, key: []byte{}, namespace: []byte{}, keyCodec: keyCodec, valueCodec: valueCodec}, nil } -func NewMapStateAutoKeyCodec[K any, V any](store common.Store, valueCodec codec.Codec[V]) (*MapState[K, V], error) { +func newMapStateAutoKeyCodec[K any, V any](store common.Store, valueCodec codec.Codec[V]) (*MapState[K, V], error) { autoKeyCodec, err := codec.DefaultCodecFor[K]() if err != nil { return nil, err } - return NewMapState[K, V](store, autoKeyCodec, valueCodec) + return newMapState[K, V](store, autoKeyCodec, valueCodec) } func (m *MapState[K, V]) Put(key K, value V) error { diff --git a/go-sdk/state/structures/priority_queue.go b/go-sdk/state/structures/priority_queue.go index 16518893..3e5d7781 100644 --- a/go-sdk/state/structures/priority_queue.go +++ b/go-sdk/state/structures/priority_queue.go @@ -30,8 +30,30 @@ type PriorityQueueState[T any] struct { valueCodec codec.Codec[T] } -// NewPriorityQueueState creates a priority queue state. itemCodec must support ordered key encoding. -func NewPriorityQueueState[T any](store common.Store, itemCodec codec.Codec[T]) (*PriorityQueueState[T], error) { +// NewPriorityQueueStateFromContext creates a PriorityQueueState using the store from ctx.GetOrCreateStore(storeName). +func NewPriorityQueueStateFromContext[T any](ctx api.Context, storeName string, itemCodec codec.Codec[T]) (*PriorityQueueState[T], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newPriorityQueueState(store, itemCodec) +} + +// NewPriorityQueueStateFromContextAutoCodec creates a PriorityQueueState with default codec for T. T must have an ordered default codec (e.g. primitive types). +func NewPriorityQueueStateFromContextAutoCodec[T any](ctx api.Context, storeName string) (*PriorityQueueState[T], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + itemCodec, err := codec.DefaultCodecFor[T]() + if err != nil { + return nil, err + } + return newPriorityQueueState(store, itemCodec) +} + +// newPriorityQueueState creates a priority queue state. itemCodec must support ordered key encoding. +func newPriorityQueueState[T any](store common.Store, itemCodec codec.Codec[T]) (*PriorityQueueState[T], error) { if store == nil { return nil, api.NewError(api.ErrStoreInternal, "priority queue state store must not be nil") } diff --git a/go-sdk/state/structures/reducing.go b/go-sdk/state/structures/reducing.go index 19ca5b9c..c96f0b53 100644 --- a/go-sdk/state/structures/reducing.go +++ b/go-sdk/state/structures/reducing.go @@ -29,7 +29,38 @@ type ReducingState[V any] struct { reduceFunc ReduceFunc[V] } -func NewReducingState[V any]( +// NewReducingStateFromContext creates a ReducingState using the store from ctx.GetOrCreateStore(storeName). +func NewReducingStateFromContext[V any]( + ctx api.Context, + storeName string, + valueCodec codec.Codec[V], + reduceFunc ReduceFunc[V], +) (*ReducingState[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newReducingState(store, valueCodec, reduceFunc) +} + +// NewReducingStateFromContextAutoCodec creates a ReducingState with default value codec from ctx.GetOrCreateStore(storeName). +func NewReducingStateFromContextAutoCodec[V any]( + ctx api.Context, + storeName string, + reduceFunc ReduceFunc[V], +) (*ReducingState[V], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + valueCodec, err := codec.DefaultCodecFor[V]() + if err != nil { + return nil, err + } + return newReducingState(store, valueCodec, reduceFunc) +} + +func newReducingState[V any]( store common.Store, valueCodec codec.Codec[V], reduceFunc ReduceFunc[V], diff --git a/go-sdk/state/structures/value.go b/go-sdk/state/structures/value.go index b77ab4f9..9b5e164a 100644 --- a/go-sdk/state/structures/value.go +++ b/go-sdk/state/structures/value.go @@ -26,7 +26,29 @@ type ValueState[T any] struct { codec codec.Codec[T] } -func NewValueState[T any](store common.Store, valueCodec codec.Codec[T]) (*ValueState[T], error) { +// NewValueStateFromContext creates a ValueState using the store from ctx.GetOrCreateStore(storeName). +func NewValueStateFromContext[T any](ctx api.Context, storeName string, valueCodec codec.Codec[T]) (*ValueState[T], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + return newValueState(store, valueCodec) +} + +// NewValueStateFromContextAutoCodec creates a ValueState with default codec for T from ctx.GetOrCreateStore(storeName). +func NewValueStateFromContextAutoCodec[T any](ctx api.Context, storeName string) (*ValueState[T], error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + valueCodec, err := codec.DefaultCodecFor[T]() + if err != nil { + return nil, err + } + return newValueState(store, valueCodec) +} + +func newValueState[T any](store common.Store, valueCodec codec.Codec[T]) (*ValueState[T], error) { if store == nil { return nil, api.NewError(api.ErrStoreInternal, "value state store must not be nil") } From 8556217427fa18b370a9c535a22887475f15d312 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 11 Mar 2026 17:09:09 +0800 Subject: [PATCH 7/9] update --- docs/Go-SDK/go-sdk-advanced-state-api.md | 314 ++++++++++++++++++ docs/Go-SDK/go-sdk-guide.md | 221 +----------- .../python-sdk-advanced-state-api-zh.md | 115 +++++++ .../python-sdk-advanced-state-api.md | 115 +++++++ docs/Python-SDK/python-sdk-guide-zh.md | 8 + docs/Python-SDK/python-sdk-guide.md | 8 + .../store/keyed/keyed_aggregating_state.py | 32 +- .../fs_api/store/keyed/keyed_list_state.py | 30 +- .../src/fs_api/store/keyed/keyed_map_state.py | 30 +- .../store/keyed/keyed_priority_queue_state.py | 30 +- .../store/keyed/keyed_reducing_state.py | 32 +- .../fs_api/store/keyed/keyed_value_state.py | 32 +- .../store/structures/aggregating_state.py | 26 +- .../src/fs_api/store/structures/list_state.py | 15 +- .../src/fs_api/store/structures/map_state.py | 24 ++ .../store/structures/priority_queue_state.py | 15 +- .../fs_api/store/structures/reducing_state.py | 26 +- .../fs_api/store/structures/value_state.py | 20 +- .../src/fs_runtime/store/fs_context.py | 81 ++--- 19 files changed, 891 insertions(+), 283 deletions(-) create mode 100644 docs/Go-SDK/go-sdk-advanced-state-api.md create mode 100644 docs/Python-SDK/python-sdk-advanced-state-api-zh.md create mode 100644 docs/Python-SDK/python-sdk-advanced-state-api.md diff --git a/docs/Go-SDK/go-sdk-advanced-state-api.md b/docs/Go-SDK/go-sdk-advanced-state-api.md new file mode 100644 index 00000000..315297a3 --- /dev/null +++ b/docs/Go-SDK/go-sdk-advanced-state-api.md @@ -0,0 +1,314 @@ + + +# Go SDK — Advanced State API + +This document describes the **typed, high-level state API** for the Function Stream Go SDK: state abstractions (ValueState, ListState, MapState, PriorityQueueState, AggregatingState, ReducingState, and Keyed\* factories) built on top of the low-level `Store`, with serialization via **codecs** and optional **keyed state** per primary key. Use it when you need structured state without manual byte encoding or key layout. + +**In this document:** + +- [When to use which API](#1-when-to-use-which-api) — choose the right state type for your use case. +- [Packages and imports](#2-packages-and-imports) — where to find types and codecs. +- [Codec contract](#3-codec-contract-and-default-codecs) — encoding, decoding, and ordering requirements. +- [Creating state](#4-creating-state-with-codec-vs-autocodec) — explicit codec vs AutoCodec constructors. +- [Non-keyed state reference](#5-non-keyed-state-structures) — methods and constructor summary. +- [AggregateFunc and ReduceFunc](#6-aggregatefunc-and-reducefunc) — interfaces for aggregation and reduction. +- [Keyed state](#7-keyed-state-factories-and-per-key-instances) — keyGroup, primaryKey, namespace, and factory methods. +- [Error handling and best practices](#8-error-handling-and-best-practices) — production guidance. +- [Examples](#9-examples) — ValueState, Keyed list, MapState, and AggregatingState. + +--- + +## 1. When to Use Which API + +The advanced state API offers typed views over a single logical store. Pick the abstraction that matches your access pattern: + +| Use case | Recommended API | Notes | +|----------|-----------------|--------| +| Single logical value (counter, config blob, last value) | **ValueState[T]** | One value per store; replace on update. | +| Append-only sequence (event log, history) | **ListState[T]** | Batch add, full read/replace; no key iteration. | +| Key-value map with range/iteration | **MapState[K,V]** | Key type **must** have an ordered codec (e.g. primitives). | +| Priority queue (min/max, top-K) | **PriorityQueueState[T]** | Element type **must** have an ordered codec. | +| Running aggregate (sum, count, custom accumulator) | **AggregatingState[T,ACC,R]** | Uses `AggregateFunc`; mergeable accumulators. | +| Running reduce (binary combine) | **ReducingState[V]** | Uses `ReduceFunc`; associative combine. | +| **Per-key** state (per user, per partition) | **Keyed\*** factories | For **keyed operators**; factory + primaryKey per record. | +| Custom key layout, bulk scan, non-typed storage | Low-level **Store** | `Put`/`Get`/`ScanComplex`/`ComplexKey`; full control. | + +**Keyed vs non-keyed** + +- **Keyed state** is for **keyed operators**: streams partitioned by a key (e.g. after keyBy). The runtime delivers records per key; each key should have isolated state. Obtain a **factory** once (from context, store name, and keyGroup), then create state **per primary key** (the stream key) via e.g. `factory.NewKeyedValue(primaryKey, stateName)`. +- **Non-keyed state** (ValueState, ListState, etc.) stores one logical entity per store. Use it when there is no key partitioning or you maintain a single global state. + +--- + +## 2. Packages and Imports + +| Package | Import path | Responsibility | +|---------|-------------|-----------------| +| **structures** | `github.com/functionstream/function-stream/go-sdk/state/structures` | Non-keyed state types: ValueState, ListState, MapState, PriorityQueueState, AggregatingState, ReducingState. | +| **keyed** | `github.com/functionstream/function-stream/go-sdk/state/keyed` | Keyed state **factories** and per-key state types (e.g. KeyedListStateFactory, KeyedListState). Use in keyed operators. | +| **codec** | `github.com/functionstream/function-stream/go-sdk/state/codec` | `Codec[T]` interface, `DefaultCodecFor[T]()`, and built-in codecs (primitives, JSONCodec). | + +All state constructors take `api.Context` (i.e. `fssdk.Context`) and a **store name**. The store is obtained internally via `ctx.GetOrCreateStore(storeName)`. The same store name always refers to the same backing store (RocksDB in the default implementation). + +--- + +## 3. Codec Contract and Default Codecs + +### 3.1 Codec interface + +`codec.Codec[T]`: + +| Method | Description | +|--------|-------------| +| `Encode(value T) ([]byte, error)` | Serialize a value to bytes. | +| `Decode(data []byte) (T, error)` | Deserialize from bytes. | +| `EncodedSize() int` | Fixed size if `> 0`; variable size if `<= 0` (used for list optimizations). | +| `IsOrderedKeyCodec() bool` | If `true`, the byte encoding is **totally ordered**: lexicographic order of bytes corresponds to a well-defined order of values. **Required** for MapState key and PriorityQueueState element. | + +### 3.2 DefaultCodecFor[T]() + +`codec.DefaultCodecFor[T]()` returns a default codec for type `T`: + +- **Primitives** (`int32`, `int64`, `uint32`, `uint64`, `float32`, `float64`, `string`, `bool`, `int`, `uint`, etc.): built-in codecs; **ordered** when used as map keys or PQ elements. +- **Struct, map, slice, array**: `JSONCodec[T]` — JSON encoding; **not ordered** (`IsOrderedKeyCodec() == false`). Do **not** use as MapState key or PriorityQueueState element type with AutoCodec (operations that depend on ordering may fail or panic). +- **Interface type without constraint**: returns error; the type parameter must be concrete. + +### 3.3 Ordering requirement + +For **MapState[K,V]** and **PriorityQueueState[T]**, the key (respectively element) type must use a codec with `IsOrderedKeyCodec() == true`. With **AutoCodec** constructors for Map or PQ, use primitive key/element types (e.g. `int64`, `string`) or provide an explicit ordered codec. + +--- + +## 4. Creating State: With Codec vs AutoCodec + +Two constructor families: + +1. **Explicit codec** — `NewXxxFromContext(ctx, storeName, codec, ...)` + You supply a `Codec[T]` (and for Map: key + value codecs; for Aggregating/Reducing: acc/value codec plus function). Full control over encoding and ordering. + +2. **AutoCodec** — `NewXxxFromContextAutoCodec(ctx, storeName)` or `(ctx, storeName, aggFunc/reduceFunc)` + The SDK uses `codec.DefaultCodecFor[T]()` for the value/accumulator type. For Map and PQ, the key/element type must have an **ordered** default (primitives); otherwise creation or operations may return `ErrStoreInternal`. + +State instances are **lightweight**. You can create them per call (e.g. inside `Process`) or cache in the Driver (e.g. in `Init`). The same store name always refers to the same underlying store; only the typed view differs. + +--- + +## 5. Non-Keyed State (structures) + +### 5.1 Semantics and methods + +| State | Semantics | Main methods | Ordered codec? | +|-------|-----------|---------------|----------------| +| **ValueState[T]** | Single replaceable value. | `Update(value T) error`; `Value() (T, bool, error)`; `Clear() error` | No | +| **ListState[T]** | Append-only list; batch add and full replace. | `Add(value T) error`; `AddAll(values []T) error`; `Get() ([]T, error)`; `Update(values []T) error`; `Clear() error` | No | +| **MapState[K,V]** | Key-value map; iteration via `All()`. | `Put(key K, value V) error`; `Get(key K) (V, bool, error)`; `Delete(key K) error`; `Clear() error`; `All() iter.Seq2[K,V]` | **Key K: yes** | +| **PriorityQueueState[T]** | Priority queue (min-first by encoded order). | `Add(value T) error`; `Peek() (T, bool, error)`; `Poll() (T, bool, error)`; `Clear() error`; `All() iter.Seq[T]` | **Item T: yes** | +| **AggregatingState[T,ACC,R]** | Running aggregation with mergeable accumulator. | `Add(value T) error`; `Get() (R, bool, error)`; `Clear() error` | No (ACC codec any) | +| **ReducingState[V]** | Running reduce with binary combine. | `Add(value V) error`; `Get() (V, bool, error)`; `Clear() error` | No | + +### 5.2 Constructor summary (non-keyed) + +| State | With codec | AutoCodec | +|-------|------------|-----------| +| ValueState[T] | `NewValueStateFromContext(ctx, storeName, valueCodec)` | `NewValueStateFromContextAutoCodec[T](ctx, storeName)` | +| ListState[T] | `NewListStateFromContext(ctx, storeName, itemCodec)` | `NewListStateFromContextAutoCodec[T](ctx, storeName)` | +| MapState[K,V] | `NewMapStateFromContext(ctx, storeName, keyCodec, valueCodec)` or `NewMapStateAutoKeyCodecFromContext(ctx, storeName, valueCodec)` | `NewMapStateFromContextAutoCodec[K,V](ctx, storeName)` | +| PriorityQueueState[T] | `NewPriorityQueueStateFromContext(ctx, storeName, itemCodec)` | `NewPriorityQueueStateFromContextAutoCodec[T](ctx, storeName)` | +| AggregatingState[T,ACC,R] | `NewAggregatingStateFromContext(ctx, storeName, accCodec, aggFunc)` | `NewAggregatingStateFromContextAutoCodec(ctx, storeName, aggFunc)` | +| ReducingState[V] | `NewReducingStateFromContext(ctx, storeName, valueCodec, reduceFunc)` | `NewReducingStateFromContextAutoCodec[V](ctx, storeName, reduceFunc)` | + +--- + +## 6. AggregateFunc and ReduceFunc + +### 6.1 AggregateFunc[T, ACC, R] + +**AggregatingState** requires an **AggregateFunc[T, ACC, R]** (in package `structures`): + +| Method | Description | +|--------|-------------| +| `CreateAccumulator() ACC` | Initial accumulator for empty state. | +| `Add(value T, accumulator ACC) ACC` | Fold one input value into the accumulator. | +| `GetResult(accumulator ACC) R` | Produce the final result from the accumulator. | +| `Merge(a, b ACC) ACC` | Combine two accumulators (e.g. for merge in distributed or checkpointed execution). | + +### 6.2 ReduceFunc[V] + +**ReducingState** requires a **ReduceFunc[V]** (function type): `func(value1, value2 V) (V, error)`. It must be **associative** (and ideally commutative) so that repeated application yields a well-defined reduced value. + +--- + +## 7. Keyed State — Factories and Per-Key Instances + +Keyed state is for **keyed operators**: when the stream is partitioned by a key (e.g. after keyBy), each key is processed with isolated state. You obtain a **factory** once (from context, store name, and **keyGroup**), then create state **per primary key** — the stream key for the current record (e.g. user ID, partition key). + +State is organized by **keyGroup** ([]byte) and **primary key** ([]byte). Create the factory from context, store name, and keyGroup; then call factory methods to get state for a given primary key. + +### 7.1 keyGroup, key (primaryKey), and namespace + +The Keyed API maps onto the store’s **ComplexKey** with three dimensions: + +| Term | Where it appears | Meaning | +|------|------------------|---------| +| **keyGroup** | Argument when creating the factory | The **keyed group**: identifies which keyed partition/group this state belongs to. Use one keyGroup per logical “keyed group” or state kind (e.g. `[]byte("counters")`, `[]byte("sessions")`). Same keyed group ⇒ same keyGroup bytes. | +| **key** | `primaryKey` in factory methods (e.g. `NewKeyedValue(primaryKey, ...)`, `NewKeyedList(primaryKey, namespace)`) | The **value of the stream key**: the key that partitioned the stream, serialized as bytes (e.g. user ID, partition key). Each distinct primaryKey gets isolated state. | +| **namespace** | `namespace` ([]byte) in factory methods that take it | **With window functions**: use the **window identifier as bytes** (e.g. serialized window bounds or window ID) so state is scoped per key *and* per window. **Without windows**: pass **empty bytes** (`nil` or `[]byte{}`). | + +**Summary:** **keyGroup** = keyed group identifier; **key** (primaryKey) = stream key value; **namespace** = window bytes when using windows, otherwise empty. + +### 7.2 Factory constructor summary (keyed) + +| Factory | With codec | AutoCodec | +|---------|------------|-----------| +| KeyedValueStateFactory[V] | `NewKeyedValueStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec)` | `NewKeyedValueStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup)` | +| KeyedListStateFactory[V] | `NewKeyedListStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec)` | `NewKeyedListStateFactoryAutoCodecFromContext[V](ctx, storeName, keyGroup)` | +| KeyedMapStateFactory[MK,MV] | `NewKeyedMapStateFactoryFromContext(ctx, storeName, keyGroup, keyCodec, valueCodec)` | `NewKeyedMapStateFactoryFromContextAutoCodec[MK,MV](ctx, storeName, keyGroup)` | +| KeyedPriorityQueueStateFactory[V] | `NewKeyedPriorityQueueStateFactoryFromContext(ctx, storeName, keyGroup, itemCodec)` | `NewKeyedPriorityQueueStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup)` | +| KeyedAggregatingStateFactory[T,ACC,R] | `NewKeyedAggregatingStateFactoryFromContext(ctx, storeName, keyGroup, accCodec, aggFunc)` | `NewKeyedAggregatingStateFactoryFromContextAutoCodec(ctx, storeName, keyGroup, aggFunc)` | +| KeyedReducingStateFactory[V] | `NewKeyedReducingStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec, reduceFunc)` | `NewKeyedReducingStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup, reduceFunc)` | + +### 7.3 Obtaining per-key state from a factory + +| Factory | Method | Returns | +|---------|--------|---------| +| KeyedValueStateFactory[V] | `NewKeyedValue(primaryKey []byte, stateName string) (*KeyedValueState[V], error)` | One value state per (primaryKey, stateName). | +| KeyedListStateFactory[V] | `NewKeyedList(primaryKey []byte, namespace []byte) (*KeyedListState[V], error)` | List state per (primaryKey, namespace). | +| KeyedMapStateFactory[MK,MV] | `NewKeyedMap(primaryKey []byte, mapName string) (*KeyedMapState[MK,MV], error)` | Map state per (primaryKey, mapName). | +| KeyedPriorityQueueStateFactory[V] | `NewKeyedPriorityQueue(primaryKey []byte, namespace []byte) (*KeyedPriorityQueueState[V], error)` | PQ state per (primaryKey, namespace). | +| KeyedAggregatingStateFactory | `NewAggregatingState(primaryKey []byte, stateName string) (*KeyedAggregatingState[T,ACC,R], error)` | Aggregating state per (primaryKey, stateName). | +| KeyedReducingStateFactory[V] | `NewReducingState(primaryKey []byte, namespace []byte) (*KeyedReducingState[V], error)` | Reducing state per (primaryKey, namespace). | + +Here **primaryKey** is the stream key value; **namespace** is the window bytes when using window functions, or empty when not. + +**Design tip:** Use a stable keyGroup per logical state (e.g. `[]byte("orders")`). In factory methods, pass the **stream key** your keyed operator received (e.g. from key extractor or message metadata) as primaryKey. + +--- + +## 8. Error Handling and Best Practices + +- **State API errors**: Creation and methods return errors compatible with `fssdk.SDKError` (e.g. `ErrStoreInternal`, `ErrStoreIO`). Codec encode/decode failures are wrapped (e.g. `"encode value state failed"`). Always check and handle errors in production. +- **Store naming**: Use stable, unique store names per logical state (e.g. `"counters"`, `"user-sessions"`). The same name in the same runtime refers to the same store. +- **Caching state**: You can create a state instance once in `Init` and reuse it in `Process`, or create it per message. Per-message creation is safe and keeps code simple when you do not need to amortize creation cost. +- **KeyGroup design**: For keyed state, use a consistent keyGroup per “logical table”. primaryKey is the **stream key** in keyed operators — use the key that identifies the current record. With **window functions**, pass the window identifier as **namespace** so state is per key and per window. +- **Ordered codec**: For MapState and PriorityQueueState with AutoCodec, use primitive key/element types. For custom struct keys, implement a `Codec[K]` with `IsOrderedKeyCodec() == true` and use the “with codec” constructor. + +--- + +## 9. Examples + +### 9.1 ValueState with AutoCodec (counter) + +```go +import ( + fssdk "github.com/functionstream/function-stream/go-sdk" + "github.com/functionstream/function-stream/go-sdk/state/structures" +) + +func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + valState, err := structures.NewValueStateFromContextAutoCodec[int64](ctx, "my-store") + if err != nil { + return err + } + cur, _, _ := valState.Value() + if err := valState.Update(cur + 1); err != nil { + return err + } + // emit or continue... + return nil +} +``` + +### 9.2 Keyed list factory (keyed operator) + +When the operator runs on a **keyed stream**, use a Keyed list factory and pass the **stream key** as primaryKey for each message: + +```go +import ( + fssdk "github.com/functionstream/function-stream/go-sdk" + "github.com/functionstream/function-stream/go-sdk/state/keyed" +) + +type Order struct { Id string; Amount int64 } + +func (p *MyProcessor) Init(ctx fssdk.Context, config map[string]string) error { + keyGroup := []byte("orders") + factory, err := keyed.NewKeyedListStateFactoryAutoCodecFromContext[Order](ctx, "app-store", keyGroup) + if err != nil { + return err + } + p.listFactory = factory + return nil +} + +func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { + userID := parseUserID(data) // []byte — stream key for this record + list, err := p.listFactory.NewKeyedList(userID, []byte{}) // empty namespace when no windows + if err != nil { + return err + } + if err := list.Add(Order{Id: "1", Amount: 100}); err != nil { + return err + } + items, err := list.Get() + if err != nil { + return err + } + // use items... + return nil +} +``` + +### 9.3 MapState and AggregatingState (sum) + +```go +import "github.com/functionstream/function-stream/go-sdk/state/structures" + +// MapState: string -> int64 (both have ordered default codecs) +m, err := structures.NewMapStateFromContextAutoCodec[string, int64](ctx, "counts") +if err != nil { + return err +} +_ = m.Put("a", 1) +v, ok, _ := m.Get("a") + +// AggregatingState: sum of int64 (ACC = int64, R = int64) +type sumAgg struct{} +func (sumAgg) CreateAccumulator() int64 { return 0 } +func (sumAgg) Add(v int64, acc int64) int64 { return acc + v } +func (sumAgg) GetResult(acc int64) int64 { return acc } +func (sumAgg) Merge(a, b int64) int64 { return a + b } + +agg, err := structures.NewAggregatingStateFromContextAutoCodec[int64, int64, int64](ctx, "sum-store", sumAgg{}) +if err != nil { + return err +} +_ = agg.Add(10) +total, _, _ := agg.Get() +``` + +--- + +## 10. See Also + +- [Go SDK Guide](go-sdk-guide.md) — main guide: Driver, Context, Store, build, and deployment. +- [Python SDK — Advanced State API](../Python-SDK/python-sdk-advanced-state-api.md) — equivalent typed state API for the Python SDK. +- [examples/go-processor/README.md](../../examples/go-processor/README.md) — example operator and build instructions. diff --git a/docs/Go-SDK/go-sdk-guide.md b/docs/Go-SDK/go-sdk-guide.md index 49592ef6..a65b30ed 100644 --- a/docs/Go-SDK/go-sdk-guide.md +++ b/docs/Go-SDK/go-sdk-guide.md @@ -303,225 +303,16 @@ if err != nil { ## 7. Advanced State API -This section describes the **high-level state API**: typed state abstractions built on top of the low-level `Store`, with serialization via **codecs** and optional **keyed state** per primary key. Use it when you want structured state (single value, list, map, priority queue, aggregation, reduction) without manual byte encoding or key layout. +The Go SDK provides a **typed, high-level state API** on top of the low-level `Store`: ValueState, ListState, MapState, PriorityQueueState, AggregatingState, ReducingState, and Keyed\* factories, with serialization via **codecs**. Use it when you need structured state (single value, list, map, priority queue, aggregation, reduction) without manual byte encoding or key layout. -### 7.1 When to Use Which API +Full reference, codec contract, keyGroup/primaryKey/namespace semantics, constructor tables, and examples are in a dedicated document: -| Use case | Recommended API | -|----------|------------------| -| Single logical value (e.g. counter, config blob) | **ValueState[T]** or raw `Store` with one key. | -| Append-only sequence (e.g. event log) | **ListState[T]** or `Store.Merge` with custom encoding. | -| Key-value map with range/iteration | **MapState[K,V]** (key type must have ordered codec). | -| Priority queue (min/max, top-K) | **PriorityQueueState[T]** (item codec must be ordered). | -| Running aggregate (sum, count, custom accumulator) | **AggregatingState[T,ACC,R]** with `AggregateFunc`. | -| Running reduce (binary combine) | **ReducingState[V]** with `ReduceFunc`. | -| **Per-key** state (e.g. per user, per partition key) | **Keyed*** factories: get a factory once, then `NewKeyedValue(primaryKey, ...)` etc. per key. **Use in keyed operators** — when the stream is partitioned by a key (e.g. after keyBy), each key gets isolated state. | -| Custom key layout, bulk scan, or non-typed storage | Low-level **Store** (`Put`/`Get`/`ScanComplex`/`ComplexKey`). | +- **[Go SDK — Advanced State API](go-sdk-advanced-state-api.md)** -**Keyed vs non-keyed:** **Keyed state is intended for keyed operators**: streams that are partitioned by a key (e.g. after a keyBy in the pipeline). In that case, the runtime delivers records per key, and each key should have its own independent state — use a **factory** and **keyGroup**, then create state per **primary key** (the stream key) via `factory.NewKeyedValue(primaryKey, stateName)` etc. Non-keyed state (e.g. `ValueState`, `ListState`) stores one logical entity per store and is used when there is no key partitioning or you maintain a single global state. +Quick reference: -### 7.2 Packages and Module Paths - -| Package | Import path | Responsibility | -|---------|-------------|----------------| -| **structures** | `github.com/functionstream/function-stream/go-sdk/state/structures` | Non-keyed state types: `ValueState`, `ListState`, `MapState`, `PriorityQueueState`, `AggregatingState`, `ReducingState`. | -| **keyed** | `github.com/functionstream/function-stream/go-sdk/state/keyed` | Keyed state **factories** and their per-key state types (e.g. `KeyedListStateFactory`, `KeyedListState`). **Use in keyed operators** — when the stream is partitioned by a key, create state per stream key. | -| **codec** | `github.com/functionstream/function-stream/go-sdk/state/codec` | `Codec[T]` interface, `DefaultCodecFor[T]()`, and built-in codecs (primitives, `JSONCodec`). | - -All state constructors take `api.Context` (i.e. `fssdk.Context`) and a **store name**; the store is obtained via `ctx.GetOrCreateStore(storeName)` internally. The same store name always refers to the same backing store (RocksDB-backed in the default implementation). - -### 7.3 Codec Contract and Default Codecs - -**Codec interface** (`codec.Codec[T]`): - -- `Encode(value T) ([]byte, error)` — serialize to bytes. -- `Decode(data []byte) (T, error)` — deserialize from bytes. -- `EncodedSize() int` — fixed size if `> 0`, variable size if `<= 0` (used for list optimization). -- `IsOrderedKeyCodec() bool` — if `true`, the byte encoding is **totally ordered** (lexicographic order of bytes corresponds to a well-defined order of values). Required for **MapState** key and **PriorityQueueState** item. - -**DefaultCodecFor[T]()** returns a default codec for type `T`: - -- **Primitives** (`int32`, `int64`, `uint32`, `uint64`, `float32`, `float64`, `string`, `bool`, `int`, `uint`, etc.): built-in fixed- or variable-size codecs; **ordered** for types used as map keys or PQ elements. -- **Struct, map, slice, array**: `JSONCodec[T]` — JSON encoding; **not ordered** (`IsOrderedKeyCodec() == false`). Do **not** use as MapState key or PriorityQueueState element type with AutoCodec (creation will succeed but operations that rely on ordering may fail or panic). -- **Interface type without constraint**: returns error (type parameter must be concrete). - -**Ordering requirement:** For `MapState[K,V]` and `PriorityQueueState[T]`, the **key** (respectively **element**) type must use a codec with `IsOrderedKeyCodec() == true`. When using **AutoCodec** constructors for Map or PQ, choose primitive key/element types (e.g. `int64`, `string`) or provide an explicit ordered codec. - -### 7.4 Creating State: With Codec vs AutoCodec - -Two constructor families: - -1. **Explicit codec** — `NewXxxFromContext(ctx, storeName, codec, ...)` - You pass a `Codec[T]` (and for Map: key + value codecs; for Aggregating/Reducing: acc/value codec + function). Full control over encoding and ordering. - -2. **AutoCodec** — `NewXxxFromContextAutoCodec(ctx, storeName)` or `(ctx, storeName, aggFunc/reduceFunc)` - The SDK calls `codec.DefaultCodecFor[T]()` for the value/accumulator type. For Map and PQ, key/element type `T` must have an **ordered** default (primitives); otherwise runtime checks may return `ErrStoreInternal`. - -State instances are **lightweight**; you may create them per invocation (e.g. `NewValueStateFromContextAutoCodec[int64](ctx, "store")` inside `Process`) or cache in the Driver (e.g. in `Init`). Same store name yields the same underlying store; only the typed view differs. - -### 7.5 Non-Keyed State (structures) — Reference - -| State | Semantics | Methods (signature summary) | Ordered codec? | -|-------|------------|-----------------------------|----------------| -| **ValueState[T]** | Single replaceable value. | `Update(value T) error`; `Value() (T, bool, error)`; `Clear() error` | No | -| **ListState[T]** | Append-only list; supports batch add and full replace. | `Add(value T) error`; `AddAll(values []T) error`; `Get() ([]T, error)`; `Update(values []T) error`; `Clear() error` | No | -| **MapState[K,V]** | Key-value map; key range iteration via `All()`. | `Put(key K, value V) error`; `Get(key K) (V, bool, error)`; `Delete(key K) error`; `Clear() error`; `All() iter.Seq2[K,V]` | **Key K: yes** | -| **PriorityQueueState[T]** | Priority queue (min-first by encoded key order). | `Add(value T) error`; `Peek() (T, bool, error)`; `Poll() (T, bool, error)`; `Clear() error`; `All() iter.Seq[T]` | **Item T: yes** | -| **AggregatingState[T,ACC,R]** | Running aggregation with mergeable accumulator. | `Add(value T) error`; `Get() (R, bool, error)`; `Clear() error` | No (ACC codec any) | -| **ReducingState[V]** | Running reduce with binary combine. | `Add(value V) error`; `Get() (V, bool, error)`; `Clear() error` | No | - -**Constructor summary (non-keyed):** - -| State | With codec | AutoCodec | -|-------|------------|-----------| -| ValueState[T] | `NewValueStateFromContext(ctx, storeName, valueCodec)` | `NewValueStateFromContextAutoCodec[T](ctx, storeName)` | -| ListState[T] | `NewListStateFromContext(ctx, storeName, itemCodec)` | `NewListStateFromContextAutoCodec[T](ctx, storeName)` | -| MapState[K,V] | `NewMapStateFromContext(ctx, storeName, keyCodec, valueCodec)` or `NewMapStateAutoKeyCodecFromContext(ctx, storeName, valueCodec)` | `NewMapStateFromContextAutoCodec[K,V](ctx, storeName)` | -| PriorityQueueState[T] | `NewPriorityQueueStateFromContext(ctx, storeName, itemCodec)` | `NewPriorityQueueStateFromContextAutoCodec[T](ctx, storeName)` | -| AggregatingState[T,ACC,R] | `NewAggregatingStateFromContext(ctx, storeName, accCodec, aggFunc)` | `NewAggregatingStateFromContextAutoCodec(ctx, storeName, aggFunc)` | -| ReducingState[V] | `NewReducingStateFromContext(ctx, storeName, valueCodec, reduceFunc)` | `NewReducingStateFromContextAutoCodec[V](ctx, storeName, reduceFunc)` | - -### 7.6 AggregateFunc and ReduceFunc - -**AggregatingState** requires an **AggregateFunc[T, ACC, R]** (defined in `structures`): - -- `CreateAccumulator() ACC` — initial accumulator for empty state. -- `Add(value T, accumulator ACC) ACC` — fold one input into the accumulator. -- `GetResult(accumulator ACC) R` — produce the final result from the accumulator. -- `Merge(a, b ACC) ACC` — combine two accumulators (e.g. for merge in distributed/checkpointed execution). - -**ReducingState** requires a **ReduceFunc[V]** (function type): `func(value1, value2 V) (V, error)`. It must be associative (and ideally commutative) so that repeated application yields a well-defined reduced value. - -### 7.7 Keyed State — Factories and Per-Key Instances - -**Keyed state is for keyed operators.** When the stream is partitioned by a key (e.g. after keyBy in the pipeline), each key is processed with isolated state. The Keyed state API lets you obtain a **factory** once (from context, store name, and **keyGroup**), then create state **per primary key** — the primary key is the stream key (e.g. user ID, partition key) for the current record. Use Keyed* factories when your operator runs on a keyed stream; use non-keyed state when the stream is not keyed or you need a single global state. - -Keyed state is organized by **keyGroup** ([]byte) and **primary key** ([]byte). You first create a **factory** from context, store name, and keyGroup; then call factory methods to obtain state for a given primary key. - -#### 7.7.1 keyGroup, key (primaryKey), and namespace - -The Keyed state API maps onto the store’s **ComplexKey** with three dimensions. Use them as follows: - -| Term | API parameter | Meaning | -|------|----------------|---------| -| **keyGroup** | `keyGroup` when creating the factory | The **keyed group**: identifies which keyed partition/group this state belongs to. One keyGroup per logical “keyed group” or state kind (e.g. one group for “counters”, another for “sessions”). Same keyed group ⇒ same keyGroup bytes; different groups ⇒ different keyGroups. | -| **key** | `primaryKey` in factory methods (`NewKeyedValue(primaryKey, ...)`, `NewKeyedList(primaryKey, namespace)`, etc.) | The **value of the key**: the stream key for the current record, serialized as bytes. This is the key that partitioned the stream (e.g. user ID, partition key). Each distinct key value gets isolated state. | -| **namespace** | `namespace` ([]byte) in factory methods | **If a window function is present**, use the **window identifier as bytes** (e.g. serialized window bounds or window ID) so that state is scoped per key *and* per window. **Without windows**, pass **empty bytes** (e.g. `nil` or `[]byte{}`). | - -**Summary:** **keyGroup** = keyed group; **key** (primaryKey) = the key’s value (stream key); **namespace** = window bytes when using window functions, otherwise **empty bytes**. - -**Factory constructor summary (keyed):** - -| Factory | With codec | AutoCodec | -|---------|------------|-----------| -| KeyedValueStateFactory[V] | `NewKeyedValueStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec)` | `NewKeyedValueStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup)` | -| KeyedListStateFactory[V] | `NewKeyedListStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec)` | `NewKeyedListStateFactoryAutoCodecFromContext[V](ctx, storeName, keyGroup)` | -| KeyedMapStateFactory[MK,MV] | `NewKeyedMapStateFactoryFromContext(ctx, storeName, keyGroup, keyCodec, valueCodec)` | `NewKeyedMapStateFactoryFromContextAutoCodec[MK,MV](ctx, storeName, keyGroup)` | -| KeyedPriorityQueueStateFactory[V] | `NewKeyedPriorityQueueStateFactoryFromContext(ctx, storeName, keyGroup, itemCodec)` | `NewKeyedPriorityQueueStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup)` | -| KeyedAggregatingStateFactory[T,ACC,R] | `NewKeyedAggregatingStateFactoryFromContext(ctx, storeName, keyGroup, accCodec, aggFunc)` | `NewKeyedAggregatingStateFactoryFromContextAutoCodec(ctx, storeName, keyGroup, aggFunc)` | -| KeyedReducingStateFactory[V] | `NewKeyedReducingStateFactoryFromContext(ctx, storeName, keyGroup, valueCodec, reduceFunc)` | `NewKeyedReducingStateFactoryFromContextAutoCodec[V](ctx, storeName, keyGroup, reduceFunc)` | - -**Obtaining per-key state from a factory:** - -- **KeyedValueStateFactory[V]**: `NewKeyedValue(primaryKey []byte, stateName string) (*KeyedValueState[V], error)` — one value state per (primaryKey, stateName). -- **KeyedListStateFactory[V]**: `NewKeyedList(primaryKey []byte, namespace []byte) (*KeyedListState[V], error)`. -- **KeyedMapStateFactory[MK,MV]**: `NewKeyedMap(primaryKey []byte, mapName string) (*KeyedMapState[MK,MV], error)`. -- **KeyedPriorityQueueStateFactory[V]**: `NewKeyedPriorityQueue(primaryKey []byte, namespace []byte) (*KeyedPriorityQueueState[V], error)`. -- **KeyedAggregatingStateFactory**: `NewAggregatingState(primaryKey []byte, stateName string) (*KeyedAggregatingState[T,ACC,R], error)`. -- **KeyedReducingStateFactory[V]**: `NewReducingState(primaryKey []byte, namespace []byte) (*KeyedReducingState[V], error)`. - -Here **primaryKey** is the key (stream key value); **namespace** is the window bytes when using window functions, or **empty bytes** when not using windows. - -**keyGroup** partitions the key space (e.g. one keyGroup per logical state name or feature). Keep it stable for a given logical state; use different keyGroups for different state entities that share the same store. **primaryKey** in factory methods is the key of the current record in a keyed stream — pass the key your keyed operator received (e.g. from the key extractor or message metadata). - -### 7.8 Error Handling and Best Practices - -- **Errors from state API**: Creation and methods return errors compatible with `fssdk.SDKError` (e.g. `ErrStoreInternal`, `ErrStoreIO`). Codec encode/decode errors are wrapped (e.g. `"encode value state failed"`). Always check and handle errors in production. -- **Store naming**: Use stable, unique store names per logical state (e.g. `"counters"`, `"user-sessions"`). Same name in the same runtime refers to the same store. -- **Caching state**: You may create a state instance once in `Init` and reuse it in `Process`, or create it per message. Creating per message is safe and keeps code simple when you do not need to amortize creation cost. -- **KeyGroup design**: For keyed state, use a consistent keyGroup per “logical table” (e.g. `[]byte("orders")`). primaryKey is the **stream key** in keyed operators — use the key that identifies the current record (e.g. from key extractor or message key). When using **window functions**, pass the window identifier as **namespace** (e.g. serialized window bounds) so state is per key and per window. -- **Ordered codec**: For MapState and PriorityQueueState with AutoCodec, use primitive key/element types. For custom struct keys, implement a `Codec[K]` with `IsOrderedKeyCodec() == true` and use the “with codec” constructor. - -### 7.9 Example: ValueState with AutoCodec (counter) - -```go -import ( - fssdk "github.com/functionstream/function-stream/go-sdk" - "github.com/functionstream/function-stream/go-sdk/state/structures" -) - -func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { - valState, err := structures.NewValueStateFromContextAutoCodec[int64](ctx, "my-store") - if err != nil { - return err - } - cur, _, _ := valState.Value() - if err := valState.Update(cur + 1); err != nil { - return err - } - // ... -} -``` - -### 7.10 Example: Keyed list factory (keyed operator) - -When your operator runs on a **keyed stream** (partitioned by key), use a Keyed list factory and pass the **stream key** as primaryKey for each message: - -```go -import ( - fssdk "github.com/functionstream/function-stream/go-sdk" - "github.com/functionstream/function-stream/go-sdk/state/keyed" -) - -type Order struct { Id string; Amount int64 } - -func (p *MyProcessor) Init(ctx fssdk.Context, config map[string]string) error { - keyGroup := []byte("orders") - factory, err := keyed.NewKeyedListStateFactoryAutoCodecFromContext[Order](ctx, "app-store", keyGroup) - if err != nil { - return err - } - p.listFactory = factory - return nil -} - -func (p *MyProcessor) Process(ctx fssdk.Context, sourceID uint32, data []byte) error { - userID := parseUserID(data) // []byte - list, err := p.listFactory.NewKeyedList(userID, "items") - if err != nil { - return err - } - if err := list.Add(Order{Id: "1", Amount: 100}); err != nil { - return err - } - items, err := list.Get() - if err != nil { - return err - } - // use items... -} -``` - -### 7.11 Example: MapState and AggregatingState (sum) - -```go -// MapState: string -> int64 count (both have ordered default codecs) -m, err := structures.NewMapStateFromContextAutoCodec[string, int64](ctx, "counts") -if err != nil { return err } -_ = m.Put("a", 1) -v, ok, _ := m.Get("a") - -// AggregatingState: sum of int64 with AutoCodec (ACC = int64, R = int64) -type sumAgg struct{} -func (sumAgg) CreateAccumulator() int64 { return 0 } -func (sumAgg) Add(v int64, acc int64) int64 { return acc + v } -func (sumAgg) GetResult(acc int64) int64 { return acc } -func (sumAgg) Merge(a, b int64) int64 { return a + b } -agg, err := structures.NewAggregatingStateFromContextAutoCodec[int64, int64, int64](ctx, "sum-store", sumAgg{}) -if err != nil { return err } -_ = agg.Add(10) -total, _, _ := agg.Get() -``` +- **Non-keyed:** `structures.NewValueStateFromContext(ctx, storeName, codec)` or `structures.NewValueStateFromContextAutoCodec[T](ctx, storeName)`; same pattern for ListState, MapState, PriorityQueueState, AggregatingState, ReducingState. +- **Keyed:** Create a factory with `keyed.NewKeyedXxxFromContext(ctx, storeName, keyGroup, ...)`, then obtain per-key state with `factory.NewKeyedValue(primaryKey, stateName)` etc. Use **keyGroup** for the keyed group, **primaryKey** for the stream key, **namespace** for window bytes (or empty). --- diff --git a/docs/Python-SDK/python-sdk-advanced-state-api-zh.md b/docs/Python-SDK/python-sdk-advanced-state-api-zh.md new file mode 100644 index 00000000..c2298b66 --- /dev/null +++ b/docs/Python-SDK/python-sdk-advanced-state-api-zh.md @@ -0,0 +1,115 @@ + + +# Python SDK — 高级状态 API + +本文档介绍 Python SDK 的**高级状态 API**：基于底层 KvStore 的带类型状态抽象（ValueState、ListState、MapState 等），通过 **codec** 序列化，并支持按主键的 **keyed state**。设计与 [Go SDK 高级状态 API](../Go-SDK/go-sdk-guide.md#7-advanced-state-api) 对齐。 + +--- + +## 1. 概述 + +当需要结构化状态（单值、列表、Map、优先队列、聚合、归约）而不想手写字节编码或 key 布局时，可使用高级状态 API。创建方式有两种：通过 **Context**（如 `ctx.getOrCreateValueState(...)`）或通过状态类型上的**类型级构造方法**（推荐，便于复用，与 Go SDK 用法一致）。 + +--- + +## 2. 创建状态的两种方式 + +### 2.1 通过 Context（getOrCreate\*） + +`Context` 提供 `getOrCreateValueState(store_name, codec)`、`getOrCreateValueStateAutoCodec(store_name)` 等方法，以及 ListState、MapState、PriorityQueueState、AggregatingState、ReducingState 与所有 Keyed\* 工厂的对应方法。运行时实现会委托给下面所述的类型级 `from_context` / `from_context_auto_codec`。 + +### 2.2 通过状态类型（推荐，与 Go SDK 一致） + +每种状态类型和 keyed 工厂提供： + +- **带 codec：** `XxxState.from_context(ctx, store_name, codec, ...)` +- **AutoCodec：** `XxxState.from_context_auto_codec(ctx, store_name)` 或带可选类型参数，由 SDK 使用默认 codec（如 PickleCodec，或 Map key / PQ 元素所需的有序 codec）。 + +状态实例是轻量的；可在每次 `process` 中创建，或在 driver 中（如 `init`）缓存。同一 store 名称对应同一底层 store。 + +--- + +## 3. 非 Keyed 状态 — 构造方法一览 + +| 状态类型 | 带 codec | AutoCodec | +|----------|----------|-----------| +| ValueState | `ValueState.from_context(ctx, store_name, codec)` | `ValueState.from_context_auto_codec(ctx, store_name)` | +| ListState | `ListState.from_context(ctx, store_name, codec)` | `ListState.from_context_auto_codec(ctx, store_name)` | +| MapState | `MapState.from_context(ctx, store_name, key_codec, value_codec)` 或 `MapState.from_context_auto_key_codec(ctx, store_name, value_codec)` | — | +| PriorityQueueState | `PriorityQueueState.from_context(ctx, store_name, codec)` | `PriorityQueueState.from_context_auto_codec(ctx, store_name)` | +| AggregatingState | `AggregatingState.from_context(ctx, store_name, acc_codec, agg_func)` | `AggregatingState.from_context_auto_codec(ctx, store_name, agg_func)` | +| ReducingState | `ReducingState.from_context(ctx, store_name, value_codec, reduce_func)` | `ReducingState.from_context_auto_codec(ctx, store_name, reduce_func)` | + +以上均可通过 Context 的 `ctx.getOrCreate*` 方法获得（如 `ctx.getOrCreateValueState(store_name, codec)`），其内部会委托给上述构造方法。 + +--- + +## 4. Keyed 状态 — 工厂与 key_group / key / namespace + +**Keyed 状态面向 keyed 算子。** 流按 key 分区（如 keyBy）时，每个 key 拥有独立状态。可先获取一次**工厂**（通过 context、store 名称、**namespace** 和 **key_group**），再按**主键**（当前记录的流 key）创建状态。 + +### 4.1 key_group、key（主键）与 namespace + +| 概念 | API 参数 | 含义 | +|------|----------|------| +| **key_group** | 创建工厂时的 `key_group` | **keyed 组**：标识该状态所属分区/组（如一组 “counters”，另一组 “sessions”）。 | +| **key** | 工厂方法参数（如 `new_keyed_value(primary_key)`） | 当前记录的**流 key 的值**（如用户 ID、分区 key）。不同 key 对应不同状态。 | +| **namespace** | 创建工厂时的 `namespace`（bytes） | **有窗口时**为**窗口标识的 bytes**；**无窗口时**传**空 bytes**（如 `b""`）。 | + +### 4.2 Keyed 工厂构造方法一览 + +| 工厂 | 带 codec | AutoCodec | +|------|----------|-----------| +| KeyedValueStateFactory | `KeyedValueStateFactory.from_context(ctx, store_name, namespace, key_group, value_codec)` | `KeyedValueStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, value_type=None)` | +| KeyedListStateFactory | `KeyedListStateFactory.from_context(ctx, store_name, namespace, key_group, value_codec)` | `KeyedListStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, value_type=None)` | +| KeyedMapStateFactory | `KeyedMapStateFactory.from_context(ctx, store_name, namespace, key_group, key_codec, value_codec)` | `KeyedMapStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, value_codec)` | +| KeyedPriorityQueueStateFactory | `KeyedPriorityQueueStateFactory.from_context(ctx, store_name, namespace, key_group, item_codec)` | `KeyedPriorityQueueStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, item_type=None)` | +| KeyedAggregatingStateFactory | `KeyedAggregatingStateFactory.from_context(ctx, store_name, namespace, key_group, acc_codec, agg_func)` | `KeyedAggregatingStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, agg_func, acc_type=None)` | +| KeyedReducingStateFactory | `KeyedReducingStateFactory.from_context(ctx, store_name, namespace, key_group, value_codec, reduce_func)` | `KeyedReducingStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, reduce_func, value_type=None)` | + +也可使用 Context 的 `ctx.getOrCreateKeyed*Factory(...)` 方法，其内部会委托给上述构造方法。 + +--- + +## 5. 示例：使用 from_context_auto_codec 的 ValueState + +```python +from fs_api import FSProcessorDriver, Context +from fs_api.store import ValueState + +class CounterProcessor(FSProcessorDriver): + def process(self, ctx: Context, source_id: int, data: bytes): + # 每条消息创建一次状态（或在 init 中缓存） + state = ValueState.from_context_auto_codec(ctx, "my-store") + cur, _ = state.value() or (0, False) + state.update(cur + 1) + ctx.emit(str(cur + 1).encode(), 0) +``` + +其他状态类型用法相同：按上表使用 `XxxState.from_context(ctx, store_name, ...)` 或 `XxxState.from_context_auto_codec(ctx, store_name)`。 + +--- + +## 6. 参见 + +- [Python SDK 指南](python-sdk-guide-zh.md) — fs_api、fs_client 及 Context/KvStore 基础用法。 +- [Go SDK 指南 — 高级状态 API](../Go-SDK/go-sdk-guide.md#7-advanced-state-api) — Go SDK 中的等价 API。 diff --git a/docs/Python-SDK/python-sdk-advanced-state-api.md b/docs/Python-SDK/python-sdk-advanced-state-api.md new file mode 100644 index 00000000..92712dfb --- /dev/null +++ b/docs/Python-SDK/python-sdk-advanced-state-api.md @@ -0,0 +1,115 @@ + + +# Python SDK — Advanced State API + +This document describes the **high-level state API** for the Python SDK: typed state abstractions (ValueState, ListState, MapState, etc.) built on top of the low-level KvStore, with serialization via **codecs** and optional **keyed state** per primary key. The design aligns with the [Go SDK Advanced State API](../Go-SDK/go-sdk-guide.md#7-advanced-state-api). + +--- + +## 1. Overview + +Use the advanced state API when you need structured state (single value, list, map, priority queue, aggregation, reduction) without manual byte encoding or key layout. You can create state either from **Context** (e.g. `ctx.getOrCreateValueState(...)`) or via **type-level constructors** on the state class (recommended for clarity and reuse, same pattern as the Go SDK). + +--- + +## 2. Creating State: Two Ways + +### 2.1 From Context (getOrCreate\*) + +`Context` defines methods such as `getOrCreateValueState(store_name, codec)`, `getOrCreateValueStateAutoCodec(store_name)`, and the same pattern for ListState, MapState, PriorityQueueState, AggregatingState, ReducingState, and all Keyed\* factories. The runtime implementation delegates to the type-level `from_context` / `from_context_auto_codec` methods below. + +### 2.2 From the state type (recommended, same as Go SDK) + +Each state type and keyed factory provides: + +- **With codec:** `XxxState.from_context(ctx, store_name, codec, ...)` — you pass the codec(s). +- **AutoCodec:** `XxxState.from_context_auto_codec(ctx, store_name)` or with optional type hint — the SDK uses a default codec (e.g. `PickleCodec`, or ordered codecs for map key / PQ element where required). + +State instances are lightweight; you may create them per message in `process` or cache in the driver (e.g. in `init`). Same store name yields the same underlying store. + +--- + +## 3. Non-Keyed State — Constructor Summary + +| State | With codec | AutoCodec | +|-------|------------|-----------| +| ValueState | `ValueState.from_context(ctx, store_name, codec)` | `ValueState.from_context_auto_codec(ctx, store_name)` | +| ListState | `ListState.from_context(ctx, store_name, codec)` | `ListState.from_context_auto_codec(ctx, store_name)` | +| MapState | `MapState.from_context(ctx, store_name, key_codec, value_codec)` or `MapState.from_context_auto_key_codec(ctx, store_name, value_codec)` | — | +| PriorityQueueState | `PriorityQueueState.from_context(ctx, store_name, codec)` | `PriorityQueueState.from_context_auto_codec(ctx, store_name)` | +| AggregatingState | `AggregatingState.from_context(ctx, store_name, acc_codec, agg_func)` | `AggregatingState.from_context_auto_codec(ctx, store_name, agg_func)` | +| ReducingState | `ReducingState.from_context(ctx, store_name, value_codec, reduce_func)` | `ReducingState.from_context_auto_codec(ctx, store_name, reduce_func)` | + +All of the above can also be obtained via the corresponding `ctx.getOrCreate*` methods (e.g. `ctx.getOrCreateValueState(store_name, codec)`), which delegate to these constructors. + +--- + +## 4. Keyed State — Factories and keyGroup / key / namespace + +**Keyed state is for keyed operators.** When the stream is partitioned by a key (e.g. after keyBy), each key gets isolated state. You obtain a **factory** once (from context, store name, **namespace**, and **key_group**), then create state **per primary key** (the stream key for the current record). + +### 4.1 keyGroup, key (primaryKey), and namespace + +| Term | API parameter | Meaning | +|------|----------------|---------| +| **key_group** | `key_group` when creating the factory | The **keyed group**: identifies which keyed partition/group this state belongs to (e.g. one group for "counters", another for "sessions"). | +| **key** | The argument to factory methods (e.g. `new_keyed_value(primary_key)`) | The **value of the stream key** for the current record (e.g. user ID, partition key). Each distinct key value gets isolated state. | +| **namespace** | `namespace` (bytes) when creating the factory | **If a window function is present**, use the **window identifier as bytes**. **Without windows**, pass **empty bytes** (e.g. `b""`). | + +### 4.2 Factory constructor summary (keyed) + +| Factory | With codec | AutoCodec | +|---------|------------|-----------| +| KeyedValueStateFactory | `KeyedValueStateFactory.from_context(ctx, store_name, namespace, key_group, value_codec)` | `KeyedValueStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, value_type=None)` | +| KeyedListStateFactory | `KeyedListStateFactory.from_context(ctx, store_name, namespace, key_group, value_codec)` | `KeyedListStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, value_type=None)` | +| KeyedMapStateFactory | `KeyedMapStateFactory.from_context(ctx, store_name, namespace, key_group, key_codec, value_codec)` | `KeyedMapStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, value_codec)` | +| KeyedPriorityQueueStateFactory | `KeyedPriorityQueueStateFactory.from_context(ctx, store_name, namespace, key_group, item_codec)` | `KeyedPriorityQueueStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, item_type=None)` | +| KeyedAggregatingStateFactory | `KeyedAggregatingStateFactory.from_context(ctx, store_name, namespace, key_group, acc_codec, agg_func)` | `KeyedAggregatingStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, agg_func, acc_type=None)` | +| KeyedReducingStateFactory | `KeyedReducingStateFactory.from_context(ctx, store_name, namespace, key_group, value_codec, reduce_func)` | `KeyedReducingStateFactory.from_context_auto_codec(ctx, store_name, namespace, key_group, reduce_func, value_type=None)` | + +You can also use the corresponding `ctx.getOrCreateKeyed*Factory(...)` methods, which delegate to these constructors. + +--- + +## 5. Example: ValueState with from_context_auto_codec + +```python +from fs_api import FSProcessorDriver, Context +from fs_api.store import ValueState + +class CounterProcessor(FSProcessorDriver): + def process(self, ctx: Context, source_id: int, data: bytes): + # Create state per message (or cache in init) + state = ValueState.from_context_auto_codec(ctx, "my-store") + cur, _ = state.value() or (0, False) + state.update(cur + 1) + ctx.emit(str(cur + 1).encode(), 0) +``` + +Same pattern for other state types: use `XxxState.from_context(ctx, store_name, ...)` or `XxxState.from_context_auto_codec(ctx, store_name)` as in the tables above. + +--- + +## 6. See also + +- [Python SDK Guide](python-sdk-guide.md) — main guide for fs_api, fs_client, and basic Context/KvStore usage. +- [Go SDK Guide — Advanced State API](../Go-SDK/go-sdk-guide.md#7-advanced-state-api) — equivalent API in the Go SDK. diff --git a/docs/Python-SDK/python-sdk-guide-zh.md b/docs/Python-SDK/python-sdk-guide-zh.md index 26fe31fc..ed2ee238 100644 --- a/docs/Python-SDK/python-sdk-guide-zh.md +++ b/docs/Python-SDK/python-sdk-guide-zh.md @@ -148,3 +148,11 @@ with FsClient(host="10.0.0.1", port=8080) as client: | BadRequestError (400) | YAML 配置不满足规范或 Kafka 参数错误 | 检查 WasmTaskBuilder 中的配置项。 | | ServerError (500) | Server 侧运行时环境（如 RocksDB）异常 | 检查服务端 conf/config.yaml 存储路径权限。 | | NotFoundError (404) | 操作了不存在的函数或无效的 Checkpoint | 确认函数名是否输入正确。 | + +--- + +## 五、Advanced State API（高级状态 API） + +带类型状态（ValueState、ListState、MapState、Keyed\* 工厂等）及 `from_context` / `from_context_auto_codec` 用法请参见独立文档： + +- **[Python SDK — 高级状态 API](python-sdk-advanced-state-api-zh.md)** diff --git a/docs/Python-SDK/python-sdk-guide.md b/docs/Python-SDK/python-sdk-guide.md index 29bb0d37..533ba2c3 100644 --- a/docs/Python-SDK/python-sdk-guide.md +++ b/docs/Python-SDK/python-sdk-guide.md @@ -148,3 +148,11 @@ with FsClient(host="10.0.0.1", port=8080) as client: | BadRequestError (400) | YAML configuration does not meet specifications or Kafka parameters are incorrect | Check configuration items in WasmTaskBuilder. | | ServerError (500) | Server-side runtime environment (e.g., RocksDB) exception | Check permissions of storage path in server conf/config.yaml. | | NotFoundError (404) | Operating on a non-existent function or invalid Checkpoint | Confirm if the function name is correct. | + +--- + +## 5. Advanced State API + +For typed state (ValueState, ListState, MapState, Keyed\* factories, etc.) and `from_context` / `from_context_auto_codec` usage, see the dedicated document: + +- **[Python SDK — Advanced State API](python-sdk-advanced-state-api.md)** diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py index 499d969e..b2bc60c1 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Generic, Optional, Protocol, Tuple, TypeVar +from typing import Any, Generic, Optional, Protocol, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -55,6 +55,36 @@ def __init__( self._acc_codec = acc_codec self._agg_func = agg_func + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + acc_codec: Codec[ACC], + agg_func: "AggregateFunc[T_agg, ACC, R]", + ) -> "KeyedAggregatingStateFactory[T_agg, ACC, R]": + """Create a KeyedAggregatingStateFactory from a context and store name (for keyed operators).""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, namespace, key_group, acc_codec, agg_func) + + @classmethod + def from_context_auto_codec( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + agg_func: "AggregateFunc[T_agg, ACC, R]", + acc_type: Optional[type] = None, + ) -> "KeyedAggregatingStateFactory[T_agg, ACC, R]": + """Create a KeyedAggregatingStateFactory with default accumulator codec from context and store name.""" + from ..codec import PickleCodec, default_codec_for + store = ctx.getOrCreateKVStore(store_name) + codec = default_codec_for(acc_type) if acc_type is not None else PickleCodec() + return cls(store, namespace, key_group, codec, agg_func) + def new_aggregating(self, key_codec: Codec[K]) -> "KeyedAggregatingState[K, T_agg, ACC, R]": ensure_ordered_key_codec(key_codec, "keyed aggregating") return KeyedAggregatingState( diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py index 72d12d4f..c3aca8e5 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py @@ -11,7 +11,7 @@ # limitations under the License. import struct -from typing import Generic, List, Optional, TypeVar +from typing import Any, Generic, List, Optional, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -45,6 +45,34 @@ def __init__( self._key_group = key_group self._value_codec = value_codec + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ) -> "KeyedListStateFactory[V]": + """Create a KeyedListStateFactory from a context and store name (for keyed operators).""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, namespace, key_group, value_codec) + + @classmethod + def from_context_auto_codec( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + value_type: Optional[type] = None, + ) -> "KeyedListStateFactory[V]": + """Create a KeyedListStateFactory with default codec from context and store name.""" + from ..codec import PickleCodec, default_codec_for + store = ctx.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return cls(store, namespace, key_group, codec) + def new_list(self, key_codec: Codec[K]) -> "KeyedListState[K, V]": ensure_ordered_key_codec(key_codec, "keyed list") return KeyedListState( diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py index 097e5551..912ae082 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py @@ -11,7 +11,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Generic, Iterator, List, Optional, Tuple, TypeVar +from typing import Any, Generic, Iterator, List, Optional, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -55,6 +55,34 @@ def __init__( self._map_key_codec = map_key_codec self._map_value_codec = map_value_codec + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + map_key_codec: Codec[MK], + map_value_codec: Codec[MV], + ) -> "KeyedMapStateFactory[MK, MV]": + """Create a KeyedMapStateFactory from a context and store name (for keyed operators).""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, namespace, key_group, map_key_codec, map_value_codec) + + @classmethod + def from_context_auto_codec( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + value_codec: Codec[MV], + ) -> "KeyedMapStateFactory[MK, MV]": + """Create a KeyedMapStateFactory with default (bytes) map key codec from context and store name.""" + from ..codec import BytesCodec + store = ctx.getOrCreateKVStore(store_name) + return cls(store, namespace, key_group, BytesCodec(), value_codec) + def new_map(self, key_codec: Codec[K]) -> "KeyedMapState[K, MK, MV]": ensure_ordered_key_codec(key_codec, "keyed map") return KeyedMapState( diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py index 70aba4c8..733d7f25 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Generic, Iterator, Optional, Tuple, TypeVar +from typing import Any, Generic, Iterator, Optional, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -45,6 +45,34 @@ def __init__( self._key_group = key_group self._value_codec = value_codec + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ) -> "KeyedPriorityQueueStateFactory[V]": + """Create a KeyedPriorityQueueStateFactory from a context and store name (for keyed operators).""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, namespace, key_group, value_codec) + + @classmethod + def from_context_auto_codec( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + item_type: Optional[type] = None, + ) -> "KeyedPriorityQueueStateFactory[V]": + """Create a KeyedPriorityQueueStateFactory with default codec from context and store name.""" + from ..codec import IntCodec, default_codec_for + store = ctx.getOrCreateKVStore(store_name) + codec = default_codec_for(item_type) if item_type is not None else IntCodec() + return cls(store, namespace, key_group, codec) + def new_priority_queue(self, key_codec: Codec[K]) -> "KeyedPriorityQueueState[K, V]": ensure_ordered_key_codec(key_codec, "keyed priority queue") return KeyedPriorityQueueState( diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py index 8813815c..b174e473 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Generic, Optional, Tuple, TypeVar +from typing import Any, Callable, Generic, Optional, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -48,6 +48,36 @@ def __init__( self._value_codec = value_codec self._reduce_func = reduce_func + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + reduce_func: ReduceFunc[V], + ) -> "KeyedReducingStateFactory[V]": + """Create a KeyedReducingStateFactory from a context and store name (for keyed operators).""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, namespace, key_group, value_codec, reduce_func) + + @classmethod + def from_context_auto_codec( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + reduce_func: ReduceFunc[V], + value_type: Optional[type] = None, + ) -> "KeyedReducingStateFactory[V]": + """Create a KeyedReducingStateFactory with default value codec from context and store name.""" + from ..codec import PickleCodec, default_codec_for + store = ctx.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return cls(store, namespace, key_group, codec, reduce_func) + def new_reducing(self, key_codec: Codec[K]) -> "KeyedReducingState[K, V]": ensure_ordered_key_codec(key_codec, "keyed reducing") return KeyedReducingState( diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py index bf8f41d0..a63a8809 100644 --- a/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py @@ -10,13 +10,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Generic, Optional, TypeVar +from typing import Any, Generic, Optional, TypeVar from ..codec import Codec from ..complexkey import ComplexKey from ..error import KvError from ..store import KvStore +from ._keyed_common import ensure_ordered_key_codec + K = TypeVar("K") V = TypeVar("V") @@ -42,6 +44,34 @@ def __init__( self._key_group = key_group self._value_codec = value_codec + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ) -> "KeyedValueStateFactory[V]": + """Create a KeyedValueStateFactory from a context and store name (for keyed operators).""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, namespace, key_group, value_codec) + + @classmethod + def from_context_auto_codec( + cls, + ctx: Any, + store_name: str, + namespace: bytes, + key_group: bytes, + value_type: Optional[type] = None, + ) -> "KeyedValueStateFactory[V]": + """Create a KeyedValueStateFactory with default codec from context and store name.""" + from ..codec import PickleCodec, default_codec_for + store = ctx.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return cls(store, namespace, key_group, codec) + def new_value(self, key_codec: Codec[K]) -> "KeyedValueState[K, V]": ensure_ordered_key_codec(key_codec, "keyed value") return KeyedValueState( diff --git a/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py index 52daa5c0..79fa20c9 100644 --- a/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Generic, Optional, Protocol, Tuple, TypeVar +from typing import Any, Generic, Optional, Protocol, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -59,6 +59,30 @@ def __init__( user_key=b"", ) + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + acc_codec: Codec[ACC], + agg_func: AggregateFunc[T, ACC, R], + ) -> "AggregatingState[T, ACC, R]": + """Create an AggregatingState from a context and store name.""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, acc_codec, agg_func) + + @classmethod + def from_context_auto_codec( + cls, + ctx: Any, + store_name: str, + agg_func: AggregateFunc[T, ACC, R], + ) -> "AggregatingState[T, ACC, R]": + """Create an AggregatingState with default (pickle) accumulator codec from context and store name.""" + from ..codec import PickleCodec + store = ctx.getOrCreateKVStore(store_name) + return cls(store, PickleCodec(), agg_func) + def add(self, value: T) -> None: raw = self._store.get(self._ck) if raw is None: diff --git a/python/functionstream-api/src/fs_api/store/structures/list_state.py b/python/functionstream-api/src/fs_api/store/structures/list_state.py index bfa53e13..3aa243df 100644 --- a/python/functionstream-api/src/fs_api/store/structures/list_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/list_state.py @@ -11,7 +11,7 @@ # limitations under the License. import struct -from typing import Generic, List, TypeVar +from typing import Generic, List, TypeVar, Any from ..codec import Codec from ..complexkey import ComplexKey @@ -36,6 +36,19 @@ def __init__(self, store: KvStore, codec: Codec[T]): user_key=b"", ) + @classmethod + def from_context(cls, ctx: Any, store_name: str, codec: Codec[T]) -> "ListState[T]": + """Create a ListState from a context and store name.""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, codec) + + @classmethod + def from_context_auto_codec(cls, ctx: Any, store_name: str) -> "ListState[T]": + """Create a ListState with default (pickle) codec from context and store name.""" + from ..codec import PickleCodec + store = ctx.getOrCreateKVStore(store_name) + return cls(store, PickleCodec()) + def add(self, value: T) -> None: payload = self._serialize_one(value) self._store.merge(self._ck, payload) diff --git a/python/functionstream-api/src/fs_api/store/structures/map_state.py b/python/functionstream-api/src/fs_api/store/structures/map_state.py index 4d3b9c59..be1ef96f 100644 --- a/python/functionstream-api/src/fs_api/store/structures/map_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/map_state.py @@ -56,6 +56,30 @@ def with_auto_key_codec( key_codec = default_codec_for(key_type) return cls(store, key_codec, value_codec) + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + key_codec: Codec[K], + value_codec: Codec[V], + ) -> "MapState[K, V]": + """Create a MapState from a context and store name.""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, key_codec, value_codec) + + @classmethod + def from_context_auto_key_codec( + cls, + ctx: Any, + store_name: str, + value_codec: Codec[V], + ) -> "MapState[K, V]": + """Create a MapState with default (bytes) key codec from context and store name.""" + from ..codec import BytesCodec + store = ctx.getOrCreateKVStore(store_name) + return cls(store, BytesCodec(), value_codec) + def put(self, key: K, value: V) -> None: encoded_key = self._key_codec.encode(key) encoded_value = self._value_codec.encode(value) diff --git a/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py index 6386181d..b71f9e2d 100644 --- a/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Generic, Iterator, Optional, Tuple, TypeVar +from typing import Any, Generic, Iterator, Optional, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -36,6 +36,19 @@ def __init__(self, store: KvStore, codec: Codec[T]): self._key = b"" self._namespace = b"" + @classmethod + def from_context(cls, ctx: Any, store_name: str, codec: Codec[T]) -> "PriorityQueueState[T]": + """Create a PriorityQueueState from a context and store name.""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, codec) + + @classmethod + def from_context_auto_codec(cls, ctx: Any, store_name: str) -> "PriorityQueueState[T]": + """Create a PriorityQueueState with default (int) codec from context and store name.""" + from ..codec import IntCodec + store = ctx.getOrCreateKVStore(store_name) + return cls(store, IntCodec()) + def _ck(self, user_key: bytes) -> ComplexKey: return ComplexKey( key_group=self._key_group, diff --git a/python/functionstream-api/src/fs_api/store/structures/reducing_state.py b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py index 7c1b298d..5738d029 100644 --- a/python/functionstream-api/src/fs_api/store/structures/reducing_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Generic, Optional, Tuple, TypeVar +from typing import Any, Callable, Generic, Optional, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -38,6 +38,30 @@ def __init__(self, store: KvStore, value_codec: Codec[V], reduce_func: ReduceFun user_key=b"", ) + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + value_codec: Codec[V], + reduce_func: ReduceFunc[V], + ) -> "ReducingState[V]": + """Create a ReducingState from a context and store name.""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, value_codec, reduce_func) + + @classmethod + def from_context_auto_codec( + cls, + ctx: Any, + store_name: str, + reduce_func: ReduceFunc[V], + ) -> "ReducingState[V]": + """Create a ReducingState with default (pickle) value codec from context and store name.""" + from ..codec import PickleCodec + store = ctx.getOrCreateKVStore(store_name) + return cls(store, PickleCodec(), reduce_func) + def add(self, value: V) -> None: raw = self._store.get(self._ck) if raw is None: diff --git a/python/functionstream-api/src/fs_api/store/structures/value_state.py b/python/functionstream-api/src/fs_api/store/structures/value_state.py index 1e5a46aa..20616646 100644 --- a/python/functionstream-api/src/fs_api/store/structures/value_state.py +++ b/python/functionstream-api/src/fs_api/store/structures/value_state.py @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Generic, Optional, Tuple, TypeVar +from typing import Any, Generic, Optional, Tuple, TypeVar from ..codec import Codec from ..complexkey import ComplexKey @@ -35,6 +35,24 @@ def __init__(self, store: KvStore, codec: Codec[T]): user_key=b"", ) + @classmethod + def from_context( + cls, + ctx: Any, + store_name: str, + codec: Codec[T], + ) -> "ValueState[T]": + """Create a ValueState from a context and store name (same as ctx.getOrCreateValueState(store_name, codec)).""" + store = ctx.getOrCreateKVStore(store_name) + return cls(store, codec) + + @classmethod + def from_context_auto_codec(cls, ctx: Any, store_name: str) -> "ValueState[T]": + """Create a ValueState with default (pickle) codec from context and store name.""" + from ..codec import PickleCodec + store = ctx.getOrCreateKVStore(store_name) + return cls(store, PickleCodec()) + def update(self, value: T) -> None: self._store.put(self._ck, self._codec.encode(value)) diff --git a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py index fe900983..6bbc05a7 100644 --- a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py +++ b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py @@ -28,10 +28,6 @@ KeyedPriorityQueueStateFactory, KeyedAggregatingStateFactory, KeyedReducingStateFactory, - PickleCodec, - BytesCodec, - IntCodec, - default_codec_for, ) from .fs_collector import emit, emit_watermark @@ -80,137 +76,108 @@ def getConfig(self) -> Dict[str, str]: return self._CONFIG.copy() def getOrCreateValueState(self, store_name: str, codec: Codec) -> ValueState: - store = self.getOrCreateKVStore(store_name) - return ValueState(store, codec) + return ValueState.from_context(self, store_name, codec) def getOrCreateValueStateAutoCodec(self, store_name: str) -> ValueState: - store = self.getOrCreateKVStore(store_name) - return ValueState(store, PickleCodec()) + return ValueState.from_context_auto_codec(self, store_name) def getOrCreateMapState(self, store_name: str, key_codec: Codec, value_codec: Codec) -> MapState: - store = self.getOrCreateKVStore(store_name) - return MapState(store, key_codec, value_codec) + return MapState.from_context(self, store_name, key_codec, value_codec) def getOrCreateMapStateAutoKeyCodec(self, store_name: str, value_codec: Codec) -> MapState: - store = self.getOrCreateKVStore(store_name) - return MapState(store, BytesCodec(), value_codec) + return MapState.from_context_auto_key_codec(self, store_name, value_codec) def getOrCreateListState(self, store_name: str, codec: Codec) -> ListState: - store = self.getOrCreateKVStore(store_name) - return ListState(store, codec) + return ListState.from_context(self, store_name, codec) def getOrCreateListStateAutoCodec(self, store_name: str) -> ListState: - store = self.getOrCreateKVStore(store_name) - return ListState(store, PickleCodec()) + return ListState.from_context_auto_codec(self, store_name) def getOrCreatePriorityQueueState(self, store_name: str, codec: Codec) -> PriorityQueueState: - store = self.getOrCreateKVStore(store_name) - return PriorityQueueState(store, codec) + return PriorityQueueState.from_context(self, store_name, codec) def getOrCreatePriorityQueueStateAutoCodec(self, store_name: str) -> PriorityQueueState: - store = self.getOrCreateKVStore(store_name) - return PriorityQueueState(store, IntCodec()) + return PriorityQueueState.from_context_auto_codec(self, store_name) def getOrCreateAggregatingState( self, store_name: str, acc_codec: Codec, agg_func: object ) -> AggregatingState: - store = self.getOrCreateKVStore(store_name) - return AggregatingState(store, acc_codec, agg_func) + return AggregatingState.from_context(self, store_name, acc_codec, agg_func) def getOrCreateAggregatingStateAutoCodec( self, store_name: str, agg_func: object ) -> AggregatingState: - store = self.getOrCreateKVStore(store_name) - return AggregatingState(store, PickleCodec(), agg_func) + return AggregatingState.from_context_auto_codec(self, store_name, agg_func) def getOrCreateReducingState( self, store_name: str, value_codec: Codec, reduce_func: object ) -> ReducingState: - store = self.getOrCreateKVStore(store_name) - return ReducingState(store, value_codec, reduce_func) + return ReducingState.from_context(self, store_name, value_codec, reduce_func) def getOrCreateReducingStateAutoCodec( self, store_name: str, reduce_func: object ) -> ReducingState: - store = self.getOrCreateKVStore(store_name) - return ReducingState(store, PickleCodec(), reduce_func) + return ReducingState.from_context_auto_codec(self, store_name, reduce_func) def getOrCreateKeyedListStateFactory( self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec ) -> KeyedListStateFactory: - store = self.getOrCreateKVStore(store_name) - return KeyedListStateFactory(store, namespace, key_group, value_codec) + return KeyedListStateFactory.from_context(self, store_name, namespace, key_group, value_codec) def getOrCreateKeyedListStateFactoryAutoCodec( self, store_name: str, namespace: bytes, key_group: bytes, value_type=None ) -> KeyedListStateFactory: - store = self.getOrCreateKVStore(store_name) - codec = default_codec_for(value_type) if value_type is not None else PickleCodec() - return KeyedListStateFactory(store, namespace, key_group, codec) + return KeyedListStateFactory.from_context_auto_codec(self, store_name, namespace, key_group, value_type) def getOrCreateKeyedValueStateFactory( self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec ) -> KeyedValueStateFactory: - store = self.getOrCreateKVStore(store_name) - return KeyedValueStateFactory(store, namespace, key_group, value_codec) + return KeyedValueStateFactory.from_context(self, store_name, namespace, key_group, value_codec) def getOrCreateKeyedValueStateFactoryAutoCodec( self, store_name: str, namespace: bytes, key_group: bytes, value_type=None ) -> KeyedValueStateFactory: - store = self.getOrCreateKVStore(store_name) - codec = default_codec_for(value_type) if value_type is not None else PickleCodec() - return KeyedValueStateFactory(store, namespace, key_group, codec) + return KeyedValueStateFactory.from_context_auto_codec(self, store_name, namespace, key_group, value_type) def getOrCreateKeyedMapStateFactory( self, store_name: str, namespace: bytes, key_group: bytes, key_codec: Codec, value_codec: Codec ) -> KeyedMapStateFactory: - store = self.getOrCreateKVStore(store_name) - return KeyedMapStateFactory(store, namespace, key_group, key_codec, value_codec) + return KeyedMapStateFactory.from_context(self, store_name, namespace, key_group, key_codec, value_codec) def getOrCreateKeyedMapStateFactoryAutoCodec( self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec ) -> KeyedMapStateFactory: - store = self.getOrCreateKVStore(store_name) - return KeyedMapStateFactory(store, namespace, key_group, BytesCodec(), value_codec) + return KeyedMapStateFactory.from_context_auto_codec(self, store_name, namespace, key_group, value_codec) def getOrCreateKeyedPriorityQueueStateFactory( self, store_name: str, namespace: bytes, key_group: bytes, item_codec: Codec ) -> KeyedPriorityQueueStateFactory: - store = self.getOrCreateKVStore(store_name) - return KeyedPriorityQueueStateFactory(store, namespace, key_group, item_codec) + return KeyedPriorityQueueStateFactory.from_context(self, store_name, namespace, key_group, item_codec) def getOrCreateKeyedPriorityQueueStateFactoryAutoCodec( self, store_name: str, namespace: bytes, key_group: bytes, item_type=None ) -> KeyedPriorityQueueStateFactory: - store = self.getOrCreateKVStore(store_name) - codec = default_codec_for(item_type) if item_type is not None else IntCodec() - return KeyedPriorityQueueStateFactory(store, namespace, key_group, codec) + return KeyedPriorityQueueStateFactory.from_context_auto_codec(self, store_name, namespace, key_group, item_type) def getOrCreateKeyedAggregatingStateFactory( self, store_name: str, namespace: bytes, key_group: bytes, acc_codec: Codec, agg_func: object ) -> KeyedAggregatingStateFactory: - store = self.getOrCreateKVStore(store_name) - return KeyedAggregatingStateFactory(store, namespace, key_group, acc_codec, agg_func) + return KeyedAggregatingStateFactory.from_context(self, store_name, namespace, key_group, acc_codec, agg_func) def getOrCreateKeyedAggregatingStateFactoryAutoCodec( self, store_name: str, namespace: bytes, key_group: bytes, agg_func: object, acc_type=None ) -> KeyedAggregatingStateFactory: - store = self.getOrCreateKVStore(store_name) - codec = default_codec_for(acc_type) if acc_type is not None else PickleCodec() - return KeyedAggregatingStateFactory(store, namespace, key_group, codec, agg_func) + return KeyedAggregatingStateFactory.from_context_auto_codec(self, store_name, namespace, key_group, agg_func, acc_type) def getOrCreateKeyedReducingStateFactory( self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec, reduce_func: object ) -> KeyedReducingStateFactory: - store = self.getOrCreateKVStore(store_name) - return KeyedReducingStateFactory(store, namespace, key_group, value_codec, reduce_func) + return KeyedReducingStateFactory.from_context(self, store_name, namespace, key_group, value_codec, reduce_func) def getOrCreateKeyedReducingStateFactoryAutoCodec( self, store_name: str, namespace: bytes, key_group: bytes, reduce_func: object, value_type=None ) -> KeyedReducingStateFactory: - store = self.getOrCreateKVStore(store_name) - codec = default_codec_for(value_type) if value_type is not None else PickleCodec() - return KeyedReducingStateFactory(store, namespace, key_group, codec, reduce_func) + return KeyedReducingStateFactory.from_context_auto_codec(self, store_name, namespace, key_group, reduce_func, value_type) __all__ = ['WitContext', 'convert_config_to_dict'] From b3f0083626f0e8b7a32fbde95afcd5d142d3bcf5 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 11 Mar 2026 17:16:55 +0800 Subject: [PATCH 8/9] update --- go-sdk/state/codec/default_codec.go | 4 ++-- go-sdk/state/codec/int_codec.go | 33 +++++++++++++++++++++++++++++ go-sdk/state/codec/uint_codec.go | 33 +++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 go-sdk/state/codec/int_codec.go create mode 100644 go-sdk/state/codec/uint_codec.go diff --git a/go-sdk/state/codec/default_codec.go b/go-sdk/state/codec/default_codec.go index 16fa868d..984a050a 100644 --- a/go-sdk/state/codec/default_codec.go +++ b/go-sdk/state/codec/default_codec.go @@ -42,13 +42,13 @@ func DefaultCodecFor[V any]() (Codec[V], error) { case reflect.String: return any(StringCodec{}).(Codec[V]), nil case reflect.Int: - return any(Int64Codec{}).(Codec[V]), nil + return any(IntCodec{}).(Codec[V]), nil case reflect.Int8: return any(Int8Codec{}).(Codec[V]), nil case reflect.Int16: return any(Int16Codec{}).(Codec[V]), nil case reflect.Uint: - return any(Uint64Codec{}).(Codec[V]), nil + return any(UintCodec{}).(Codec[V]), nil case reflect.Uint8: return any(Uint8Codec{}).(Codec[V]), nil case reflect.Uint16: diff --git a/go-sdk/state/codec/int_codec.go b/go-sdk/state/codec/int_codec.go new file mode 100644 index 00000000..402ecbbc --- /dev/null +++ b/go-sdk/state/codec/int_codec.go @@ -0,0 +1,33 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +// IntCodec implements Codec[int] by delegating to Int64Codec. +// It is used for reflect.Int (platform-sized int) so that DefaultCodecFor[int]() +// returns a valid Codec[int] instead of panicking on type assertion. +type IntCodec struct{} + +var _ Codec[int] = IntCodec{} + +func (c IntCodec) Encode(value int) ([]byte, error) { + return Int64Codec{}.Encode(int64(value)) +} + +func (c IntCodec) Decode(data []byte) (int, error) { + v, err := Int64Codec{}.Decode(data) + return int(v), err +} + +func (c IntCodec) EncodedSize() int { return 8 } + +func (c IntCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/uint_codec.go b/go-sdk/state/codec/uint_codec.go new file mode 100644 index 00000000..b2d618df --- /dev/null +++ b/go-sdk/state/codec/uint_codec.go @@ -0,0 +1,33 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +// UintCodec implements Codec[uint] by delegating to Uint64Codec. +// It is used for reflect.Uint (platform-sized uint) so that DefaultCodecFor[uint]() +// returns a valid Codec[uint] instead of panicking on type assertion. +type UintCodec struct{} + +var _ Codec[uint] = UintCodec{} + +func (c UintCodec) Encode(value uint) ([]byte, error) { + return Uint64Codec{}.Encode(uint64(value)) +} + +func (c UintCodec) Decode(data []byte) (uint, error) { + v, err := Uint64Codec{}.Decode(data) + return uint(v), err +} + +func (c UintCodec) EncodedSize() int { return 8 } + +func (c UintCodec) IsOrderedKeyCodec() bool { return true } From 10dd416337aa64811e390c8447cac909842c0bcb Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 11 Mar 2026 18:22:38 +0800 Subject: [PATCH 9/9] fix --- go-sdk/state/keyed/keyed_map_state.go | 10 ++++++++-- go-sdk/state/keyed/keyed_priority_queue_state.go | 10 ++++++++-- go-sdk/state/structures/map.go | 10 ++++++++-- go-sdk/state/structures/priority_queue.go | 10 ++++++++-- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/go-sdk/state/keyed/keyed_map_state.go b/go-sdk/state/keyed/keyed_map_state.go index fbc92132..b2c15ff3 100644 --- a/go-sdk/state/keyed/keyed_map_state.go +++ b/go-sdk/state/keyed/keyed_map_state.go @@ -181,8 +181,14 @@ func (s *KeyedMapState[MK, MV]) All() iter.Seq2[MK, MV] { return } - k, _ := s.factory.mapKeyCodec.Decode(keyRaw) - v, _ := s.factory.mapValueCodec.Decode(valRaw) + k, err := s.factory.mapKeyCodec.Decode(keyRaw) + if err != nil { + continue // skip entry on decode error to avoid yielding corrupted zero values + } + v, err := s.factory.mapValueCodec.Decode(valRaw) + if err != nil { + continue + } if !yield(k, v) { return diff --git a/go-sdk/state/keyed/keyed_priority_queue_state.go b/go-sdk/state/keyed/keyed_priority_queue_state.go index f18f8112..156ddb32 100644 --- a/go-sdk/state/keyed/keyed_priority_queue_state.go +++ b/go-sdk/state/keyed/keyed_priority_queue_state.go @@ -145,7 +145,10 @@ func (s *KeyedPriorityQueueState[V]) Poll() (V, bool, error) { return val, found, err } - userKey, _ := s.factory.valueCodec.Encode(val) + userKey, err := s.factory.valueCodec.Encode(val) + if err != nil { + return val, true, fmt.Errorf("encode pq element for delete failed: %w", err) + } ck := api.ComplexKey{ KeyGroup: s.factory.groupKey, Key: s.primaryKey, @@ -188,7 +191,10 @@ func (s *KeyedPriorityQueueState[V]) All() iter.Seq[V] { return } - v, _ := s.factory.valueCodec.Decode(userKey) + v, err := s.factory.valueCodec.Decode(userKey) + if err != nil { + continue // skip entry on decode error to avoid yielding corrupted zero values + } if !yield(v) { return } diff --git a/go-sdk/state/structures/map.go b/go-sdk/state/structures/map.go index a5adcecc..bdd08f0a 100644 --- a/go-sdk/state/structures/map.go +++ b/go-sdk/state/structures/map.go @@ -156,8 +156,14 @@ func (m *MapState[K, V]) All() iter.Seq2[K, V] { return } - k, _ := m.keyCodec.Decode(keyRaw) - v, _ := m.valueCodec.Decode(valRaw) + k, err := m.keyCodec.Decode(keyRaw) + if err != nil { + continue // skip entry on decode error to avoid yielding corrupted zero values + } + v, err := m.valueCodec.Decode(valRaw) + if err != nil { + continue + } if !yield(k, v) { return diff --git a/go-sdk/state/structures/priority_queue.go b/go-sdk/state/structures/priority_queue.go index 3e5d7781..8345cb97 100644 --- a/go-sdk/state/structures/priority_queue.go +++ b/go-sdk/state/structures/priority_queue.go @@ -115,7 +115,10 @@ func (q *PriorityQueueState[T]) Poll() (T, bool, error) { return val, found, err } - userKey, _ := q.valueCodec.Encode(val) + userKey, err := q.valueCodec.Encode(val) + if err != nil { + return val, true, fmt.Errorf("encode pq element for delete failed: %w", err) + } if err = q.store.Delete(q.ck(userKey)); err != nil { return val, true, err } @@ -149,7 +152,10 @@ func (q *PriorityQueueState[T]) All() iter.Seq[T] { return } - v, _ := q.valueCodec.Decode(userKey) + v, err := q.valueCodec.Decode(userKey) + if err != nil { + continue // skip entry on decode error to avoid yielding corrupted zero values + } if !yield(v) { return }