diff --git a/go-sdk/impl/context.go b/go-sdk/impl/context.go index 54d374a1..a6b5626b 100644 --- a/go-sdk/impl/context.go +++ b/go-sdk/impl/context.go @@ -13,15 +13,12 @@ package impl import ( - "strings" - "sync" - "github.com/functionstream/function-stream/go-sdk/api" "github.com/functionstream/function-stream/go-sdk/bindings/functionstream/core/collector" + "strings" ) type runtimeContext struct { - mu sync.RWMutex config map[string]string stores map[string]*storeImpl closed bool @@ -35,9 +32,7 @@ func newRuntimeContext(config map[string]string) *runtimeContext { } func (c *runtimeContext) Emit(targetID uint32, data []byte) error { - c.mu.RLock() closed := c.closed - c.mu.RUnlock() if closed { return api.NewError(api.ErrRuntimeClosed, "emit on closed context") } @@ -46,9 +41,7 @@ func (c *runtimeContext) Emit(targetID uint32, data []byte) error { } func (c *runtimeContext) EmitWatermark(targetID uint32, watermark uint64) error { - c.mu.RLock() closed := c.closed - c.mu.RUnlock() if closed { return api.NewError(api.ErrRuntimeClosed, "emit watermark on closed context") } @@ -62,8 +55,6 @@ func (c *runtimeContext) GetOrCreateStore(name string) (api.Store, error) { return nil, api.NewError(api.ErrStoreInvalidName, "store name must not be empty") } - c.mu.Lock() - defer c.mu.Unlock() if c.closed { return nil, api.NewError(api.ErrRuntimeClosed, "store request on closed context") } @@ -77,21 +68,16 @@ func (c *runtimeContext) GetOrCreateStore(name string) (api.Store, error) { } func (c *runtimeContext) Config() map[string]string { - c.mu.RLock() - defer c.mu.RUnlock() return cloneStringMap(c.config) } func (c *runtimeContext) Close() error { - c.mu.Lock() if c.closed { - c.mu.Unlock() return nil } c.closed = true stores := c.stores c.stores = make(map[string]*storeImpl) - c.mu.Unlock() var firstErr error for _, store := range stores { diff --git a/go-sdk/impl/state.go b/go-sdk/impl/state.go new file mode 100644 index 00000000..2befbedb --- /dev/null +++ b/go-sdk/impl/state.go @@ -0,0 +1,144 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package impl + +import ( + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/keyed" + "github.com/functionstream/function-stream/go-sdk/state/structures" +) + +func getStoreFromContext(ctx api.Context, storeName string) (*storeImpl, error) { + store, err := ctx.GetOrCreateStore(storeName) + if err != nil { + return nil, err + } + s, ok := store.(*storeImpl) + if !ok { + return nil, api.NewError(api.ErrStoreInternal, "store %q is not the default implementation", storeName) + } + return s, nil +} + +func NewValueState[T any](ctx api.Context, storeName string, valueCodec codec.Codec[T]) (*structures.ValueState[T], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewValueState(s, valueCodec) +} + +func NewListState[T any](ctx api.Context, storeName string, itemCodec codec.Codec[T]) (*structures.ListState[T], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewListState(s, itemCodec) +} + +func NewMapState[K any, V any](ctx api.Context, storeName string, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*structures.MapState[K, V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewMapState(s, keyCodec, valueCodec) +} + +func NewMapStateAutoKeyCodec[K any, V any](ctx api.Context, storeName string, valueCodec codec.Codec[V]) (*structures.MapState[K, V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewMapStateAutoKeyCodec[K, V](s, valueCodec) +} + +func NewPriorityQueueState[T any](ctx api.Context, storeName string, itemCodec codec.Codec[T]) (*structures.PriorityQueueState[T], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewPriorityQueueState(s, itemCodec) +} + +func NewAggregatingState[T any, ACC any, R any](ctx api.Context, storeName string, accCodec codec.Codec[ACC], aggFunc structures.AggregateFunc[T, ACC, R]) (*structures.AggregatingState[T, ACC, R], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewAggregatingState(s, accCodec, aggFunc) +} + +func NewReducingState[V any](ctx api.Context, storeName string, valueCodec codec.Codec[V], reduceFunc structures.ReduceFunc[V]) (*structures.ReducingState[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return structures.NewReducingState(s, valueCodec, reduceFunc) +} + +func NewKeyedListStateFactory[V any](ctx api.Context, storeName, name string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedListStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedListStateFactory(s, name, keyGroup, valueCodec) +} + +func NewKeyedListStateFactoryAutoCodec[V any](ctx api.Context, storeName, name string, keyGroup []byte) (*keyed.KeyedListStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedListStateFactoryAutoCodec[V](s, name, keyGroup) +} + +func NewKeyedValueStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V]) (*keyed.KeyedValueStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedValueStateFactory(s, keyGroup, valueCodec) +} + +func NewKeyedMapStateFactory[MK any, MV any](ctx api.Context, storeName string, keyGroup []byte, keyCodec codec.Codec[MK], valueCodec codec.Codec[MV]) (*keyed.KeyedMapStateFactory[MK, MV], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedMapStateFactory(s, keyGroup, keyCodec, valueCodec) +} + +func NewKeyedPriorityQueueStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, itemCodec codec.Codec[V]) (*keyed.KeyedPriorityQueueStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedPriorityQueueStateFactory(s, keyGroup, itemCodec) +} + +func NewKeyedAggregatingStateFactory[T any, ACC any, R any](ctx api.Context, storeName string, keyGroup []byte, accCodec codec.Codec[ACC], aggFunc keyed.AggregateFunc[T, ACC, R]) (*keyed.KeyedAggregatingStateFactory[T, ACC, R], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedAggregatingStateFactory(s, keyGroup, accCodec, aggFunc) +} + +func NewKeyedReducingStateFactory[V any](ctx api.Context, storeName string, keyGroup []byte, valueCodec codec.Codec[V], reduceFunc keyed.ReduceFunc[V]) (*keyed.KeyedReducingStateFactory[V], error) { + s, err := getStoreFromContext(ctx, storeName) + if err != nil { + return nil, err + } + return keyed.NewKeyedReducingStateFactory(s, keyGroup, valueCodec, reduceFunc) +} diff --git a/go-sdk/state/codec/bool_codec.go b/go-sdk/state/codec/bool_codec.go new file mode 100644 index 00000000..d18fbe63 --- /dev/null +++ b/go-sdk/state/codec/bool_codec.go @@ -0,0 +1,42 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import "fmt" + +type BoolCodec struct{} + +func (c BoolCodec) Encode(value bool) ([]byte, error) { + if value { + return []byte{1}, nil + } + return []byte{0}, nil +} + +func (c BoolCodec) Decode(data []byte) (bool, error) { + if len(data) != 1 { + return false, fmt.Errorf("invalid bool payload length: %d", len(data)) + } + switch data[0] { + case 0: + return false, nil + case 1: + return true, nil + default: + return false, fmt.Errorf("invalid bool payload byte: %d", data[0]) + } +} + +func (c BoolCodec) EncodedSize() int { return 1 } + +func (c BoolCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/bytes_codec.go b/go-sdk/state/codec/bytes_codec.go new file mode 100644 index 00000000..b428d506 --- /dev/null +++ b/go-sdk/state/codec/bytes_codec.go @@ -0,0 +1,25 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import "github.com/functionstream/function-stream/go-sdk/state/common" + +type BytesCodec struct{} + +func (c BytesCodec) Encode(value []byte) ([]byte, error) { return common.DupBytes(value), nil } + +func (c BytesCodec) Decode(data []byte) ([]byte, error) { return common.DupBytes(data), nil } + +func (c BytesCodec) EncodedSize() int { return -1 } + +func (c BytesCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/default_codec.go b/go-sdk/state/codec/default_codec.go new file mode 100644 index 00000000..7a18321b --- /dev/null +++ b/go-sdk/state/codec/default_codec.go @@ -0,0 +1,51 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "fmt" + "reflect" +) + +func DefaultCodecFor[V any]() (Codec[V], error) { + var zero V + t := reflect.TypeOf(zero) + if t == nil { + return nil, fmt.Errorf("default codec: type parameter V must not be interface without type constraint") + } + k := t.Kind() + switch k { + case reflect.Bool: + return any(BoolCodec{}).(Codec[V]), nil + case reflect.Int32: + return any(OrderedInt32Codec{}).(Codec[V]), nil + case reflect.Int64: + return any(OrderedInt64Codec{}).(Codec[V]), nil + case reflect.Uint32: + return any(OrderedUint32Codec{}).(Codec[V]), nil + case reflect.Uint64: + return any(OrderedUint64Codec{}).(Codec[V]), nil + case reflect.Float32: + return any(OrderedFloat32Codec{}).(Codec[V]), nil + case reflect.Float64: + return any(OrderedFloat64Codec{}).(Codec[V]), nil + case reflect.String: + return any(StringCodec{}).(Codec[V]), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Uint, reflect.Uint8, reflect.Uint16: + return any(JSONCodec[V]{}).(Codec[V]), nil + case reflect.Struct, reflect.Map, reflect.Slice, reflect.Array, reflect.Interface: + return any(JSONCodec[V]{}).(Codec[V]), nil + default: + return any(JSONCodec[V]{}).(Codec[V]), nil + } +} diff --git a/go-sdk/state/codec/float32_codec.go b/go-sdk/state/codec/float32_codec.go new file mode 100644 index 00000000..8967596a --- /dev/null +++ b/go-sdk/state/codec/float32_codec.go @@ -0,0 +1,37 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" + "math" +) + +type Float32Codec struct{} + +func (c Float32Codec) Encode(value float32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, math.Float32bits(value)) + return out, nil +} + +func (c Float32Codec) Decode(data []byte) (float32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid float32 payload length: %d", len(data)) + } + return math.Float32frombits(binary.BigEndian.Uint32(data)), nil +} + +func (c Float32Codec) EncodedSize() int { return 4 } +func (c Float32Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/float64_codec.go b/go-sdk/state/codec/float64_codec.go new file mode 100644 index 00000000..1e5d484c --- /dev/null +++ b/go-sdk/state/codec/float64_codec.go @@ -0,0 +1,37 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" + "math" +) + +type Float64Codec struct{} + +func (c Float64Codec) Encode(value float64) ([]byte, error) { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, math.Float64bits(value)) + return out, nil +} + +func (c Float64Codec) Decode(data []byte) (float64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid float64 payload length: %d", len(data)) + } + return math.Float64frombits(binary.BigEndian.Uint64(data)), nil +} + +func (c Float64Codec) EncodedSize() int { return 8 } +func (c Float64Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/int32_codec.go b/go-sdk/state/codec/int32_codec.go new file mode 100644 index 00000000..40fe0507 --- /dev/null +++ b/go-sdk/state/codec/int32_codec.go @@ -0,0 +1,36 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" +) + +type Int32Codec struct{} + +func (c Int32Codec) Encode(value int32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, uint32(value)) + return out, nil +} + +func (c Int32Codec) Decode(data []byte) (int32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid int32 payload length: %d", len(data)) + } + return int32(binary.BigEndian.Uint32(data)), nil +} + +func (c Int32Codec) EncodedSize() int { return 4 } +func (c Int32Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/int64_codec.go b/go-sdk/state/codec/int64_codec.go new file mode 100644 index 00000000..60046e9a --- /dev/null +++ b/go-sdk/state/codec/int64_codec.go @@ -0,0 +1,36 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" +) + +type Int64Codec struct{} + +func (c Int64Codec) Encode(value int64) ([]byte, error) { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, uint64(value)) + return out, nil +} + +func (c Int64Codec) Decode(data []byte) (int64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid int64 payload length: %d", len(data)) + } + return int64(binary.BigEndian.Uint64(data)), nil +} + +func (c Int64Codec) EncodedSize() int { return 8 } +func (c Int64Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/interface.go b/go-sdk/state/codec/interface.go new file mode 100644 index 00000000..2e35b4f7 --- /dev/null +++ b/go-sdk/state/codec/interface.go @@ -0,0 +1,28 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +// Codec is the core encoding interface. EncodedSize reports encoded byte length: +// >0 for fixed-size, <=0 for variable-size. +// IsOrderedKeyCodec reports whether the encoding is byte-orderable (for use as map/keyed state key). +type Codec[T any] interface { + Encode(value T) ([]byte, error) + Decode(data []byte) (T, error) + EncodedSize() int + IsOrderedKeyCodec() bool +} + +func FixedEncodedSize[T any](c Codec[T]) (int, bool) { + n := c.EncodedSize() + return n, n > 0 +} diff --git a/go-sdk/state/codec/json_codec.go b/go-sdk/state/codec/json_codec.go new file mode 100644 index 00000000..abf65b8d --- /dev/null +++ b/go-sdk/state/codec/json_codec.go @@ -0,0 +1,30 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import "encoding/json" + +type JSONCodec[T any] struct{} + +func (c JSONCodec[T]) Encode(value T) ([]byte, error) { return json.Marshal(value) } + +func (c JSONCodec[T]) Decode(data []byte) (T, error) { + var out T + if err := json.Unmarshal(data, &out); err != nil { + return out, err + } + return out, nil +} + +func (c JSONCodec[T]) EncodedSize() int { return -1 } +func (c JSONCodec[T]) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/ordered_float32_codec.go b/go-sdk/state/codec/ordered_float32_codec.go new file mode 100644 index 00000000..ea9a3c68 --- /dev/null +++ b/go-sdk/state/codec/ordered_float32_codec.go @@ -0,0 +1,50 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" + "math" +) + +type OrderedFloat32Codec struct{} + +func (c OrderedFloat32Codec) Encode(value float32) ([]byte, error) { + bits := math.Float32bits(value) + if (bits & (uint32(1) << 31)) != 0 { + bits = ^bits + } else { + bits ^= uint32(1) << 31 + } + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, bits) + return out, nil +} + +func (c OrderedFloat32Codec) Decode(data []byte) (float32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid ordered float32 payload length: %d", len(data)) + } + encoded := binary.BigEndian.Uint32(data) + if (encoded & (uint32(1) << 31)) != 0 { + encoded ^= uint32(1) << 31 + } else { + encoded = ^encoded + } + return math.Float32frombits(encoded), nil +} + +func (c OrderedFloat32Codec) EncodedSize() int { return 4 } + +func (c OrderedFloat32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_float64_codec.go b/go-sdk/state/codec/ordered_float64_codec.go new file mode 100644 index 00000000..d032e2f0 --- /dev/null +++ b/go-sdk/state/codec/ordered_float64_codec.go @@ -0,0 +1,50 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" + "math" +) + +type OrderedFloat64Codec struct{} + +func (c OrderedFloat64Codec) Encode(value float64) ([]byte, error) { + bits := math.Float64bits(value) + if (bits & (uint64(1) << 63)) != 0 { + bits = ^bits + } else { + bits ^= uint64(1) << 63 + } + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, bits) + return out, nil +} + +func (c OrderedFloat64Codec) Decode(data []byte) (float64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid ordered float64 payload length: %d", len(data)) + } + encoded := binary.BigEndian.Uint64(data) + if (encoded & (uint64(1) << 63)) != 0 { + encoded ^= uint64(1) << 63 + } else { + encoded = ^encoded + } + return math.Float64frombits(encoded), nil +} + +func (c OrderedFloat64Codec) EncodedSize() int { return 8 } + +func (c OrderedFloat64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int32_codec.go b/go-sdk/state/codec/ordered_int32_codec.go new file mode 100644 index 00000000..1b48e92d --- /dev/null +++ b/go-sdk/state/codec/ordered_int32_codec.go @@ -0,0 +1,37 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" +) + +type OrderedInt32Codec struct{} + +func (c OrderedInt32Codec) Encode(value int32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, uint32(value)^(uint32(1)<<31)) + return out, nil +} + +func (c OrderedInt32Codec) Decode(data []byte) (int32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid ordered int32 payload length: %d", len(data)) + } + return int32(binary.BigEndian.Uint32(data) ^ (uint32(1) << 31)), nil +} + +func (c OrderedInt32Codec) EncodedSize() int { return 4 } + +func (c OrderedInt32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int64_codec.go b/go-sdk/state/codec/ordered_int64_codec.go new file mode 100644 index 00000000..dc8b5346 --- /dev/null +++ b/go-sdk/state/codec/ordered_int64_codec.go @@ -0,0 +1,27 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import "github.com/functionstream/function-stream/go-sdk/state/common" + +type OrderedInt64Codec struct{} + +func (c OrderedInt64Codec) Encode(value int64) ([]byte, error) { + return common.EncodeInt64Lex(value), nil +} + +func (c OrderedInt64Codec) Decode(data []byte) (int64, error) { return common.DecodeInt64Lex(data) } + +func (c OrderedInt64Codec) EncodedSize() int { return 8 } + +func (c OrderedInt64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_int_codec.go b/go-sdk/state/codec/ordered_int_codec.go new file mode 100644 index 00000000..ebdc9e90 --- /dev/null +++ b/go-sdk/state/codec/ordered_int_codec.go @@ -0,0 +1,37 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import "strconv" + +type OrderedIntCodec struct{} + +func (c OrderedIntCodec) Encode(value int) ([]byte, error) { + if strconv.IntSize == 32 { + return OrderedInt32Codec{}.Encode(int32(value)) + } + return OrderedInt64Codec{}.Encode(int64(value)) +} + +func (c OrderedIntCodec) Decode(data []byte) (int, error) { + if strconv.IntSize == 32 { + v, err := OrderedInt32Codec{}.Decode(data) + return int(v), err + } + v, err := OrderedInt64Codec{}.Decode(data) + return int(v), err +} + +func (c OrderedIntCodec) EncodedSize() int { return strconv.IntSize / 8 } + +func (c OrderedIntCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint32_codec.go b/go-sdk/state/codec/ordered_uint32_codec.go new file mode 100644 index 00000000..a6a5ce1e --- /dev/null +++ b/go-sdk/state/codec/ordered_uint32_codec.go @@ -0,0 +1,37 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" +) + +type OrderedUint32Codec struct{} + +func (c OrderedUint32Codec) Encode(value uint32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, value) + return out, nil +} + +func (c OrderedUint32Codec) Decode(data []byte) (uint32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid ordered uint32 payload length: %d", len(data)) + } + return binary.BigEndian.Uint32(data), nil +} + +func (c OrderedUint32Codec) EncodedSize() int { return 4 } + +func (c OrderedUint32Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint64_codec.go b/go-sdk/state/codec/ordered_uint64_codec.go new file mode 100644 index 00000000..239b21ff --- /dev/null +++ b/go-sdk/state/codec/ordered_uint64_codec.go @@ -0,0 +1,37 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" +) + +type OrderedUint64Codec struct{} + +func (c OrderedUint64Codec) Encode(value uint64) ([]byte, error) { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, value) + return out, nil +} + +func (c OrderedUint64Codec) Decode(data []byte) (uint64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid ordered uint64 payload length: %d", len(data)) + } + return binary.BigEndian.Uint64(data), nil +} + +func (c OrderedUint64Codec) EncodedSize() int { return 8 } + +func (c OrderedUint64Codec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/ordered_uint_codec.go b/go-sdk/state/codec/ordered_uint_codec.go new file mode 100644 index 00000000..6642d8d1 --- /dev/null +++ b/go-sdk/state/codec/ordered_uint_codec.go @@ -0,0 +1,37 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import "strconv" + +type OrderedUintCodec struct{} + +func (c OrderedUintCodec) Encode(value uint) ([]byte, error) { + if strconv.IntSize == 32 { + return OrderedUint32Codec{}.Encode(uint32(value)) + } + return OrderedUint64Codec{}.Encode(uint64(value)) +} + +func (c OrderedUintCodec) Decode(data []byte) (uint, error) { + if strconv.IntSize == 32 { + v, err := OrderedUint32Codec{}.Decode(data) + return uint(v), err + } + v, err := OrderedUint64Codec{}.Decode(data) + return uint(v), err +} + +func (c OrderedUintCodec) EncodedSize() int { return strconv.IntSize / 8 } + +func (c OrderedUintCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/string_codec.go b/go-sdk/state/codec/string_codec.go new file mode 100644 index 00000000..0f4f2e21 --- /dev/null +++ b/go-sdk/state/codec/string_codec.go @@ -0,0 +1,23 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +type StringCodec struct{} + +func (c StringCodec) Encode(value string) ([]byte, error) { return []byte(value), nil } + +func (c StringCodec) Decode(data []byte) (string, error) { return string(data), nil } + +func (c StringCodec) EncodedSize() int { return -1 } + +func (c StringCodec) IsOrderedKeyCodec() bool { return true } diff --git a/go-sdk/state/codec/uint32_codec.go b/go-sdk/state/codec/uint32_codec.go new file mode 100644 index 00000000..958b9609 --- /dev/null +++ b/go-sdk/state/codec/uint32_codec.go @@ -0,0 +1,36 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" +) + +type Uint32Codec struct{} + +func (c Uint32Codec) Encode(value uint32) ([]byte, error) { + out := make([]byte, 4) + binary.BigEndian.PutUint32(out, value) + return out, nil +} + +func (c Uint32Codec) Decode(data []byte) (uint32, error) { + if len(data) != 4 { + return 0, fmt.Errorf("invalid uint32 payload length: %d", len(data)) + } + return binary.BigEndian.Uint32(data), nil +} + +func (c Uint32Codec) EncodedSize() int { return 4 } +func (c Uint32Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/codec/uint64_codec.go b/go-sdk/state/codec/uint64_codec.go new file mode 100644 index 00000000..f4001e24 --- /dev/null +++ b/go-sdk/state/codec/uint64_codec.go @@ -0,0 +1,36 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "encoding/binary" + "fmt" +) + +type Uint64Codec struct{} + +func (c Uint64Codec) Encode(value uint64) ([]byte, error) { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, value) + return out, nil +} + +func (c Uint64Codec) Decode(data []byte) (uint64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid uint64 payload length: %d", len(data)) + } + return binary.BigEndian.Uint64(data), nil +} + +func (c Uint64Codec) EncodedSize() int { return 8 } +func (c Uint64Codec) IsOrderedKeyCodec() bool { return false } diff --git a/go-sdk/state/common/common.go b/go-sdk/state/common/common.go new file mode 100644 index 00000000..a31c2a37 --- /dev/null +++ b/go-sdk/state/common/common.go @@ -0,0 +1,67 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "encoding/binary" + "fmt" + "strings" + + "github.com/functionstream/function-stream/go-sdk/api" +) + +type Store = api.Store + +func ValidateStateName(name string) (string, error) { + stateName := strings.TrimSpace(name) + if stateName == "" { + return "", api.NewError(api.ErrStoreInvalidName, "state name must not be empty") + } + return stateName, nil +} + +func EncodeInt64Lex(v int64) []byte { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, uint64(v)^(uint64(1)<<63)) + return out +} + +func DecodeInt64Lex(data []byte) (int64, error) { + if len(data) != 8 { + return 0, fmt.Errorf("invalid int64 lex key length: %d", len(data)) + } + return int64(binary.BigEndian.Uint64(data) ^ (uint64(1) << 63)), nil +} + +func EncodePriorityUserKey(priority int64, seq uint64) []byte { + out := make([]byte, 16) + copy(out[:8], EncodeInt64Lex(priority)) + binary.BigEndian.PutUint64(out[8:], seq) + return out +} + +func DecodePriorityUserKey(data []byte) (int64, error) { + if len(data) != 16 { + return 0, fmt.Errorf("invalid priority key length: %d", len(data)) + } + return DecodeInt64Lex(data[:8]) +} + +func DupBytes(input []byte) []byte { + if input == nil { + return nil + } + out := make([]byte, len(input)) + copy(out, input) + return out +} diff --git a/go-sdk/state/keyed/keyed_aggregating_state.go b/go-sdk/state/keyed/keyed_aggregating_state.go new file mode 100644 index 00000000..4787889c --- /dev/null +++ b/go-sdk/state/keyed/keyed_aggregating_state.go @@ -0,0 +1,136 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyed + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type AggregateFunc[T any, ACC any, R any] interface { + CreateAccumulator() ACC + Add(value T, accumulator ACC) ACC + GetResult(accumulator ACC) R + Merge(a ACC, b ACC) ACC +} + +type KeyedAggregatingStateFactory[T any, ACC any, R any] struct { + inner *keyedStateFactory + groupKey []byte + accCodec codec.Codec[ACC] + aggFunc AggregateFunc[T, ACC, R] +} + +func NewKeyedAggregatingStateFactory[T any, ACC any, R any]( + store common.Store, + keyGroup []byte, + accCodec codec.Codec[ACC], + aggFunc AggregateFunc[T, ACC, R], +) (*KeyedAggregatingStateFactory[T, ACC, R], error) { + + inner, err := newKeyedStateFactory(store, "", "aggregating") + if err != nil { + return nil, err + } + + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed aggregating state factory key_group must not be nil") + } + if accCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed aggregating state factory acc_codec must not be nil") + } + if aggFunc == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed aggregating state factory agg_func must not be nil") + } + + return &KeyedAggregatingStateFactory[T, ACC, R]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + accCodec: accCodec, + aggFunc: aggFunc, + }, nil +} + +func (f *KeyedAggregatingStateFactory[T, ACC, R]) NewAggregatingState(primaryKey []byte, stateName string) (*KeyedAggregatingState[T, ACC, R], error) { + return &KeyedAggregatingState[T, ACC, R]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: []byte(stateName), + }, nil +} + +type KeyedAggregatingState[T any, ACC any, R any] struct { + factory *KeyedAggregatingStateFactory[T, ACC, R] + primaryKey []byte + namespace []byte +} + +func (s *KeyedAggregatingState[T, ACC, R]) buildCK() api.ComplexKey { + return api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + } +} + +func (s *KeyedAggregatingState[T, ACC, R]) Add(value T) error { + ck := s.buildCK() + + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil { + return fmt.Errorf("failed to get accumulator: %w", err) + } + + var acc ACC + if !found { + acc = s.factory.aggFunc.CreateAccumulator() + } else { + var err error + acc, err = s.factory.accCodec.Decode(raw) + if err != nil { + return fmt.Errorf("failed to decode accumulator: %w", err) + } + } + + newAcc := s.factory.aggFunc.Add(value, acc) + + encoded, err := s.factory.accCodec.Encode(newAcc) + if err != nil { + return fmt.Errorf("failed to encode new accumulator: %w", err) + } + return s.factory.inner.store.Put(ck, encoded) +} + +func (s *KeyedAggregatingState[T, ACC, R]) Get() (R, bool, error) { + var zero R + ck := s.buildCK() + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + + acc, err := s.factory.accCodec.Decode(raw) + if err != nil { + return zero, false, err + } + + return s.factory.aggFunc.GetResult(acc), true, nil +} + +func (s *KeyedAggregatingState[T, ACC, R]) Clear() error { + return s.factory.inner.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/keyed/keyed_list_state.go b/go-sdk/state/keyed/keyed_list_state.go new file mode 100644 index 00000000..4c02a0b8 --- /dev/null +++ b/go-sdk/state/keyed/keyed_list_state.go @@ -0,0 +1,269 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyed + +import ( + "encoding/binary" + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type KeyedListStateFactory[V any] struct { + inner *keyedStateFactory + keyGroup []byte + fixedSize int + valueCodec codec.Codec[V] + isFixed bool +} + +func NewKeyedListStateFactory[V any](store common.Store, name string, keyGroup []byte, valueCodec codec.Codec[V]) (*KeyedListStateFactory[V], error) { + inner, err := newKeyedStateFactory(store, name, "list") + if err != nil { + return nil, err + } + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list state factory %q key group must not be nil", inner.name) + } + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list value codec must not be nil") + } + fixedSize, isFixed := codec.FixedEncodedSize[V](valueCodec) + return &KeyedListStateFactory[V]{ + inner: inner, + keyGroup: common.DupBytes(keyGroup), + fixedSize: fixedSize, + valueCodec: valueCodec, + isFixed: isFixed, + }, nil +} + +func NewKeyedListStateFactoryAutoCodec[V any](store common.Store, name string, keyGroup []byte) (*KeyedListStateFactory[V], error) { + valueCodec, err := codec.DefaultCodecFor[V]() + if err != nil { + return nil, err + } + return NewKeyedListStateFactory[V](store, name, keyGroup, valueCodec) +} + +type KeyedListState[V any] struct { + factory *KeyedListStateFactory[V] + complexKey api.ComplexKey + fixedSize int + valueCodec codec.Codec[V] + serialize func(V) ([]byte, error) + serializeBatch func([]V) ([]byte, error) + decode func([]byte) ([]V, error) +} + +func newKeyedListFromFactory[V any](f *KeyedListStateFactory[V], key []byte, namespace []byte) (*KeyedListState[V], error) { + if f == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list factory must not be nil") + } + if key == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list key must not be nil") + } + if namespace == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed list namespace must not be nil") + } + s := &KeyedListState[V]{ + factory: f, + valueCodec: f.valueCodec, + complexKey: api.ComplexKey{ + KeyGroup: f.keyGroup, + Key: key, + Namespace: namespace, + UserKey: []byte{}, + }, + fixedSize: f.fixedSize, + } + if f.isFixed { + s.serialize = s.serializeValueFixed + s.serializeBatch = s.serializeValuesFixedBatch + s.decode = s.deserializeValuesFixed + } else { + s.serialize = s.serializeValueVarLen + s.serializeBatch = s.serializeValuesVarLenBatch + s.decode = s.deserializeValuesVarLen + } + return s, nil +} + +func NewKeyedListFromFactory[V any](f *KeyedListStateFactory[V], key []byte, namespace []byte) (*KeyedListState[V], error) { + return newKeyedListFromFactory[V](f, key, namespace) +} + +func (s *KeyedListState[V]) Add(value V) error { + payload, err := s.serialize(value) + if err != nil { + return err + } + return s.factory.inner.store.Merge(s.complexKey, payload) +} + +func (s *KeyedListState[V]) AddAll(values []V) error { + payload, err := s.serializeBatch(values) + if err != nil { + return err + } + if err := s.factory.inner.store.Merge(s.complexKey, payload); err != nil { + return err + } + return nil +} + +func (s *KeyedListState[V]) Get() ([]V, error) { + raw, found, err := s.factory.inner.store.Get(s.complexKey) + if err != nil { + return nil, err + } + if !found { + return []V{}, nil + } + return s.decode(raw) +} + +// Update replaces the list with the given values (one Put with batch payload). +func (s *KeyedListState[V]) Update(values []V) error { + if len(values) == 0 { + return s.Clear() + } + payload, err := s.serializeBatch(values) + if err != nil { + return err + } + return s.factory.inner.store.Put(s.complexKey, payload) +} + +func (s *KeyedListState[V]) Clear() error { + return s.factory.inner.store.Delete(s.complexKey) +} + +func (s *KeyedListState[V]) serializeValueVarLen(value V) ([]byte, error) { + encoded, err := s.valueCodec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode keyed list value failed: %w", err) + } + out := make([]byte, 4, 4+len(encoded)) + binary.BigEndian.PutUint32(out, uint32(len(encoded))) + out = append(out, encoded...) + return out, nil +} + +func (s *KeyedListState[V]) serializeValuesVarLenBatch(values []V) ([]byte, error) { + total := 0 + encodedValues := make([][]byte, 0, len(values)) + for _, value := range values { + encoded, err := s.valueCodec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode keyed list value failed: %w", err) + } + encodedValues = append(encodedValues, encoded) + total += 4 + len(encoded) + } + out := make([]byte, 0, total) + for _, encoded := range encodedValues { + var lenBuf [4]byte + binary.BigEndian.PutUint32(lenBuf[:], uint32(len(encoded))) + out = append(out, lenBuf[:]...) + out = append(out, encoded...) + } + return out, nil +} + +func (s *KeyedListState[V]) deserializeValuesVarLen(raw []byte) ([]V, error) { + out := make([]V, 0, 16) + idx := 0 + for idx < len(raw) { + if len(raw)-idx < 4 { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted keyed list payload: truncated length") + } + + itemLen := int(binary.BigEndian.Uint32(raw[idx : idx+4])) + idx += 4 + + if itemLen < 0 || len(raw)-idx < itemLen { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted keyed list payload: invalid element length") + } + + itemRaw := raw[idx : idx+itemLen] + idx += itemLen + + value, err := s.valueCodec.Decode(itemRaw) + if err != nil { + return nil, fmt.Errorf("decode keyed list value failed: %w", err) + } + out = append(out, value) + } + return out, nil +} + +func (s *KeyedListState[V]) serializeValueFixed(value V) ([]byte, error) { + if s.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + encoded, err := s.valueCodec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode keyed list value failed: %w", err) + } + if len(encoded) != s.fixedSize { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec encoded unexpected length: got %d, want %d", len(encoded), s.fixedSize) + } + out := make([]byte, 0, s.fixedSize) + out = append(out, encoded...) + return out, nil +} + +func (s *KeyedListState[V]) serializeValuesFixedBatch(values []V) ([]byte, error) { + if s.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + total := s.fixedSize * len(values) + out := make([]byte, 0, total) + for _, value := range values { + encoded, err := s.valueCodec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode keyed list value failed: %w", err) + } + if len(encoded) != s.fixedSize { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec encoded unexpected length: got %d, want %d", len(encoded), s.fixedSize) + } + out = append(out, encoded...) + } + return out, nil +} + +func (s *KeyedListState[V]) deserializeValuesFixed(raw []byte) ([]V, error) { + if s.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + + if len(raw)%s.fixedSize != 0 { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted keyed list payload: fixed-size data length mismatch") + } + + count := len(raw) / s.fixedSize + out := make([]V, 0, count) + + for idx := 0; idx < len(raw); idx += s.fixedSize { + itemRaw := raw[idx : idx+s.fixedSize] + value, err := s.valueCodec.Decode(itemRaw) + if err != nil { + return nil, fmt.Errorf("decode keyed list value failed: %w", err) + } + out = append(out, value) + } + return out, nil +} diff --git a/go-sdk/state/keyed/keyed_map_state.go b/go-sdk/state/keyed/keyed_map_state.go new file mode 100644 index 00000000..8fc7e09b --- /dev/null +++ b/go-sdk/state/keyed/keyed_map_state.go @@ -0,0 +1,168 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyed + +import ( + "fmt" + "iter" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type KeyedMapStateFactory[MK any, MV any] struct { + inner *keyedStateFactory + groupKey []byte + mapKeyCodec codec.Codec[MK] + mapValueCodec codec.Codec[MV] +} + +func NewKeyedMapStateFactory[MK any, MV any]( + store common.Store, + keyGroup []byte, + mapKeyCodec codec.Codec[MK], + mapValueCodec codec.Codec[MV], +) (*KeyedMapStateFactory[MK, MV], error) { + + inner, err := newKeyedStateFactory(store, "", "map") + if err != nil { + return nil, err + } + + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed map state factory key_group must not be nil") + } + if mapKeyCodec == nil || mapValueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed map state factory map_key_codec and map_value_codec must not be nil") + } + + if !mapKeyCodec.IsOrderedKeyCodec() { + return nil, api.NewError(api.ErrStoreInternal, "map key codec must be ordered") + } + + return &KeyedMapStateFactory[MK, MV]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + mapKeyCodec: mapKeyCodec, + mapValueCodec: mapValueCodec, + }, nil +} + +type KeyedMapState[MK any, MV any] struct { + factory *KeyedMapStateFactory[MK, MV] + primaryKey []byte + namespace []byte +} + +func (f *KeyedMapStateFactory[MK, MV]) NewKeyedMap(primaryKey []byte, mapName string) (*KeyedMapState[MK, MV], error) { + if primaryKey == nil || mapName == "" { + return nil, api.NewError(api.ErrStoreInternal, "primary key and map name are required") + } + return &KeyedMapState[MK, MV]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: []byte(mapName), + }, nil +} + +func (s *KeyedMapState[MK, MV]) buildCK(mapKey MK) (api.ComplexKey, error) { + encodedMapKey, err := s.factory.mapKeyCodec.Encode(mapKey) + if err != nil { + return api.ComplexKey{}, fmt.Errorf("encode map userKey failed: %w", err) + } + return api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: encodedMapKey, + }, nil +} + +func (s *KeyedMapState[MK, MV]) Put(mapKey MK, value MV) error { + ck, err := s.buildCK(mapKey) + if err != nil { + return err + } + encodedValue, err := s.factory.mapValueCodec.Encode(value) + if err != nil { + return err + } + return s.factory.inner.store.Put(ck, encodedValue) +} + +func (s *KeyedMapState[MK, MV]) Get(mapKey MK) (MV, bool, error) { + var zero MV + ck, err := s.buildCK(mapKey) + if err != nil { + return zero, false, err + } + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + decoded, err := s.factory.mapValueCodec.Decode(raw) + if err != nil { + return zero, false, err + } + return decoded, true, nil +} + +func (s *KeyedMapState[MK, MV]) Delete(mapKey MK) error { + ck, err := s.buildCK(mapKey) + if err != nil { + return err + } + return s.factory.inner.store.Delete(ck) +} + +func (s *KeyedMapState[MK, MV]) Clear() error { + return s.factory.inner.store.DeletePrefix(api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + }) +} + +func (s *KeyedMapState[MK, MV]) All() iter.Seq2[MK, MV] { + return func(yield func(MK, MV) bool) { + iter, err := s.factory.inner.store.ScanComplex( + s.factory.groupKey, + s.primaryKey, + s.namespace, + ) + if err != nil { + return + } + defer iter.Close() + + for { + has, err := iter.HasNext() + if err != nil || !has { + return + } + keyRaw, valRaw, ok, err := iter.Next() + if err != nil || !ok { + return + } + + k, _ := s.factory.mapKeyCodec.Decode(keyRaw) + v, _ := s.factory.mapValueCodec.Decode(valRaw) + + if !yield(k, v) { + return + } + } + } +} diff --git a/go-sdk/state/keyed/keyed_priority_queue_state.go b/go-sdk/state/keyed/keyed_priority_queue_state.go new file mode 100644 index 00000000..0e524664 --- /dev/null +++ b/go-sdk/state/keyed/keyed_priority_queue_state.go @@ -0,0 +1,177 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyed + +import ( + "fmt" + "iter" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type KeyedPriorityQueueStateFactory[V any] struct { + inner *keyedStateFactory + groupKey []byte + valueCodec codec.Codec[V] +} + +func NewKeyedPriorityQueueStateFactory[V any]( + store common.Store, + keyGroup []byte, + valueCodec codec.Codec[V], +) (*KeyedPriorityQueueStateFactory[V], error) { + + inner, err := newKeyedStateFactory(store, "", "pq") + if err != nil { + return nil, err + } + + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed priority queue state factory key_group must not be nil") + } + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed priority queue state factory value codec must not be nil") + } + + if !valueCodec.IsOrderedKeyCodec() { + return nil, api.NewError(api.ErrStoreInternal, "priority queue value codec must be ordered") + } + + return &KeyedPriorityQueueStateFactory[V]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + valueCodec: valueCodec, + }, nil +} + +type KeyedPriorityQueueState[V any] struct { + factory *KeyedPriorityQueueStateFactory[V] + primaryKey []byte + namespace []byte +} + +func (f *KeyedPriorityQueueStateFactory[V]) NewKeyedPriorityQueue(primaryKey []byte, namespace []byte) (*KeyedPriorityQueueState[V], error) { + if primaryKey == nil || namespace == nil { + return nil, api.NewError(api.ErrStoreInternal, "primary key and queue name are required") + } + return &KeyedPriorityQueueState[V]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: common.DupBytes(namespace), + }, nil +} + +func (s *KeyedPriorityQueueState[V]) Add(value V) error { + userKey, err := s.factory.valueCodec.Encode(value) + if err != nil { + return fmt.Errorf("encode pq element failed: %w", err) + } + + ck := api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: userKey, + } + + return s.factory.inner.store.Put(ck, []byte{}) +} + +func (s *KeyedPriorityQueueState[V]) Peek() (V, bool, error) { + var zero V + + iter, err := s.factory.inner.store.ScanComplex( + s.factory.groupKey, + s.primaryKey, + s.namespace, + ) + if err != nil { + return zero, false, err + } + defer iter.Close() + + has, err := iter.HasNext() + if err != nil || !has { + return zero, false, err + } + + userKey, _, ok, err := iter.Next() + if err != nil || !ok { + return zero, false, err + } + + val, err := s.factory.valueCodec.Decode(userKey) + if err != nil { + return zero, false, err + } + return val, true, nil +} + +func (s *KeyedPriorityQueueState[V]) Poll() (V, bool, error) { + val, found, err := s.Peek() + if err != nil || !found { + return val, found, err + } + + userKey, _ := s.factory.valueCodec.Encode(val) + ck := api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: userKey, + } + + err = s.factory.inner.store.Delete(ck) + return val, true, err +} + +func (s *KeyedPriorityQueueState[V]) Clear() error { + return s.factory.inner.store.DeletePrefix(api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + }) +} + +func (s *KeyedPriorityQueueState[V]) All() iter.Seq[V] { + return func(yield func(V) bool) { + iter, err := s.factory.inner.store.ScanComplex( + s.factory.groupKey, + s.primaryKey, + s.namespace, + ) + if err != nil { + return + } + defer iter.Close() + + for { + has, err := iter.HasNext() + if err != nil || !has { + return + } + userKey, _, ok, err := iter.Next() + if err != nil || !ok { + return + } + + v, _ := s.factory.valueCodec.Decode(userKey) + if !yield(v) { + return + } + } + } +} diff --git a/go-sdk/state/keyed/keyed_reducing_state.go b/go-sdk/state/keyed/keyed_reducing_state.go new file mode 100644 index 00000000..e608ee3d --- /dev/null +++ b/go-sdk/state/keyed/keyed_reducing_state.go @@ -0,0 +1,131 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyed + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type ReduceFunc[V any] func(value1 V, value2 V) (V, error) + +type KeyedReducingStateFactory[V any] struct { + inner *keyedStateFactory + groupKey []byte + valueCodec codec.Codec[V] + reduceFunc ReduceFunc[V] +} + +func NewKeyedReducingStateFactory[V any]( + store common.Store, + keyGroup []byte, + valueCodec codec.Codec[V], + reduceFunc ReduceFunc[V], +) (*KeyedReducingStateFactory[V], error) { + + inner, err := newKeyedStateFactory(store, "", "reducing") + if err != nil { + return nil, err + } + + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed reducing state factory key_group must not be nil") + } + if valueCodec == nil || reduceFunc == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed reducing state factory value_codec and reduce_func must not be nil") + } + + return &KeyedReducingStateFactory[V]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + valueCodec: valueCodec, + reduceFunc: reduceFunc, + }, nil +} + +func (f *KeyedReducingStateFactory[V]) NewReducingState(primaryKey []byte, namespace []byte) (*KeyedReducingState[V], error) { + if primaryKey == nil || namespace == nil { + return nil, api.NewError(api.ErrStoreInternal, "primary key and state name are required") + } + return &KeyedReducingState[V]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: common.DupBytes(namespace), + }, nil +} + +type KeyedReducingState[V any] struct { + factory *KeyedReducingStateFactory[V] + primaryKey []byte + namespace []byte +} + +func (s *KeyedReducingState[V]) buildCK() api.ComplexKey { + return api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + } +} + +func (s *KeyedReducingState[V]) Add(value V) error { + ck := s.buildCK() + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil { + return fmt.Errorf("failed to get old value for reducing state: %w", err) + } + + var result V + if !found { + result = value + } else { + oldValue, err := s.factory.valueCodec.Decode(raw) + if err != nil { + return fmt.Errorf("failed to decode old value: %w", err) + } + + result, err = s.factory.reduceFunc(oldValue, value) + if err != nil { + return fmt.Errorf("error in user reduce function: %w", err) + } + } + + encoded, err := s.factory.valueCodec.Encode(result) + if err != nil { + return fmt.Errorf("failed to encode reduced value: %w", err) + } + + return s.factory.inner.store.Put(ck, encoded) +} + +func (s *KeyedReducingState[V]) Get() (V, bool, error) { + var zero V + ck := s.buildCK() + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + val, err := s.factory.valueCodec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("failed to decode value: %w", err) + } + return val, true, nil +} + +func (s *KeyedReducingState[V]) Clear() error { + return s.factory.inner.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/keyed/keyed_state_factory.go b/go-sdk/state/keyed/keyed_state_factory.go new file mode 100644 index 00000000..acc601ec --- /dev/null +++ b/go-sdk/state/keyed/keyed_state_factory.go @@ -0,0 +1,49 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyed + +import ( + "encoding/base64" + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type keyedStateFactory struct { + store common.Store + name string +} + +func newKeyedStateFactory(store common.Store, name string, kind string) (*keyedStateFactory, error) { + stateName, err := common.ValidateStateName(name) + if err != nil { + return nil, err + } + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed %s state factory %q store must not be nil", kind, stateName) + } + return &keyedStateFactory{store: store, name: stateName}, nil +} + +func keyedSubStateName[K any](base string, keyCodec codec.Codec[K], key K, kind string) (string, error) { + if keyCodec == nil { + return "", api.NewError(api.ErrStoreInternal, "key codec must not be nil") + } + encoded, err := keyCodec.Encode(key) + if err != nil { + return "", fmt.Errorf("encode keyed state key failed: %w", err) + } + return fmt.Sprintf("%s/%s/%s", base, kind, base64.RawURLEncoding.EncodeToString(encoded)), nil +} diff --git a/go-sdk/state/keyed/keyed_value_state.go b/go-sdk/state/keyed/keyed_value_state.go new file mode 100644 index 00000000..c181e992 --- /dev/null +++ b/go-sdk/state/keyed/keyed_value_state.go @@ -0,0 +1,105 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyed + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type KeyedValueStateFactory[V any] struct { + inner *keyedStateFactory + groupKey []byte + valueCodec codec.Codec[V] +} + +func NewKeyedValueStateFactory[V any]( + store common.Store, + keyGroup []byte, + valueCodec codec.Codec[V], +) (*KeyedValueStateFactory[V], error) { + + inner, err := newKeyedStateFactory(store, "", "value") + if err != nil { + return nil, err + } + + if keyGroup == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed value state factory key_group must not be nil") + } + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "keyed value state factory value codec must not be nil") + } + + return &KeyedValueStateFactory[V]{ + inner: inner, + groupKey: common.DupBytes(keyGroup), + valueCodec: valueCodec, + }, nil +} + +type KeyedValueState[V any] struct { + factory *KeyedValueStateFactory[V] + primaryKey []byte + namespace []byte +} + +func (f *KeyedValueStateFactory[V]) NewKeyedValue(primaryKey []byte, stateName string) (*KeyedValueState[V], error) { + if primaryKey == nil || stateName == "" { + return nil, api.NewError(api.ErrStoreInternal, "primary key and state name are required") + } + return &KeyedValueState[V]{ + factory: f, + primaryKey: common.DupBytes(primaryKey), + namespace: []byte(stateName), + }, nil +} + +func (s *KeyedValueState[V]) buildCK() api.ComplexKey { + return api.ComplexKey{ + KeyGroup: s.factory.groupKey, + Key: s.primaryKey, + Namespace: s.namespace, + UserKey: []byte{}, + } +} + +func (s *KeyedValueState[V]) Update(value V) error { + ck := s.buildCK() + encoded, err := s.factory.valueCodec.Encode(value) + if err != nil { + return fmt.Errorf("encode value state failed: %w", err) + } + return s.factory.inner.store.Put(ck, encoded) +} + +func (s *KeyedValueState[V]) Value() (V, bool, error) { + var zero V + ck := s.buildCK() + raw, found, err := s.factory.inner.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + decoded, err := s.factory.valueCodec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("decode value state failed: %w", err) + } + return decoded, true, nil +} + +func (s *KeyedValueState[V]) Clear() error { + return s.factory.inner.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/structures/aggregating.go b/go-sdk/state/structures/aggregating.go new file mode 100644 index 00000000..ab6f3270 --- /dev/null +++ b/go-sdk/state/structures/aggregating.go @@ -0,0 +1,115 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package structures + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type AggregateFunc[T any, ACC any, R any] interface { + CreateAccumulator() ACC + Add(value T, accumulator ACC) ACC + GetResult(accumulator ACC) R + Merge(a ACC, b ACC) ACC +} + +type AggregatingState[T any, ACC any, R any] struct { + store common.Store + complexKey api.ComplexKey + accCodec codec.Codec[ACC] + aggFunc AggregateFunc[T, ACC, R] +} + +func NewAggregatingState[T any, ACC any, R any]( + store common.Store, + accCodec codec.Codec[ACC], + aggFunc AggregateFunc[T, ACC, R], +) (*AggregatingState[T, ACC, R], error) { + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "aggregating state store must not be nil") + } + if accCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "aggregating state acc codec must not be nil") + } + if aggFunc == nil { + return nil, api.NewError(api.ErrStoreInternal, "aggregating state agg func must not be nil") + } + ck := api.ComplexKey{ + KeyGroup: []byte{}, + Key: []byte{}, + Namespace: []byte{}, + UserKey: []byte{}, + } + return &AggregatingState[T, ACC, R]{ + store: store, + complexKey: ck, + accCodec: accCodec, + aggFunc: aggFunc, + }, nil +} + +func (s *AggregatingState[T, ACC, R]) buildCK() api.ComplexKey { + return s.complexKey +} + +func (s *AggregatingState[T, ACC, R]) Add(value T) error { + ck := s.buildCK() + + raw, found, err := s.store.Get(ck) + if err != nil { + return fmt.Errorf("failed to get accumulator: %w", err) + } + + var acc ACC + if !found { + acc = s.aggFunc.CreateAccumulator() + } else { + var err error + acc, err = s.accCodec.Decode(raw) + if err != nil { + return fmt.Errorf("failed to decode accumulator: %w", err) + } + } + + newAcc := s.aggFunc.Add(value, acc) + + encoded, err := s.accCodec.Encode(newAcc) + if err != nil { + return fmt.Errorf("failed to encode new accumulator: %w", err) + } + return s.store.Put(ck, encoded) +} + +func (s *AggregatingState[T, ACC, R]) Get() (R, bool, error) { + var zero R + ck := s.buildCK() + raw, found, err := s.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + + acc, err := s.accCodec.Decode(raw) + if err != nil { + return zero, false, err + } + + return s.aggFunc.GetResult(acc), true, nil +} + +func (s *AggregatingState[T, ACC, R]) Clear() error { + return s.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/structures/list.go b/go-sdk/state/structures/list.go new file mode 100644 index 00000000..83d2c845 --- /dev/null +++ b/go-sdk/state/structures/list.go @@ -0,0 +1,217 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package structures + +import ( + "encoding/binary" + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type ListState[T any] struct { + store common.Store + complexKey api.ComplexKey + codec codec.Codec[T] + fixedSize int + serialize func(T) ([]byte, error) + decode func([]byte) ([]T, error) + serializeBatch func([]T) ([]byte, error) +} + +func NewListState[T any](store common.Store, itemCodec codec.Codec[T]) (*ListState[T], error) { + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "list state store must not be nil") + } + if itemCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "list state codec must not be nil") + } + fixedSize, isFixed := codec.FixedEncodedSize[T](itemCodec) + l := &ListState[T]{ + store: store, + complexKey: api.ComplexKey{ + KeyGroup: []byte{}, + Key: []byte{}, + Namespace: []byte{}, + UserKey: []byte{}, + }, + codec: itemCodec, + fixedSize: fixedSize, + } + if isFixed { + l.serialize = l.serializeValueFixed + l.serializeBatch = l.serializeValuesFixedBatch + l.decode = l.deserializeValuesFixed + } else { + l.serialize = l.serializeValueVarLen + l.serializeBatch = l.serializeValuesVarLenBatch + l.decode = l.deserializeValuesVarLen + } + return l, nil +} + +func (l *ListState[T]) Add(value T) error { + payload, err := l.serialize(value) + if err != nil { + return err + } + return l.store.Merge(l.complexKey, payload) +} + +func (l *ListState[T]) AddAll(values []T) error { + payload, err := l.serializeBatch(values) + if err != nil { + return err + } + if err := l.store.Merge(l.complexKey, payload); err != nil { + return err + } + return nil +} + +func (l *ListState[T]) Get() ([]T, error) { + raw, found, err := l.store.Get(l.complexKey) + if err != nil { + return nil, err + } + if !found { + return []T{}, nil + } + return l.decode(raw) +} + +func (l *ListState[T]) Update(values []T) error { + if len(values) == 0 { + return l.Clear() + } + payload, err := l.serializeBatch(values) + if err != nil { + return err + } + return l.store.Put(l.complexKey, payload) +} + +func (l *ListState[T]) Clear() error { + return l.store.Delete(l.complexKey) +} + +func (l *ListState[T]) serializeValueVarLen(value T) ([]byte, error) { + encoded, err := l.codec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode list value failed: %w", err) + } + out := make([]byte, 4, 4+len(encoded)) + binary.BigEndian.PutUint32(out, uint32(len(encoded))) + out = append(out, encoded...) + return out, nil +} + +func (l *ListState[T]) serializeValuesVarLenBatch(values []T) ([]byte, error) { + total := 0 + encodedValues := make([][]byte, 0, len(values)) + for _, value := range values { + encoded, err := l.codec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode list value failed: %w", err) + } + encodedValues = append(encodedValues, encoded) + total += 4 + len(encoded) + } + out := make([]byte, 0, total) + for _, encoded := range encodedValues { + var lenBuf [4]byte + binary.BigEndian.PutUint32(lenBuf[:], uint32(len(encoded))) + out = append(out, lenBuf[:]...) + out = append(out, encoded...) + } + return out, nil +} + +func (l *ListState[T]) deserializeValuesVarLen(raw []byte) ([]T, error) { + out := make([]T, 0, 16) + idx := 0 + for idx < len(raw) { + if len(raw)-idx < 4 { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted list payload: truncated length") + } + itemLen := int(binary.BigEndian.Uint32(raw[idx : idx+4])) + idx += 4 + if itemLen < 0 || len(raw)-idx < itemLen { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted list payload: invalid element length") + } + itemRaw := raw[idx : idx+itemLen] + idx += itemLen + value, err := l.codec.Decode(itemRaw) + if err != nil { + return nil, fmt.Errorf("decode list value failed: %w", err) + } + out = append(out, value) + } + return out, nil +} + +func (l *ListState[T]) serializeValueFixed(value T) ([]byte, error) { + if l.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + encoded, err := l.codec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode list value failed: %w", err) + } + if len(encoded) != l.fixedSize { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec encoded unexpected length: got %d, want %d", len(encoded), l.fixedSize) + } + out := make([]byte, 0, l.fixedSize) + out = append(out, encoded...) + return out, nil +} + +func (l *ListState[T]) serializeValuesFixedBatch(values []T) ([]byte, error) { + if l.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + total := l.fixedSize * len(values) + out := make([]byte, 0, total) + for _, value := range values { + encoded, err := l.codec.Encode(value) + if err != nil { + return nil, fmt.Errorf("encode list value failed: %w", err) + } + if len(encoded) != l.fixedSize { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec encoded unexpected length: got %d, want %d", len(encoded), l.fixedSize) + } + out = append(out, encoded...) + } + return out, nil +} + +func (l *ListState[T]) deserializeValuesFixed(raw []byte) ([]T, error) { + if l.fixedSize <= 0 { + return nil, api.NewError(api.ErrResultUnexpected, "fixed-size codec must report positive size") + } + if len(raw)%l.fixedSize != 0 { + return nil, api.NewError(api.ErrResultUnexpected, "corrupted list payload: fixed-size data length mismatch") + } + out := make([]T, 0, len(raw)/l.fixedSize) + for idx := 0; idx < len(raw); idx += l.fixedSize { + itemRaw := raw[idx : idx+l.fixedSize] + value, err := l.codec.Decode(itemRaw) + if err != nil { + return nil, fmt.Errorf("decode list value failed: %w", err) + } + out = append(out, value) + } + return out, nil +} diff --git a/go-sdk/state/structures/map.go b/go-sdk/state/structures/map.go new file mode 100644 index 00000000..a4b89352 --- /dev/null +++ b/go-sdk/state/structures/map.go @@ -0,0 +1,261 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package structures + +import ( + "fmt" + "iter" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type MapEntry[K any, V any] struct { + Key K + Value V +} + +type MapState[K any, V any] struct { + store common.Store + keyGroup []byte + key []byte + namespace []byte + keyCodec codec.Codec[K] + valueCodec codec.Codec[V] +} + +func NewMapState[K any, V any](store common.Store, keyCodec codec.Codec[K], valueCodec codec.Codec[V]) (*MapState[K, V], error) { + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "map state store must not be nil") + } + if keyCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "map state key codec must not be nil") + } + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "map state value codec must not be nil") + } + if !keyCodec.IsOrderedKeyCodec() { + return nil, api.NewError(api.ErrStoreInternal, "map state key codec must be ordered (IsOrderedKeyCodec)") + } + return &MapState[K, V]{store: store, keyGroup: []byte{}, key: []byte{}, namespace: []byte{}, keyCodec: keyCodec, valueCodec: valueCodec}, nil +} + +func NewMapStateAutoKeyCodec[K any, V any](store common.Store, valueCodec codec.Codec[V]) (*MapState[K, V], error) { + autoKeyCodec, err := inferOrderedKeyCodec[K]() + if err != nil { + return nil, err + } + return NewMapState[K, V](store, autoKeyCodec, valueCodec) +} + +func (m *MapState[K, V]) Put(key K, value V) error { + encodedKey, err := m.keyCodec.Encode(key) + if err != nil { + return fmt.Errorf("encode map key failed: %w", err) + } + encodedValue, err := m.valueCodec.Encode(value) + if err != nil { + return fmt.Errorf("encode map value failed: %w", err) + } + return m.store.Put(m.ck(encodedKey), encodedValue) +} + +func (m *MapState[K, V]) Get(key K) (V, bool, error) { + var zero V + encodedKey, err := m.keyCodec.Encode(key) + if err != nil { + return zero, false, fmt.Errorf("encode map key failed: %w", err) + } + raw, found, err := m.store.Get(m.ck(encodedKey)) + if err != nil { + return zero, false, err + } + if !found { + return zero, false, nil + } + decoded, err := m.valueCodec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("decode map value failed: %w", err) + } + return decoded, true, nil +} + +func (m *MapState[K, V]) Delete(key K) error { + encodedKey, err := m.keyCodec.Encode(key) + if err != nil { + return fmt.Errorf("encode map key failed: %w", err) + } + return m.store.Delete(m.ck(encodedKey)) +} + +func (m *MapState[K, V]) Len() (uint64, error) { + iter, err := m.store.ScanComplex(m.keyGroup, m.key, m.namespace) + if err != nil { + return 0, err + } + defer iter.Close() + var count uint64 + for { + has, err := iter.HasNext() + if err != nil { + return 0, err + } + if !has { + return count, nil + } + _, _, ok, err := iter.Next() + if err != nil { + return 0, err + } + if !ok { + return count, nil + } + count++ + } +} + +func (m *MapState[K, V]) Entries() ([]MapEntry[K, V], error) { + iter, err := m.store.ScanComplex(m.keyGroup, m.key, m.namespace) + if err != nil { + return nil, err + } + defer iter.Close() + out := make([]MapEntry[K, V], 0, 16) + for { + has, err := iter.HasNext() + if err != nil { + return nil, err + } + if !has { + return out, nil + } + keyBytes, valueBytes, ok, err := iter.Next() + if err != nil { + return nil, err + } + if !ok { + return out, nil + } + decodedKey, err := m.keyCodec.Decode(keyBytes) + if err != nil { + return nil, fmt.Errorf("decode map key failed: %w", err) + } + decodedValue, err := m.valueCodec.Decode(valueBytes) + if err != nil { + return nil, fmt.Errorf("decode map value failed: %w", err) + } + out = append(out, MapEntry[K, V]{Key: decodedKey, Value: decodedValue}) + } +} + +func (m *MapState[K, V]) Range(startInclusive K, endExclusive K) ([]MapEntry[K, V], error) { + startBytes, err := m.keyCodec.Encode(startInclusive) + if err != nil { + return nil, fmt.Errorf("encode map start key failed: %w", err) + } + endBytes, err := m.keyCodec.Encode(endExclusive) + if err != nil { + return nil, fmt.Errorf("encode map end key failed: %w", err) + } + userKeys, err := m.store.ListComplex(m.keyGroup, m.key, m.namespace, startBytes, endBytes) + if err != nil { + return nil, err + } + out := make([]MapEntry[K, V], 0, len(userKeys)) + for _, userKey := range userKeys { + raw, found, err := m.store.Get(m.ck(userKey)) + if err != nil { + return nil, err + } + if !found { + return nil, api.NewError(api.ErrResultUnexpected, "map range key disappeared during scan") + } + decodedKey, err := m.keyCodec.Decode(userKey) + if err != nil { + return nil, fmt.Errorf("decode map key failed: %w", err) + } + decodedValue, err := m.valueCodec.Decode(raw) + if err != nil { + return nil, fmt.Errorf("decode map value failed: %w", err) + } + out = append(out, MapEntry[K, V]{Key: decodedKey, Value: decodedValue}) + } + return out, nil +} + +func (m *MapState[K, V]) Clear() error { + return m.store.DeletePrefix(api.ComplexKey{KeyGroup: m.keyGroup, Key: m.key, Namespace: m.namespace, UserKey: nil}) +} + +func (m *MapState[K, V]) All() iter.Seq2[K, V] { + return func(yield func(K, V) bool) { + it, err := m.store.ScanComplex(m.keyGroup, m.key, m.namespace) + if err != nil { + return + } + defer it.Close() + + for { + has, err := it.HasNext() + if err != nil || !has { + return + } + keyRaw, valRaw, ok, err := it.Next() + if err != nil || !ok { + return + } + + k, _ := m.keyCodec.Decode(keyRaw) + v, _ := m.valueCodec.Decode(valRaw) + + if !yield(k, v) { + return + } + } + } +} + +func (m *MapState[K, V]) ck(userKey []byte) api.ComplexKey { + return api.ComplexKey{KeyGroup: m.keyGroup, Key: m.key, Namespace: m.namespace, UserKey: userKey} +} + +func inferOrderedKeyCodec[K any]() (codec.Codec[K], error) { + var zero K + switch any(zero).(type) { + case string: + return any(codec.StringCodec{}).(codec.Codec[K]), nil + case []byte: + return any(codec.BytesCodec{}).(codec.Codec[K]), nil + case bool: + return any(codec.BoolCodec{}).(codec.Codec[K]), nil + case int: + return any(codec.OrderedIntCodec{}).(codec.Codec[K]), nil + case uint: + return any(codec.OrderedUintCodec{}).(codec.Codec[K]), nil + case int32: + return any(codec.OrderedInt32Codec{}).(codec.Codec[K]), nil + case uint32: + return any(codec.OrderedUint32Codec{}).(codec.Codec[K]), nil + case int64: + return any(codec.OrderedInt64Codec{}).(codec.Codec[K]), nil + case uint64: + return any(codec.OrderedUint64Codec{}).(codec.Codec[K]), nil + case float32: + return any(codec.OrderedFloat32Codec{}).(codec.Codec[K]), nil + case float64: + return any(codec.OrderedFloat64Codec{}).(codec.Codec[K]), nil + default: + return nil, api.NewError(api.ErrStoreInternal, "unsupported map key type for auto codec") + } +} diff --git a/go-sdk/state/structures/priority_queue.go b/go-sdk/state/structures/priority_queue.go new file mode 100644 index 00000000..16518893 --- /dev/null +++ b/go-sdk/state/structures/priority_queue.go @@ -0,0 +1,136 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package structures + +import ( + "fmt" + "iter" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +// PriorityQueueState holds a priority queue. itemCodec must be ordered (IsOrderedKeyCodec() true). +type PriorityQueueState[T any] struct { + store common.Store + keyGroup []byte + key []byte + namespace []byte + valueCodec codec.Codec[T] +} + +// NewPriorityQueueState creates a priority queue state. itemCodec must support ordered key encoding. +func NewPriorityQueueState[T any](store common.Store, itemCodec codec.Codec[T]) (*PriorityQueueState[T], error) { + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "priority queue state store must not be nil") + } + if itemCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "priority queue state codec must not be nil") + } + if !itemCodec.IsOrderedKeyCodec() { + return nil, api.NewError(api.ErrStoreInternal, "priority queue codec must support ordered key encoding") + } + return &PriorityQueueState[T]{ + store: store, + keyGroup: []byte{}, + key: []byte{}, + namespace: []byte{}, + valueCodec: itemCodec, + }, nil +} + +func (q *PriorityQueueState[T]) ck(userKey []byte) api.ComplexKey { + return api.ComplexKey{KeyGroup: q.keyGroup, Key: q.key, Namespace: q.namespace, UserKey: userKey} +} + +func (q *PriorityQueueState[T]) Add(value T) error { + userKey, err := q.valueCodec.Encode(value) + if err != nil { + return fmt.Errorf("encode pq element failed: %w", err) + } + return q.store.Put(q.ck(userKey), []byte{}) +} + +func (q *PriorityQueueState[T]) Peek() (T, bool, error) { + var zero T + it, err := q.store.ScanComplex(q.keyGroup, q.key, q.namespace) + if err != nil { + return zero, false, err + } + defer it.Close() + + has, err := it.HasNext() + if err != nil || !has { + return zero, false, err + } + + userKey, _, ok, err := it.Next() + if err != nil || !ok { + return zero, false, err + } + + val, err := q.valueCodec.Decode(userKey) + if err != nil { + return zero, false, err + } + return val, true, nil +} + +func (q *PriorityQueueState[T]) Poll() (T, bool, error) { + val, found, err := q.Peek() + if err != nil || !found { + return val, found, err + } + + userKey, _ := q.valueCodec.Encode(val) + if err = q.store.Delete(q.ck(userKey)); err != nil { + return val, true, err + } + return val, true, nil +} + +func (q *PriorityQueueState[T]) Clear() error { + return q.store.DeletePrefix(api.ComplexKey{ + KeyGroup: q.keyGroup, + Key: q.key, + Namespace: q.namespace, + UserKey: nil, + }) +} + +func (q *PriorityQueueState[T]) All() iter.Seq[T] { + return func(yield func(T) bool) { + it, err := q.store.ScanComplex(q.keyGroup, q.key, q.namespace) + if err != nil { + return + } + defer it.Close() + + for { + has, err := it.HasNext() + if err != nil || !has { + return + } + userKey, _, ok, err := it.Next() + if err != nil || !ok { + return + } + + v, _ := q.valueCodec.Decode(userKey) + if !yield(v) { + return + } + } + } +} diff --git a/go-sdk/state/structures/reducing.go b/go-sdk/state/structures/reducing.go new file mode 100644 index 00000000..19ca5b9c --- /dev/null +++ b/go-sdk/state/structures/reducing.go @@ -0,0 +1,107 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package structures + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type ReduceFunc[V any] func(value1 V, value2 V) (V, error) + +type ReducingState[V any] struct { + store common.Store + complexKey api.ComplexKey + valueCodec codec.Codec[V] + reduceFunc ReduceFunc[V] +} + +func NewReducingState[V any]( + store common.Store, + valueCodec codec.Codec[V], + reduceFunc ReduceFunc[V], +) (*ReducingState[V], error) { + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "reducing state store must not be nil") + } + if valueCodec == nil || reduceFunc == nil { + return nil, api.NewError(api.ErrStoreInternal, "reducing state value codec and reduce function are required") + } + ck := api.ComplexKey{ + KeyGroup: []byte{}, + Key: []byte{}, + Namespace: []byte{}, + UserKey: []byte{}, + } + return &ReducingState[V]{ + store: store, + complexKey: ck, + valueCodec: valueCodec, + reduceFunc: reduceFunc, + }, nil +} + +func (s *ReducingState[V]) buildCK() api.ComplexKey { + return s.complexKey +} + +func (s *ReducingState[V]) Add(value V) error { + ck := s.buildCK() + raw, found, err := s.store.Get(ck) + if err != nil { + return fmt.Errorf("failed to get old value for reducing state: %w", err) + } + + var result V + if !found { + result = value + } else { + oldValue, err := s.valueCodec.Decode(raw) + if err != nil { + return fmt.Errorf("failed to decode old value: %w", err) + } + + result, err = s.reduceFunc(oldValue, value) + if err != nil { + return fmt.Errorf("error in user reduce function: %w", err) + } + } + + encoded, err := s.valueCodec.Encode(result) + if err != nil { + return fmt.Errorf("failed to encode reduced value: %w", err) + } + + return s.store.Put(ck, encoded) +} + +func (s *ReducingState[V]) Get() (V, bool, error) { + var zero V + ck := s.buildCK() + raw, found, err := s.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + val, err := s.valueCodec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("failed to decode value: %w", err) + } + return val, true, nil +} + +func (s *ReducingState[V]) Clear() error { + return s.store.Delete(s.buildCK()) +} diff --git a/go-sdk/state/structures/value.go b/go-sdk/state/structures/value.go new file mode 100644 index 00000000..b77ab4f9 --- /dev/null +++ b/go-sdk/state/structures/value.go @@ -0,0 +1,73 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package structures + +import ( + "fmt" + + "github.com/functionstream/function-stream/go-sdk/api" + "github.com/functionstream/function-stream/go-sdk/state/codec" + "github.com/functionstream/function-stream/go-sdk/state/common" +) + +type ValueState[T any] struct { + store common.Store + complexKey api.ComplexKey + codec codec.Codec[T] +} + +func NewValueState[T any](store common.Store, valueCodec codec.Codec[T]) (*ValueState[T], error) { + if store == nil { + return nil, api.NewError(api.ErrStoreInternal, "value state store must not be nil") + } + if valueCodec == nil { + return nil, api.NewError(api.ErrStoreInternal, "value state codec must not be nil") + } + ck := api.ComplexKey{ + KeyGroup: []byte{}, + Key: []byte{}, + Namespace: []byte{}, + UserKey: []byte{}, + } + return &ValueState[T]{store: store, complexKey: ck, codec: valueCodec}, nil +} + +func (v *ValueState[T]) buildCK() api.ComplexKey { + return v.complexKey +} + +func (v *ValueState[T]) Update(value T) error { + encoded, err := v.codec.Encode(value) + if err != nil { + return fmt.Errorf("encode value state failed: %w", err) + } + return v.store.Put(v.buildCK(), encoded) +} + +func (v *ValueState[T]) Value() (T, bool, error) { + var zero T + ck := v.buildCK() + raw, found, err := v.store.Get(ck) + if err != nil || !found { + return zero, found, err + } + decoded, err := v.codec.Decode(raw) + if err != nil { + return zero, false, fmt.Errorf("decode value state failed: %w", err) + } + return decoded, true, nil +} + +func (v *ValueState[T]) Clear() error { + return v.store.Delete(v.buildCK()) +} diff --git a/python/functionstream-api/pyproject.toml b/python/functionstream-api/pyproject.toml index b7c107c6..adc2651b 100644 --- a/python/functionstream-api/pyproject.toml +++ b/python/functionstream-api/pyproject.toml @@ -25,7 +25,9 @@ dependencies = [ "cloudpickle>=2.0.0", ] -[tool.setuptools] -package-dir = {"" = "src"} -packages = ["fs_api", "fs_api.store"] +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-dir] +"" = "src" diff --git a/python/functionstream-api/src/fs_api/__init__.py b/python/functionstream-api/src/fs_api/__init__.py index 1ecdd42d..7c185282 100644 --- a/python/functionstream-api/src/fs_api/__init__.py +++ b/python/functionstream-api/src/fs_api/__init__.py @@ -23,6 +23,42 @@ ComplexKey, KvIterator, KvStore, + Codec, + JsonCodec, + PickleCodec, + BytesCodec, + StringCodec, + BoolCodec, + Int64Codec, + Uint64Codec, + Int32Codec, + Uint32Codec, + Float64Codec, + Float32Codec, + OrderedInt64Codec, + OrderedUint64Codec, + OrderedInt32Codec, + OrderedUint32Codec, + OrderedFloat64Codec, + OrderedFloat32Codec, + ValueState, + MapEntry, + MapState, + infer_ordered_key_codec, + create_map_state_auto_key_codec, + ListState, + PriorityQueueState, + AggregateFunc, + AggregatingState, + ReduceFunc, + ReducingState, + KeyedStateFactory, + KeyedValueState, + KeyedMapState, + KeyedListState, + KeyedPriorityQueueState, + KeyedAggregatingState, + KeyedReducingState, ) __all__ = [ @@ -35,5 +71,41 @@ "ComplexKey", "KvIterator", "KvStore", + "Codec", + "JsonCodec", + "PickleCodec", + "BytesCodec", + "StringCodec", + "BoolCodec", + "Int64Codec", + "Uint64Codec", + "Int32Codec", + "Uint32Codec", + "Float64Codec", + "Float32Codec", + "OrderedInt64Codec", + "OrderedUint64Codec", + "OrderedInt32Codec", + "OrderedUint32Codec", + "OrderedFloat64Codec", + "OrderedFloat32Codec", + "ValueState", + "MapEntry", + "MapState", + "infer_ordered_key_codec", + "create_map_state_auto_key_codec", + "ListState", + "PriorityQueueState", + "AggregateFunc", + "AggregatingState", + "ReduceFunc", + "ReducingState", + "KeyedStateFactory", + "KeyedValueState", + "KeyedMapState", + "KeyedListState", + "KeyedPriorityQueueState", + "KeyedAggregatingState", + "KeyedReducingState", ] diff --git a/python/functionstream-api/src/fs_api/context.py b/python/functionstream-api/src/fs_api/context.py index 52cbd8a5..5c053b17 100644 --- a/python/functionstream-api/src/fs_api/context.py +++ b/python/functionstream-api/src/fs_api/context.py @@ -10,19 +10,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -fs_api.context - -Context: Context object -""" import abc -from typing import Dict -from .store import KvStore +from typing import Dict, Optional, Type +from .store import ( + Codec, + KvStore, + ValueState, + MapState, + ListState, + PriorityQueueState, + AggregatingState, + ReducingState, + KeyedListStateFactory, + KeyedValueStateFactory, + KeyedMapStateFactory, + KeyedPriorityQueueStateFactory, + KeyedAggregatingStateFactory, + KeyedReducingStateFactory, +) -class Context(abc.ABC): - """Context object""" +class Context(abc.ABC): @abc.abstractmethod def emit(self, data: bytes, channel: int = 0): pass @@ -37,11 +46,135 @@ def getOrCreateKVStore(self, name: str) -> KvStore: @abc.abstractmethod def getConfig(self) -> Dict[str, str]: - """ - Get global configuration Map + pass + + @abc.abstractmethod + def getOrCreateValueState(self, store_name: str, codec: Codec) -> ValueState: + pass + + @abc.abstractmethod + def getOrCreateValueStateAutoCodec(self, store_name: str) -> ValueState: + pass + + @abc.abstractmethod + def getOrCreateMapState(self, store_name: str, key_codec: Codec, value_codec: Codec) -> MapState: + pass + + @abc.abstractmethod + def getOrCreateMapStateAutoKeyCodec(self, store_name: str, value_codec: Codec) -> MapState: + pass + + @abc.abstractmethod + def getOrCreateListState(self, store_name: str, codec: Codec) -> ListState: + pass + + @abc.abstractmethod + def getOrCreateListStateAutoCodec(self, store_name: str) -> ListState: + pass + + @abc.abstractmethod + def getOrCreatePriorityQueueState(self, store_name: str, codec: Codec) -> PriorityQueueState: + pass + + @abc.abstractmethod + def getOrCreatePriorityQueueStateAutoCodec(self, store_name: str) -> PriorityQueueState: + pass + + @abc.abstractmethod + def getOrCreateAggregatingState( + self, store_name: str, acc_codec: Codec, agg_func: object + ) -> AggregatingState: + pass + + @abc.abstractmethod + def getOrCreateAggregatingStateAutoCodec( + self, store_name: str, agg_func: object + ) -> AggregatingState: + pass + + @abc.abstractmethod + def getOrCreateReducingState( + self, store_name: str, value_codec: Codec, reduce_func: object + ) -> ReducingState: + pass + + @abc.abstractmethod + def getOrCreateReducingStateAutoCodec( + self, store_name: str, reduce_func: object + ) -> ReducingState: + pass + + @abc.abstractmethod + def getOrCreateKeyedListStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedListStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedListStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_type: Optional[Type] = None + ) -> KeyedListStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedValueStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedValueStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedValueStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_type: Optional[Type] = None + ) -> KeyedValueStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedMapStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, key_codec: Codec, value_codec: Codec + ) -> KeyedMapStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedMapStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedMapStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedPriorityQueueStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, item_codec: Codec + ) -> KeyedPriorityQueueStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedPriorityQueueStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, item_type: Optional[Type] = None + ) -> KeyedPriorityQueueStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedAggregatingStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, acc_codec: Codec, agg_func: object + ) -> KeyedAggregatingStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedAggregatingStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, agg_func: object, acc_type: Optional[Type] = None + ) -> KeyedAggregatingStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedReducingStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec, reduce_func: object + ) -> KeyedReducingStateFactory: + pass + + @abc.abstractmethod + def getOrCreateKeyedReducingStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, reduce_func: object, value_type: Optional[Type] = None + ) -> KeyedReducingStateFactory: + pass - Returns: - Dict[str, str]: Configuration dictionary - """ -__all__ = ['Context'] +__all__ = ["Context"] diff --git a/python/functionstream-api/src/fs_api/store/__init__.py b/python/functionstream-api/src/fs_api/store/__init__.py index 1cfdc8f8..1cf1eb8a 100644 --- a/python/functionstream-api/src/fs_api/store/__init__.py +++ b/python/functionstream-api/src/fs_api/store/__init__.py @@ -5,24 +5,119 @@ # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on "AS IS" BASIS, +# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from .error import KvError, KvNotFoundError, KvIOError, KvOtherError from .complexkey import ComplexKey from .iterator import KvIterator from .store import KvStore +from .common import StateKind + +from .codec import ( + Codec, + JsonCodec, + PickleCodec, + BytesCodec, + StringCodec, + BoolCodec, + Int64Codec, + Uint64Codec, + Int32Codec, + Uint32Codec, + Float64Codec, + Float32Codec, + OrderedInt64Codec, + OrderedUint64Codec, + OrderedInt32Codec, + OrderedUint32Codec, + OrderedFloat64Codec, + OrderedFloat32Codec, + default_codec_for, +) + +from .structures import ( + ValueState, + ListState, + MapEntry, + MapState, + infer_ordered_key_codec, + create_map_state_auto_key_codec, + PriorityQueueState, + AggregateFunc, + AggregatingState, + ReduceFunc, + ReducingState, +) + +from .keyed import ( + KeyedStateFactory, + KeyedListStateFactory, + KeyedValueStateFactory, + KeyedMapStateFactory, + KeyedPriorityQueueStateFactory, + KeyedAggregatingStateFactory, + KeyedReducingStateFactory, + KeyedValueState, + KeyedMapState, + KeyedListState, + KeyedPriorityQueueState, + KeyedAggregatingState, + KeyedReducingState, +) __all__ = [ - 'KvError', - 'KvNotFoundError', - 'KvIOError', - 'KvOtherError', - 'ComplexKey', - 'KvIterator', - 'KvStore', + "KvError", + "KvNotFoundError", + "KvIOError", + "KvOtherError", + "ComplexKey", + "KvIterator", + "KvStore", + "StateKind", + "Codec", + "JsonCodec", + "PickleCodec", + "BytesCodec", + "StringCodec", + "BoolCodec", + "Int64Codec", + "Uint64Codec", + "Int32Codec", + "Uint32Codec", + "Float64Codec", + "Float32Codec", + "OrderedInt64Codec", + "OrderedUint64Codec", + "OrderedInt32Codec", + "OrderedUint32Codec", + "OrderedFloat64Codec", + "OrderedFloat32Codec", + "default_codec_for", + "ValueState", + "MapEntry", + "MapState", + "infer_ordered_key_codec", + "create_map_state_auto_key_codec", + "ListState", + "PriorityQueueState", + "AggregateFunc", + "AggregatingState", + "ReduceFunc", + "ReducingState", + "KeyedStateFactory", + "KeyedListStateFactory", + "KeyedValueStateFactory", + "KeyedMapStateFactory", + "KeyedPriorityQueueStateFactory", + "KeyedAggregatingStateFactory", + "KeyedReducingStateFactory", + "KeyedValueState", + "KeyedMapState", + "KeyedListState", + "KeyedPriorityQueueState", + "KeyedAggregatingState", + "KeyedReducingState", ] - diff --git a/python/functionstream-api/src/fs_api/store/codec/__init__.py b/python/functionstream-api/src/fs_api/store/codec/__init__.py new file mode 100644 index 00000000..1438430f --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/__init__.py @@ -0,0 +1,53 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import Codec +from .json_codec import JsonCodec +from .pickle_codec import PickleCodec +from .bytes_codec import BytesCodec +from .string_codec import StringCodec +from .bool_codec import BoolCodec +from .int64_codec import Int64Codec +from .uint64_codec import Uint64Codec +from .int32_codec import Int32Codec +from .uint32_codec import Uint32Codec +from .float64_codec import Float64Codec +from .float32_codec import Float32Codec +from .ordered_int64_codec import OrderedInt64Codec +from .ordered_uint64_codec import OrderedUint64Codec +from .ordered_int32_codec import OrderedInt32Codec +from .ordered_uint32_codec import OrderedUint32Codec +from .ordered_float64_codec import OrderedFloat64Codec +from .ordered_float32_codec import OrderedFloat32Codec +from .default_codec import default_codec_for + +__all__ = [ + "Codec", + "JsonCodec", + "PickleCodec", + "BytesCodec", + "StringCodec", + "BoolCodec", + "Int64Codec", + "Uint64Codec", + "Int32Codec", + "Uint32Codec", + "Float64Codec", + "Float32Codec", + "OrderedInt64Codec", + "OrderedUint64Codec", + "OrderedInt32Codec", + "OrderedUint32Codec", + "OrderedFloat64Codec", + "OrderedFloat32Codec", + "default_codec_for", +] diff --git a/python/functionstream-api/src/fs_api/store/codec/base.py b/python/functionstream-api/src/fs_api/store/codec/base.py new file mode 100644 index 00000000..604d87ab --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/base.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, TypeVar + +T = TypeVar("T") + + +class Codec(Generic[T]): + supports_ordered_keys: bool = False + + def encode(self, value: T) -> bytes: + raise NotImplementedError + + def decode(self, data: bytes) -> T: + raise NotImplementedError diff --git a/python/functionstream-api/src/fs_api/store/codec/bool_codec.py b/python/functionstream-api/src/fs_api/store/codec/bool_codec.py new file mode 100644 index 00000000..6f259010 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/bool_codec.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import Codec + + +class BoolCodec(Codec[bool]): + supports_ordered_keys = True + + def encode(self, value: bool) -> bytes: + return b"\x01" if value else b"\x00" + + def decode(self, data: bytes) -> bool: + if len(data) != 1: + raise ValueError(f"invalid bool payload length: {len(data)}") + if data == b"\x00": + return False + if data == b"\x01": + return True + raise ValueError(f"invalid bool payload byte: {data[0]}") diff --git a/python/functionstream-api/src/fs_api/store/codec/bytes_codec.py b/python/functionstream-api/src/fs_api/store/codec/bytes_codec.py new file mode 100644 index 00000000..3ac38b15 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/bytes_codec.py @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import Codec + + +class BytesCodec(Codec[bytes]): + supports_ordered_keys = True + + def encode(self, value: bytes) -> bytes: + return bytes(value) + + def decode(self, data: bytes) -> bytes: + return bytes(data) diff --git a/python/functionstream-api/src/fs_api/store/codec/default_codec.py b/python/functionstream-api/src/fs_api/store/codec/default_codec.py new file mode 100644 index 00000000..edc10d2e --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/default_codec.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Type + +from .base import Codec +from .bool_codec import BoolCodec +from .bytes_codec import BytesCodec +from .json_codec import JsonCodec +from .ordered_float32_codec import OrderedFloat32Codec +from .ordered_float64_codec import OrderedFloat64Codec +from .ordered_int32_codec import OrderedInt32Codec +from .ordered_int64_codec import OrderedInt64Codec +from .ordered_uint32_codec import OrderedUint32Codec +from .ordered_uint64_codec import OrderedUint64Codec +from .pickle_codec import PickleCodec +from .string_codec import StringCodec + + +def default_codec_for(value_type: Type[Any]) -> Codec[Any]: + """ + Return a default Codec for the given type, aligned with Go DefaultCodecFor. + Built-in types use ordered codecs where applicable; list/dict use JsonCodec; else PickleCodec. + """ + if value_type is bool: + return BoolCodec() + if value_type is int: + return OrderedInt64Codec() + if value_type is float: + return OrderedFloat64Codec() + if value_type is str: + return StringCodec() + if value_type is bytes: + return BytesCodec() + try: + if issubclass(value_type, int) and value_type is not bool: + return OrderedInt64Codec() + except TypeError: + pass + if value_type is list or value_type is dict: + return JsonCodec() + return PickleCodec() diff --git a/python/functionstream-api/src/fs_api/store/codec/float32_codec.py b/python/functionstream-api/src/fs_api/store/codec/float32_codec.py new file mode 100644 index 00000000..5dd6150f --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/float32_codec.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Float32Codec(Codec[float]): + def encode(self, value: float) -> bytes: + return struct.pack(">f", value) + + def decode(self, data: bytes) -> float: + if len(data) != 4: + raise ValueError(f"invalid float32 payload length: {len(data)}") + return struct.unpack(">f", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/float64_codec.py b/python/functionstream-api/src/fs_api/store/codec/float64_codec.py new file mode 100644 index 00000000..42879eaf --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/float64_codec.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Float64Codec(Codec[float]): + def encode(self, value: float) -> bytes: + return struct.pack(">d", value) + + def decode(self, data: bytes) -> float: + if len(data) != 8: + raise ValueError(f"invalid float64 payload length: {len(data)}") + return struct.unpack(">d", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/int32_codec.py b/python/functionstream-api/src/fs_api/store/codec/int32_codec.py new file mode 100644 index 00000000..eb850e7e --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/int32_codec.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Int32Codec(Codec[int]): + def encode(self, value: int) -> bytes: + return struct.pack(">i", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid int32 payload length: {len(data)}") + return struct.unpack(">i", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/int64_codec.py b/python/functionstream-api/src/fs_api/store/codec/int64_codec.py new file mode 100644 index 00000000..31208faa --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/int64_codec.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Int64Codec(Codec[int]): + def encode(self, value: int) -> bytes: + return struct.pack(">q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid int64 payload length: {len(data)}") + return struct.unpack(">q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/json_codec.py b/python/functionstream-api/src/fs_api/store/codec/json_codec.py new file mode 100644 index 00000000..b35416d2 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/json_codec.py @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import TypeVar + +from .base import Codec + +T = TypeVar("T") + + +class JsonCodec(Codec[T]): + def encode(self, value: T) -> bytes: + return json.dumps(value).encode("utf-8") + + def decode(self, data: bytes) -> T: + return json.loads(data.decode("utf-8")) diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py new file mode 100644 index 00000000..0dced99a --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_float32_codec.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedFloat32Codec(Codec[float]): + supports_ordered_keys = True + + def encode(self, value: float) -> bytes: + bits = struct.unpack(">I", struct.pack(">f", value))[0] + if bits & (1 << 31): + mapped = (~bits) & 0xFFFFFFFF + else: + mapped = bits ^ (1 << 31) + return struct.pack(">I", mapped) + + def decode(self, data: bytes) -> float: + if len(data) != 4: + raise ValueError(f"invalid ordered float32 payload length: {len(data)}") + mapped = struct.unpack(">I", data)[0] + if mapped & (1 << 31): + bits = mapped ^ (1 << 31) + else: + bits = (~mapped) & 0xFFFFFFFF + return struct.unpack(">f", struct.pack(">I", bits))[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py new file mode 100644 index 00000000..37e5100a --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_float64_codec.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedFloat64Codec(Codec[float]): + supports_ordered_keys = True + + def encode(self, value: float) -> bytes: + bits = struct.unpack(">Q", struct.pack(">d", value))[0] + if bits & (1 << 63): + mapped = (~bits) & 0xFFFFFFFFFFFFFFFF + else: + mapped = bits ^ (1 << 63) + return struct.pack(">Q", mapped) + + def decode(self, data: bytes) -> float: + if len(data) != 8: + raise ValueError(f"invalid ordered float64 payload length: {len(data)}") + mapped = struct.unpack(">Q", data)[0] + if mapped & (1 << 63): + bits = mapped ^ (1 << 63) + else: + bits = (~mapped) & 0xFFFFFFFFFFFFFFFF + return struct.unpack(">d", struct.pack(">Q", bits))[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py new file mode 100644 index 00000000..9787bf6c --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_int32_codec.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedInt32Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + mapped = (value & 0xFFFFFFFF) ^ (1 << 31) + return struct.pack(">I", mapped) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid ordered int32 payload length: {len(data)}") + mapped = struct.unpack(">I", data)[0] + raw = mapped ^ (1 << 31) + if raw >= (1 << 31): + return raw - (1 << 32) + return raw diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py new file mode 100644 index 00000000..dadd9c68 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_int64_codec.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedInt64Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + mapped = (value & 0xFFFFFFFFFFFFFFFF) ^ (1 << 63) + return struct.pack(">Q", mapped) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid ordered int64 payload length: {len(data)}") + mapped = struct.unpack(">Q", data)[0] + raw = mapped ^ (1 << 63) + if raw >= (1 << 63): + return raw - (1 << 64) + return raw diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py new file mode 100644 index 00000000..9f1ac2e1 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_uint32_codec.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedUint32Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("ordered uint32 value must be >= 0") + return struct.pack(">I", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid ordered uint32 payload length: {len(data)}") + return struct.unpack(">I", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py b/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py new file mode 100644 index 00000000..dab4a927 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/ordered_uint64_codec.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class OrderedUint64Codec(Codec[int]): + supports_ordered_keys = True + + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("ordered uint64 value must be >= 0") + return struct.pack(">Q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid ordered uint64 payload length: {len(data)}") + return struct.unpack(">Q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/pickle_codec.py b/python/functionstream-api/src/fs_api/store/codec/pickle_codec.py new file mode 100644 index 00000000..84066fc2 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/pickle_codec.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TypeVar + +import cloudpickle + +from .base import Codec + +T = TypeVar("T") + + +class PickleCodec(Codec[T]): + def encode(self, value: T) -> bytes: + return cloudpickle.dumps(value) + + def decode(self, data: bytes) -> T: + return cloudpickle.loads(data) diff --git a/python/functionstream-api/src/fs_api/store/codec/string_codec.py b/python/functionstream-api/src/fs_api/store/codec/string_codec.py new file mode 100644 index 00000000..6bb112a4 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/string_codec.py @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import Codec + + +class StringCodec(Codec[str]): + supports_ordered_keys = True + + def encode(self, value: str) -> bytes: + return value.encode("utf-8") + + def decode(self, data: bytes) -> str: + return data.decode("utf-8") diff --git a/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py b/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py new file mode 100644 index 00000000..281531d1 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/uint32_codec.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Uint32Codec(Codec[int]): + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("uint32 value must be >= 0") + return struct.pack(">I", value) + + def decode(self, data: bytes) -> int: + if len(data) != 4: + raise ValueError(f"invalid uint32 payload length: {len(data)}") + return struct.unpack(">I", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py b/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py new file mode 100644 index 00000000..634d7dcb --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/codec/uint64_codec.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +from .base import Codec + + +class Uint64Codec(Codec[int]): + def encode(self, value: int) -> bytes: + if value < 0: + raise ValueError("uint64 value must be >= 0") + return struct.pack(">Q", value) + + def decode(self, data: bytes) -> int: + if len(data) != 8: + raise ValueError(f"invalid uint64 payload length: {len(data)}") + return struct.unpack(">Q", data)[0] diff --git a/python/functionstream-api/src/fs_api/store/common/__init__.py b/python/functionstream-api/src/fs_api/store/common/__init__.py new file mode 100644 index 00000000..4056a4e2 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/common/__init__.py @@ -0,0 +1,88 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct +from enum import IntEnum +from typing import Tuple + +from ..error import KvError + + +class StateKind(IntEnum): + VALUE = 0 + LIST = 1 + PRIORITY_QUEUE = 2 + MAP = 3 + AGGREGATING = 4 + REDUCING = 5 + + def prefix(self) -> bytes: + if self == StateKind.VALUE: + return b"__fssdk__/value/" + if self == StateKind.LIST: + return b"__fssdk__/list/" + if self == StateKind.PRIORITY_QUEUE: + return b"__fssdk__/priority_queue/" + if self == StateKind.MAP: + return b"" + if self == StateKind.AGGREGATING: + return b"__fssdk__/aggregating/" + if self == StateKind.REDUCING: + return b"__fssdk__/reducing/" + return b"" + + def group(self) -> bytes: + if self in (StateKind.VALUE, StateKind.AGGREGATING, StateKind.REDUCING): + return b"" + if self == StateKind.LIST: + return b"__fssdk__/list" + if self == StateKind.PRIORITY_QUEUE: + return b"__fssdk__/priority_queue" + if self == StateKind.MAP: + return b"__fssdk__/map" + return b"" + + +def validate_state_name(name: str) -> None: + if not isinstance(name, str) or not name.strip(): + raise KvError("state name must be a non-empty string") + + +def encode_int64_lex(value: int) -> bytes: + mapped = (value & 0xFFFFFFFFFFFFFFFF) ^ (1 << 63) + return struct.pack(">Q", mapped) + + +def encode_priority_key(priority: int, seq: int) -> bytes: + return encode_int64_lex(priority) + struct.pack(">Q", seq) + + +def decode_priority_key(data: bytes) -> Tuple[int, int]: + if len(data) != 16: + raise KvError("invalid priority queue key length") + mapped_priority = struct.unpack(">Q", data[:8])[0] + unsigned_priority = mapped_priority ^ (1 << 63) + if unsigned_priority >= (1 << 63): + priority = unsigned_priority - (1 << 64) + else: + priority = unsigned_priority + seq = struct.unpack(">Q", data[8:])[0] + return priority, seq + + +__all__ = [ + "StateKind", + "validate_state_name", + "encode_int64_lex", + "encode_priority_key", + "decode_priority_key", +] diff --git a/python/functionstream-api/src/fs_api/store/keyed/__init__.py b/python/functionstream-api/src/fs_api/store/keyed/__init__.py new file mode 100644 index 00000000..f12cdfbb --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/__init__.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .keyed_state_factory import KeyedStateFactory +from .keyed_value_state import KeyedValueState, KeyedValueStateFactory +from .keyed_list_state import KeyedListState, KeyedListStateFactory +from .keyed_map_state import KeyedMapEntry, KeyedMapState, KeyedMapStateFactory +from .keyed_priority_queue_state import KeyedPriorityQueueState, KeyedPriorityQueueStateFactory +from .keyed_aggregating_state import AggregateFunc, KeyedAggregatingState, KeyedAggregatingStateFactory +from .keyed_reducing_state import KeyedReducingState, KeyedReducingStateFactory, ReduceFunc + +__all__ = [ + "KeyedStateFactory", + "KeyedListStateFactory", + "KeyedValueStateFactory", + "KeyedMapStateFactory", + "KeyedPriorityQueueStateFactory", + "KeyedAggregatingStateFactory", + "KeyedReducingStateFactory", + "KeyedValueState", + "KeyedListState", + "KeyedMapEntry", + "KeyedMapState", + "KeyedPriorityQueueState", + "KeyedAggregatingState", + "KeyedReducingState", + "AggregateFunc", + "ReduceFunc", +] diff --git a/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py b/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py new file mode 100644 index 00000000..81dc850f --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/_keyed_common.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from ..error import KvError + +KEYED_VALUE_GROUP = b"__fssdk__/keyed/value" +KEYED_LIST_GROUP = b"__fssdk__/keyed/list" +KEYED_MAP_GROUP = b"__fssdk__/keyed/map" +KEYED_PQ_GROUP = b"__fssdk__/keyed/pq" +KEYED_AGGREGATING_GROUP = b"__fssdk__/keyed/aggregating" +KEYED_REDUCING_GROUP = b"__fssdk__/keyed/reducing" + + +def ensure_ordered_key_codec(codec: Any, label: str) -> None: + if not getattr(codec, "supports_ordered_keys", False): + raise KvError(f"{label} key codec must support ordered key encoding") + + +__all__ = [ + "ensure_ordered_key_codec", + "KEYED_VALUE_GROUP", + "KEYED_LIST_GROUP", + "KEYED_MAP_GROUP", + "KEYED_PQ_GROUP", + "KEYED_AGGREGATING_GROUP", + "KEYED_REDUCING_GROUP", +] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py new file mode 100644 index 00000000..c0e7173a --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_aggregating_state.py @@ -0,0 +1,124 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Optional, Protocol, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +from ._keyed_common import KEYED_AGGREGATING_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +T_agg = TypeVar("T_agg") +ACC = TypeVar("ACC") +R = TypeVar("R") + + +class AggregateFunc(Protocol[T_agg, ACC, R]): + def create_accumulator(self) -> ACC: ... + def add(self, value: T_agg, accumulator: ACC) -> ACC: ... + def get_result(self, accumulator: ACC) -> R: ... + def merge(self, a: ACC, b: ACC) -> ACC: ... + + +class KeyedAggregatingStateFactory(Generic[T_agg, ACC, R]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + acc_codec: Codec[ACC], + agg_func: "AggregateFunc[T_agg, ACC, R]", + ): + if store is None: + raise KvError("keyed aggregating state factory store must not be None") + if namespace is None: + raise KvError("keyed aggregating state factory namespace must not be None") + if key_group is None: + raise KvError("keyed aggregating state factory key_group must not be None") + if acc_codec is None or agg_func is None: + raise KvError("keyed aggregating state factory acc_codec and agg_func must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._acc_codec = acc_codec + self._agg_func = agg_func + + def new_aggregating(self, key_codec: Codec[K]) -> "KeyedAggregatingState[K, T_agg, ACC, R]": + ensure_ordered_key_codec(key_codec, "keyed aggregating") + return KeyedAggregatingState( + self._store, + self._namespace, + key_codec, + self._acc_codec, + self._agg_func, + self._key_group, + ) + + +class KeyedAggregatingState(Generic[K, T_agg, ACC, R]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + acc_codec: Codec[ACC], + agg_func: AggregateFunc[T_agg, ACC, R], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed aggregating state namespace must not be None") + if key_group is None: + raise KvError("keyed aggregating state key_group must not be None") + if acc_codec is None: + raise KvError("keyed aggregating state acc_codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._key_codec = key_codec + self._acc_codec = acc_codec + self._agg_func = agg_func + ensure_ordered_key_codec(key_codec, "keyed aggregating") + + def _build_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=b"", + ) + + def add(self, key: K, value: T_agg) -> None: + ck = self._build_ck(key) + raw = self._store.get(ck) + if raw is None: + acc = self._agg_func.create_accumulator() + else: + acc = self._acc_codec.decode(raw) + new_acc = self._agg_func.add(value, acc) + self._store.put(ck, self._acc_codec.encode(new_acc)) + + def get(self, key: K) -> Tuple[Optional[R], bool]: + ck = self._build_ck(key) + raw = self._store.get(ck) + if raw is None: + return (None, False) + acc = self._acc_codec.decode(raw) + return (self._agg_func.get_result(acc), True) + + def clear(self, key: K) -> None: + self._store.delete(self._build_ck(key)) + + +__all__ = ["AggregateFunc", "KeyedAggregatingState", "KeyedAggregatingStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py new file mode 100644 index 00000000..ee49d213 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_list_state.py @@ -0,0 +1,140 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct +from typing import Generic, List, Optional, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +from ._keyed_common import KEYED_LIST_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +V = TypeVar("V") + + +class KeyedListStateFactory(Generic[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ): + if store is None: + raise KvError("keyed list state factory store must not be None") + if namespace is None: + raise KvError("keyed list state factory namespace must not be None") + if key_group is None: + raise KvError("keyed list state factory key_group must not be None") + if value_codec is None: + raise KvError("keyed list value codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._value_codec = value_codec + + def new_list(self, key_codec: Codec[K]) -> "KeyedListState[K, V]": + ensure_ordered_key_codec(key_codec, "keyed list") + return KeyedListState( + self._store, + self._namespace, + key_codec, + self._value_codec, + self._key_group, + ) + + +class KeyedListState(Generic[K, V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + value_codec: Codec[V], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed list state namespace must not be None") + if key_group is None: + raise KvError("keyed list state key_group must not be None") + if value_codec is None: + raise KvError("keyed list state value_codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._key_codec = key_codec + self._value_codec = value_codec + ensure_ordered_key_codec(key_codec, "keyed list") + + def _build_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=b"", + ) + + def add(self, key: K, value: V) -> None: + payload = self._serialize_one(value) + self._store.merge(self._build_ck(key), payload) + + def add_all(self, key: K, values: List[V]) -> None: + if not values: + return + payload = self._serialize_batch(values) + self._store.merge(self._build_ck(key), payload) + + def get(self, key: K) -> List[V]: + raw = self._store.get(self._build_ck(key)) + if raw is None: + return [] + return self._deserialize(raw) + + def update(self, key: K, values: List[V]) -> None: + if len(values) == 0: + self.clear(key) + return + self._store.put(self._build_ck(key), self._serialize_batch(values)) + + def clear(self, key: K) -> None: + self._store.delete(self._build_ck(key)) + + def _serialize_one(self, value: V) -> bytes: + encoded = self._value_codec.encode(value) + return struct.pack(">I", len(encoded)) + encoded + + def _serialize_batch(self, values: List[V]) -> bytes: + parts = [] + for v in values: + encoded = self._value_codec.encode(v) + parts.append(struct.pack(">I", len(encoded)) + encoded) + return b"".join(parts) + + def _deserialize(self, raw: bytes) -> List[V]: + out = [] + idx = 0 + while idx < len(raw): + if len(raw) - idx < 4: + raise KvError("corrupted keyed list payload: truncated length") + (item_len,) = struct.unpack(">I", raw[idx : idx + 4]) + idx += 4 + if item_len < 0 or len(raw) - idx < item_len: + raise KvError("corrupted keyed list payload: invalid element length") + out.append(self._value_codec.decode(raw[idx : idx + item_len])) + idx += item_len + return out + + +__all__ = ["KeyedListState", "KeyedListStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py new file mode 100644 index 00000000..0234e002 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_map_state.py @@ -0,0 +1,193 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Generic, List, Optional, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +from ._keyed_common import KEYED_MAP_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +MK = TypeVar("MK") +MV = TypeVar("MV") + + +@dataclass +class KeyedMapEntry(Generic[MK, MV]): + key: MK + value: MV + + +class KeyedMapStateFactory(Generic[MK, MV]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + map_key_codec: Codec[MK], + map_value_codec: Codec[MV], + ): + if store is None: + raise KvError("keyed map state factory store must not be None") + if namespace is None: + raise KvError("keyed map state factory namespace must not be None") + if key_group is None: + raise KvError("keyed map state factory key_group must not be None") + if map_key_codec is None or map_value_codec is None: + raise KvError("keyed map state factory map_key_codec and map_value_codec must not be None") + ensure_ordered_key_codec(map_key_codec, "keyed map inner") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._map_key_codec = map_key_codec + self._map_value_codec = map_value_codec + + def new_map(self, key_codec: Codec[K]) -> "KeyedMapState[K, MK, MV]": + ensure_ordered_key_codec(key_codec, "keyed map") + return KeyedMapState( + self._store, + self._namespace, + key_codec, + self._map_key_codec, + self._map_value_codec, + self._key_group, + ) + + +class KeyedMapState(Generic[K, MK, MV]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + map_key_codec: Codec[MK], + map_value_codec: Codec[MV], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed map state namespace must not be None") + if key_group is None: + raise KvError("keyed map state key_group must not be None") + if map_key_codec is None or map_value_codec is None: + raise KvError("keyed map state map_key_codec and map_value_codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._key_codec = key_codec + self._map_key_codec = map_key_codec + self._map_value_codec = map_value_codec + ensure_ordered_key_codec(key_codec, "keyed map") + ensure_ordered_key_codec(map_key_codec, "keyed map inner") + + def put(self, key: K, map_key: MK, value: MV) -> None: + ck = ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=self._map_key_codec.encode(map_key), + ) + self._store.put(ck, self._map_value_codec.encode(value)) + + def get(self, key: K, map_key: MK) -> Optional[MV]: + ck = ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=self._map_key_codec.encode(map_key), + ) + raw = self._store.get(ck) + if raw is None: + return None + return self._map_value_codec.decode(raw) + + def delete(self, key: K, map_key: MK) -> None: + ck = ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=self._map_key_codec.encode(map_key), + ) + self._store.delete(ck) + + def clear(self, key: K) -> None: + prefix_ck = ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=b"", + ) + self._store.delete_prefix(prefix_ck) + + def all(self, key: K): + it = self._store.scan_complex( + self._key_group, + self._key_codec.encode(key), + self._namespace, + ) + while it.has_next(): + item = it.next() + if item is None: + break + key_bytes, value_bytes = item + yield self._map_key_codec.decode(key_bytes), self._map_value_codec.decode(value_bytes) + + def len(self, key: K) -> int: + it = self._store.scan_complex( + self._key_group, + self._key_codec.encode(key), + self._namespace, + ) + n = 0 + while it.has_next(): + it.next() + n += 1 + return n + + def entries(self, key: K) -> List[KeyedMapEntry[MK, MV]]: + return [ + KeyedMapEntry(key=k, value=v) + for k, v in self.all(key) + ] + + def range( + self, key: K, start_inclusive: MK, end_exclusive: MK + ) -> List[KeyedMapEntry[MK, MV]]: + key_bytes = self._key_codec.encode(key) + start_bytes = self._map_key_codec.encode(start_inclusive) + end_bytes = self._map_key_codec.encode(end_exclusive) + user_keys = self._store.list_complex( + self._key_group, key_bytes, self._namespace, + start_bytes, end_bytes, + ) + out = [] + for uk in user_keys: + ck = ComplexKey( + key_group=self._key_group, + key=key_bytes, + namespace=self._namespace, + user_key=uk, + ) + raw = self._store.get(ck) + if raw is None: + raise KvError("map range key disappeared during scan") + out.append(KeyedMapEntry( + key=self._map_key_codec.decode(uk), + value=self._map_value_codec.decode(raw), + )) + return out + + +__all__ = ["KeyedMapEntry", "KeyedMapState", "KeyedMapStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py new file mode 100644 index 00000000..1bd3857a --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_priority_queue_state.py @@ -0,0 +1,140 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Iterator, Optional, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +from ._keyed_common import KEYED_PQ_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +V = TypeVar("V") + + +class KeyedPriorityQueueStateFactory(Generic[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ): + if store is None: + raise KvError("keyed priority queue state factory store must not be None") + if namespace is None: + raise KvError("keyed priority queue state factory namespace must not be None") + if key_group is None: + raise KvError("keyed priority queue state factory key_group must not be None") + if value_codec is None: + raise KvError("keyed priority queue state factory value codec must not be None") + ensure_ordered_key_codec(value_codec, "keyed priority queue value") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._value_codec = value_codec + + def new_priority_queue(self, key_codec: Codec[K]) -> "KeyedPriorityQueueState[K, V]": + ensure_ordered_key_codec(key_codec, "keyed priority queue") + return KeyedPriorityQueueState( + self._store, + self._namespace, + key_codec, + self._value_codec, + self._key_group, + ) + + +class KeyedPriorityQueueState(Generic[K, V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + value_codec: Codec[V], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed priority queue state namespace must not be None") + if key_group is None: + raise KvError("keyed priority queue state key_group must not be None") + if value_codec is None: + raise KvError("keyed priority queue state value_codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._key_codec = key_codec + self._value_codec = value_codec + ensure_ordered_key_codec(key_codec, "keyed priority queue") + ensure_ordered_key_codec(value_codec, "keyed priority queue value") + + def _ck(self, key: K, user_key: bytes) -> ComplexKey: + return ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=user_key, + ) + + def _prefix_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=b"", + ) + + def add(self, key: K, value: V) -> None: + user_key = self._value_codec.encode(value) + self._store.put(self._ck(key, user_key), b"") + + def peek(self, key: K) -> Tuple[Optional[V], bool]: + it = self._store.scan_complex( + self._key_group, + self._key_codec.encode(key), + self._namespace, + ) + if not it.has_next(): + return (None, False) + item = it.next() + if item is None: + return (None, False) + user_key, _ = item + return (self._value_codec.decode(user_key), True) + + def poll(self, key: K) -> Tuple[Optional[V], bool]: + val, found = self.peek(key) + if not found or val is None: + return (val, found) + self._store.delete(self._ck(key, self._value_codec.encode(val))) + return (val, True) + + def clear(self, key: K) -> None: + self._store.delete_prefix(self._prefix_ck(key)) + + def all(self, key: K) -> Iterator[V]: + it = self._store.scan_complex( + self._key_group, + self._key_codec.encode(key), + self._namespace, + ) + while it.has_next(): + item = it.next() + if item is None: + break + user_key, _ = item + yield self._value_codec.decode(user_key) + + +__all__ = ["KeyedPriorityQueueState", "KeyedPriorityQueueStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py new file mode 100644 index 00000000..c9c176c0 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_reducing_state.py @@ -0,0 +1,114 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Generic, Optional, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +from ._keyed_common import KEYED_REDUCING_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +V = TypeVar("V") + +ReduceFunc = Callable[[V, V], V] + + +class KeyedReducingStateFactory(Generic[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + reduce_func: ReduceFunc[V], + ): + if store is None: + raise KvError("keyed reducing state factory store must not be None") + if namespace is None: + raise KvError("keyed reducing state factory namespace must not be None") + if key_group is None: + raise KvError("keyed reducing state factory key_group must not be None") + if value_codec is None or reduce_func is None: + raise KvError("keyed reducing state factory value_codec and reduce_func must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._value_codec = value_codec + self._reduce_func = reduce_func + + def new_reducing(self, key_codec: Codec[K]) -> "KeyedReducingState[K, V]": + ensure_ordered_key_codec(key_codec, "keyed reducing") + return KeyedReducingState( + self._store, + self._namespace, + key_codec, + self._value_codec, + self._reduce_func, + self._key_group, + ) + + +class KeyedReducingState(Generic[K, V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + value_codec: Codec[V], + reduce_func: ReduceFunc[V], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed reducing state namespace must not be None") + if key_group is None: + raise KvError("keyed reducing state key_group must not be None") + if value_codec is None: + raise KvError("keyed reducing state value_codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._key_codec = key_codec + self._value_codec = value_codec + self._reduce_func = reduce_func + ensure_ordered_key_codec(key_codec, "keyed reducing") + + def _build_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=b"", + ) + + def add(self, key: K, value: V) -> None: + ck = self._build_ck(key) + raw = self._store.get(ck) + if raw is None: + result = value + else: + result = self._reduce_func(self._value_codec.decode(raw), value) + self._store.put(ck, self._value_codec.encode(result)) + + def get(self, key: K) -> Tuple[Optional[V], bool]: + raw = self._store.get(self._build_ck(key)) + if raw is None: + return (None, False) + return (self._value_codec.decode(raw), True) + + def clear(self, key: K) -> None: + self._store.delete(self._build_ck(key)) + + +__all__ = ["ReduceFunc", "KeyedReducingState", "KeyedReducingStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py new file mode 100644 index 00000000..ebbfa1b8 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_state_factory.py @@ -0,0 +1,113 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TypeVar + +from ..codec import Codec +from ..error import KvError +from ..store import KvStore + +from .keyed_aggregating_state import AggregateFunc, KeyedAggregatingState +from .keyed_list_state import KeyedListState +from .keyed_map_state import KeyedMapState +from .keyed_priority_queue_state import KeyedPriorityQueueState +from .keyed_reducing_state import KeyedReducingState, ReduceFunc +from .keyed_value_state import KeyedValueState + +K = TypeVar("K") +V = TypeVar("V") +MK = TypeVar("MK") +MV = TypeVar("MV") +T_agg = TypeVar("T_agg") +ACC = TypeVar("ACC") +R = TypeVar("R") + + +class KeyedStateFactory: + def __init__(self, store: KvStore, namespace: bytes, key_group: bytes): + if store is None: + raise KvError("keyed state factory store must not be None") + if namespace is None: + raise KvError("keyed state factory namespace must not be None") + if key_group is None: + raise KvError("keyed state factory key_group must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._kind = None + + def new_keyed_value(self, key_codec: Codec[K], value_codec: Codec[V]) -> KeyedValueState[K, V]: + self._claim_kind("value") + if value_codec is None: + raise KvError("keyed value state value_codec must not be None") + return KeyedValueState(self._store, self._namespace, key_codec, value_codec, self._key_group) + + def new_keyed_map( + self, key_codec: Codec[K], map_key_codec: Codec[MK], map_value_codec: Codec[MV] + ) -> KeyedMapState[K, MK, MV]: + self._claim_kind("map") + if map_key_codec is None or map_value_codec is None: + raise KvError("keyed map state map_key_codec and map_value_codec must not be None") + return KeyedMapState( + self._store, self._namespace, key_codec, map_key_codec, map_value_codec, self._key_group + ) + + def new_keyed_list(self, key_codec: Codec[K], value_codec: Codec[V]) -> KeyedListState[K, V]: + self._claim_kind("list") + if value_codec is None: + raise KvError("keyed list state value_codec must not be None") + return KeyedListState(self._store, self._namespace, key_codec, value_codec, self._key_group) + + def new_keyed_priority_queue( + self, key_codec: Codec[K], value_codec: Codec[V] + ) -> KeyedPriorityQueueState[K, V]: + self._claim_kind("priority_queue") + if value_codec is None: + raise KvError("keyed priority queue state value_codec must not be None") + return KeyedPriorityQueueState( + self._store, self._namespace, key_codec, value_codec, self._key_group + ) + + def new_keyed_aggregating( + self, + key_codec: Codec[K], + acc_codec: Codec[ACC], + agg_func: AggregateFunc[T_agg, ACC, R], + ) -> KeyedAggregatingState[K, T_agg, ACC, R]: + self._claim_kind("aggregating") + if acc_codec is None or agg_func is None: + raise KvError("keyed aggregating state acc_codec and agg_func must not be None") + return KeyedAggregatingState( + self._store, self._namespace, key_codec, acc_codec, agg_func, self._key_group + ) + + def new_keyed_reducing( + self, key_codec: Codec[K], value_codec: Codec[V], reduce_func: ReduceFunc[V] + ) -> KeyedReducingState[K, V]: + self._claim_kind("reducing") + if value_codec is None or reduce_func is None: + raise KvError("keyed reducing state value_codec and reduce_func must not be None") + return KeyedReducingState( + self._store, self._namespace, key_codec, value_codec, reduce_func, self._key_group + ) + + def _claim_kind(self, kind: str) -> None: + if self._kind is None: + self._kind = kind + return + if self._kind != kind: + raise KvError( + f"keyed state factory already bound to '{self._kind}', cannot create '{kind}'" + ) + + +__all__ = ["KeyedStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py new file mode 100644 index 00000000..334c4337 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/keyed/keyed_value_state.py @@ -0,0 +1,103 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Optional, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +from ._keyed_common import KEYED_VALUE_GROUP, ensure_ordered_key_codec + +K = TypeVar("K") +V = TypeVar("V") + + +class KeyedValueStateFactory(Generic[V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_group: bytes, + value_codec: Codec[V], + ): + if store is None: + raise KvError("keyed value state factory store must not be None") + if namespace is None: + raise KvError("keyed value state factory namespace must not be None") + if key_group is None: + raise KvError("keyed value state factory key_group must not be None") + if value_codec is None: + raise KvError("keyed value state factory value codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._value_codec = value_codec + + def new_value(self, key_codec: Codec[K]) -> "KeyedValueState[K, V]": + ensure_ordered_key_codec(key_codec, "keyed value") + return KeyedValueState( + self._store, + self._namespace, + key_codec, + self._value_codec, + self._key_group, + ) + + +class KeyedValueState(Generic[K, V]): + def __init__( + self, + store: KvStore, + namespace: bytes, + key_codec: Codec[K], + value_codec: Codec[V], + key_group: bytes, + ): + if namespace is None: + raise KvError("keyed value state namespace must not be None") + if key_group is None: + raise KvError("keyed value state key_group must not be None") + if value_codec is None: + raise KvError("keyed value state value_codec must not be None") + self._store = store + self._namespace = namespace + self._key_group = key_group + self._key_codec = key_codec + self._value_codec = value_codec + ensure_ordered_key_codec(key_codec, "keyed value") + + def _build_ck(self, key: K) -> ComplexKey: + return ComplexKey( + key_group=self._key_group, + key=self._key_codec.encode(key), + namespace=self._namespace, + user_key=b"", + ) + + def set(self, key: K, value: V) -> None: + ck = self._build_ck(key) + self._store.put(ck, self._value_codec.encode(value)) + + def get(self, key: K) -> Optional[V]: + ck = self._build_ck(key) + raw = self._store.get(ck) + if raw is None: + return None + return self._value_codec.decode(raw) + + def delete(self, key: K) -> None: + self._store.delete(self._build_ck(key)) + + +__all__ = ["KeyedValueState", "KeyedValueStateFactory"] diff --git a/python/functionstream-api/src/fs_api/store/structures/__init__.py b/python/functionstream-api/src/fs_api/store/structures/__init__.py new file mode 100644 index 00000000..9dce8ee3 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/__init__.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .value_state import ValueState +from .list_state import ListState +from .map_state import ( + MapEntry, + MapState, + infer_ordered_key_codec, + create_map_state_auto_key_codec, +) +from .priority_queue_state import PriorityQueueState +from .aggregating_state import AggregateFunc, AggregatingState +from .reducing_state import ReduceFunc, ReducingState + +__all__ = [ + "ValueState", + "ListState", + "MapEntry", + "MapState", + "infer_ordered_key_codec", + "create_map_state_auto_key_codec", + "PriorityQueueState", + "AggregateFunc", + "AggregatingState", + "ReduceFunc", + "ReducingState", +] diff --git a/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py new file mode 100644 index 00000000..52daa5c0 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/aggregating_state.py @@ -0,0 +1,82 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Optional, Protocol, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +T = TypeVar("T") +ACC = TypeVar("ACC") +R = TypeVar("R") + + +class AggregateFunc(Protocol[T, ACC, R]): + def create_accumulator(self) -> ACC: + ... + + def add(self, value: T, accumulator: ACC) -> ACC: + ... + + def get_result(self, accumulator: ACC) -> R: + ... + + def merge(self, a: ACC, b: ACC) -> ACC: + ... + + +class AggregatingState(Generic[T, ACC, R]): + def __init__( + self, + store: KvStore, + acc_codec: Codec[ACC], + agg_func: AggregateFunc[T, ACC, R], + ): + if store is None: + raise KvError("aggregating state store must not be None") + if acc_codec is None: + raise KvError("aggregating state acc codec must not be None") + if agg_func is None: + raise KvError("aggregating state agg func must not be None") + self._store = store + self._acc_codec = acc_codec + self._agg_func = agg_func + self._ck = ComplexKey( + key_group=b"", + key=b"", + namespace=b"", + user_key=b"", + ) + + def add(self, value: T) -> None: + raw = self._store.get(self._ck) + if raw is None: + acc = self._agg_func.create_accumulator() + else: + acc = self._acc_codec.decode(raw) + new_acc = self._agg_func.add(value, acc) + self._store.put(self._ck, self._acc_codec.encode(new_acc)) + + def get(self) -> Tuple[Optional[R], bool]: + raw = self._store.get(self._ck) + if raw is None: + return (None, False) + acc = self._acc_codec.decode(raw) + return (self._agg_func.get_result(acc), True) + + def clear(self) -> None: + self._store.delete(self._ck) + + +__all__ = ["AggregateFunc", "AggregatingState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/list_state.py b/python/functionstream-api/src/fs_api/store/structures/list_state.py new file mode 100644 index 00000000..bfa53e13 --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/list_state.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct +from typing import Generic, List, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +T = TypeVar("T") + + +class ListState(Generic[T]): + def __init__(self, store: KvStore, codec: Codec[T]): + if store is None: + raise KvError("list state store must not be None") + if codec is None: + raise KvError("list state codec must not be None") + self._store = store + self._codec = codec + self._ck = ComplexKey( + key_group=b"", + key=b"", + namespace=b"", + user_key=b"", + ) + + def add(self, value: T) -> None: + payload = self._serialize_one(value) + self._store.merge(self._ck, payload) + + def add_all(self, values: List[T]) -> None: + if not values: + return + payload = self._serialize_batch(values) + self._store.merge(self._ck, payload) + + def get(self) -> List[T]: + raw = self._store.get(self._ck) + if raw is None: + return [] + return self._deserialize(raw) + + def update(self, values: List[T]) -> None: + if len(values) == 0: + self.clear() + return + payload = self._serialize_batch(values) + self._store.put(self._ck, payload) + + def clear(self) -> None: + self._store.delete(self._ck) + + def _serialize_one(self, value: T) -> bytes: + encoded = self._codec.encode(value) + return struct.pack(">I", len(encoded)) + encoded + + def _serialize_batch(self, values: List[T]) -> bytes: + parts: List[bytes] = [] + for v in values: + encoded = self._codec.encode(v) + parts.append(struct.pack(">I", len(encoded)) + encoded) + return b"".join(parts) + + def _deserialize(self, raw: bytes) -> List[T]: + out: List[T] = [] + idx = 0 + while idx < len(raw): + if len(raw) - idx < 4: + raise KvError("corrupted list payload: truncated length") + (item_len,) = struct.unpack(">I", raw[idx : idx + 4]) + idx += 4 + if item_len < 0 or len(raw) - idx < item_len: + raise KvError("corrupted list payload: invalid element length") + item_raw = raw[idx : idx + item_len] + idx += item_len + out.append(self._codec.decode(item_raw)) + return out + + +__all__ = ["ListState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/map_state.py b/python/functionstream-api/src/fs_api/store/structures/map_state.py new file mode 100644 index 00000000..a1ceb8ee --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/map_state.py @@ -0,0 +1,165 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Generator, Generic, List, Optional, Tuple, Type, TypeVar + +from ..codec import ( + BoolCodec, + BytesCodec, + Codec, + OrderedFloat64Codec, + OrderedInt64Codec, + StringCodec, +) +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +K = TypeVar("K") +V = TypeVar("V") + + +@dataclass +class MapEntry(Generic[K, V]): + key: K + value: V + + +class MapState(Generic[K, V]): + def __init__(self, store: KvStore, key_codec: Codec[K], value_codec: Codec[V]): + if store is None: + raise KvError("map state store must not be None") + if key_codec is None or value_codec is None: + raise KvError("map state codecs must not be None") + if not getattr(key_codec, "supports_ordered_keys", False): + raise KvError("map state key codec must support ordered key encoding for range scans") + self._store = store + self._key_codec = key_codec + self._value_codec = value_codec + self._key_group = b"" + self._key = b"" + self._namespace = b"" + + @classmethod + def with_auto_key_codec( + cls, + store: KvStore, + key_type: Type[K], + value_codec: Codec[V], + ) -> "MapState[K, V]": + key_codec = infer_ordered_key_codec(key_type) + return cls(store, key_codec, value_codec) + + def put(self, key: K, value: V) -> None: + encoded_key = self._key_codec.encode(key) + encoded_value = self._value_codec.encode(value) + self._store.put(self._ck(encoded_key), encoded_value) + + def get(self, key: K) -> Optional[V]: + encoded_key = self._key_codec.encode(key) + raw = self._store.get(self._ck(encoded_key)) + if raw is None: + return None + return self._value_codec.decode(raw) + + def delete(self, key: K) -> None: + encoded_key = self._key_codec.encode(key) + self._store.delete(self._ck(encoded_key)) + + def clear(self) -> None: + self._store.delete_prefix(self._ck(b"")) + + def all(self) -> Generator[Tuple[K, V], None, None]: + it = self._store.scan_complex(self._key_group, self._key, self._namespace) + while it.has_next(): + item = it.next() + if item is None: + break + key_bytes, value_bytes = item + yield self._key_codec.decode(key_bytes), self._value_codec.decode(value_bytes) + + def len(self) -> int: + it = self._store.scan_complex(self._key_group, self._key, self._namespace) + size = 0 + while it.has_next(): + item = it.next() + if item is None: + break + size += 1 + return size + + def entries(self) -> List[MapEntry[K, V]]: + it = self._store.scan_complex(self._key_group, self._key, self._namespace) + out: List[MapEntry[K, V]] = [] + while it.has_next(): + item = it.next() + if item is None: + break + key_bytes, value_bytes = item + out.append( + MapEntry( + key=self._key_codec.decode(key_bytes), + value=self._value_codec.decode(value_bytes), + ) + ) + return out + + def range(self, start_inclusive: K, end_exclusive: K) -> List[MapEntry[K, V]]: + start_bytes = self._key_codec.encode(start_inclusive) + end_bytes = self._key_codec.encode(end_exclusive) + user_keys = self._store.list_complex(self._key_group, self._key, self._namespace, start_bytes, end_bytes) + out: List[MapEntry[K, V]] = [] + for user_key in user_keys: + raw = self._store.get(self._ck(user_key)) + if raw is None: + raise KvError("map range key disappeared during scan") + out.append( + MapEntry( + key=self._key_codec.decode(user_key), + value=self._value_codec.decode(raw), + ) + ) + return out + + def _ck(self, user_key: bytes) -> ComplexKey: + return ComplexKey( + key_group=self._key_group, + key=self._key, + namespace=self._namespace, + user_key=user_key, + ) + + +def infer_ordered_key_codec(key_type: Type[Any]) -> Codec[Any]: + if key_type is str: + return StringCodec() + if key_type is bytes: + return BytesCodec() + if key_type is bool: + return BoolCodec() + if key_type is int: + return OrderedInt64Codec() + if key_type is float: + return OrderedFloat64Codec() + raise KvError("unsupported map key type for auto codec") + + +def create_map_state_auto_key_codec( + store: KvStore, + key_type: Type[K], + value_codec: Codec[V], +) -> MapState[K, V]: + return MapState.with_auto_key_codec(store, key_type, value_codec) + + +__all__ = ["MapEntry", "MapState", "infer_ordered_key_codec", "create_map_state_auto_key_codec"] diff --git a/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py new file mode 100644 index 00000000..6386181d --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/priority_queue_state.py @@ -0,0 +1,84 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Iterator, Optional, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +T = TypeVar("T") + + +class PriorityQueueState(Generic[T]): + """State for a priority queue. codec must support ordered key encoding (supports_ordered_keys=True).""" + + def __init__(self, store: KvStore, codec: Codec[T]): + if store is None: + raise KvError("priority queue store must not be None") + if codec is None: + raise KvError("priority queue codec must not be None") + if not getattr(codec, "supports_ordered_keys", False): + raise KvError("priority queue codec must support ordered key encoding") + self._store = store + self._codec = codec + self._key_group = b"" + self._key = b"" + self._namespace = b"" + + def _ck(self, user_key: bytes) -> ComplexKey: + return ComplexKey( + key_group=self._key_group, + key=self._key, + namespace=self._namespace, + user_key=user_key, + ) + + def add(self, value: T) -> None: + user_key = self._codec.encode(value) + self._store.put(self._ck(user_key), b"") + + def peek(self) -> Tuple[Optional[T], bool]: + it = self._store.scan_complex(self._key_group, self._key, self._namespace) + if not it.has_next(): + return (None, False) + item = it.next() + if item is None: + return (None, False) + user_key, _ = item + return (self._codec.decode(user_key), True) + + def poll(self) -> Tuple[Optional[T], bool]: + val, found = self.peek() + if not found or val is None: + return (val, found) + user_key = self._codec.encode(val) + self._store.delete(self._ck(user_key)) + return (val, True) + + def clear(self) -> None: + self._store.delete_prefix( + ComplexKey(key_group=self._key_group, key=self._key, namespace=self._namespace, user_key=b"") + ) + + def all(self) -> Iterator[T]: + it = self._store.scan_complex(self._key_group, self._key, self._namespace) + while it.has_next(): + item = it.next() + if item is None: + break + user_key, _ = item + yield self._codec.decode(user_key) + + +__all__ = ["PriorityQueueState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/reducing_state.py b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py new file mode 100644 index 00000000..7c1b298d --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/reducing_state.py @@ -0,0 +1,60 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Generic, Optional, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +V = TypeVar("V") + +ReduceFunc = Callable[[V, V], V] + + +class ReducingState(Generic[V]): + def __init__(self, store: KvStore, value_codec: Codec[V], reduce_func: ReduceFunc[V]): + if store is None: + raise KvError("reducing state store must not be None") + if value_codec is None or reduce_func is None: + raise KvError("reducing state value codec and reduce function are required") + self._store = store + self._value_codec = value_codec + self._reduce_func = reduce_func + self._ck = ComplexKey( + key_group=b"", + key=b"", + namespace=b"", + user_key=b"", + ) + + def add(self, value: V) -> None: + raw = self._store.get(self._ck) + if raw is None: + result = value + else: + old_value = self._value_codec.decode(raw) + result = self._reduce_func(old_value, value) + self._store.put(self._ck, self._value_codec.encode(result)) + + def get(self) -> Tuple[Optional[V], bool]: + raw = self._store.get(self._ck) + if raw is None: + return (None, False) + return (self._value_codec.decode(raw), True) + + def clear(self) -> None: + self._store.delete(self._ck) + + +__all__ = ["ReduceFunc", "ReducingState"] diff --git a/python/functionstream-api/src/fs_api/store/structures/value_state.py b/python/functionstream-api/src/fs_api/store/structures/value_state.py new file mode 100644 index 00000000..1e5a46aa --- /dev/null +++ b/python/functionstream-api/src/fs_api/store/structures/value_state.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generic, Optional, Tuple, TypeVar + +from ..codec import Codec +from ..complexkey import ComplexKey +from ..error import KvError +from ..store import KvStore + +T = TypeVar("T") + + +class ValueState(Generic[T]): + def __init__(self, store: KvStore, codec: Codec[T]): + if store is None: + raise KvError("value state store must not be None") + if codec is None: + raise KvError("value state codec must not be None") + self._store = store + self._codec = codec + self._ck = ComplexKey( + key_group=b"", + key=b"", + namespace=b"", + user_key=b"", + ) + + def update(self, value: T) -> None: + self._store.put(self._ck, self._codec.encode(value)) + + def value(self) -> Tuple[Optional[T], bool]: + raw = self._store.get(self._ck) + if raw is None: + return (None, False) + return (self._codec.decode(raw), True) + + def clear(self) -> None: + self._store.delete(self._ck) + + +__all__ = ["ValueState"] diff --git a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py index 0dd9d624..ee86e62e 100644 --- a/python/functionstream-runtime/src/fs_runtime/store/fs_context.py +++ b/python/functionstream-runtime/src/fs_runtime/store/fs_context.py @@ -13,7 +13,26 @@ from typing import Dict, List, Tuple from fs_api.context import Context -from fs_api.store import KvStore +from fs_api.store import ( + Codec, + KvStore, + ValueState, + MapState, + ListState, + PriorityQueueState, + AggregatingState, + ReducingState, + KeyedListStateFactory, + KeyedValueStateFactory, + KeyedMapStateFactory, + KeyedPriorityQueueStateFactory, + KeyedAggregatingStateFactory, + KeyedReducingStateFactory, + PickleCodec, + BytesCodec, + OrderedInt64Codec, + default_codec_for, +) from .fs_collector import emit, emit_watermark from .fs_store import FSStore @@ -60,6 +79,139 @@ def getOrCreateKVStore(self, name: str) -> KvStore: def getConfig(self) -> Dict[str, str]: return self._CONFIG.copy() + def getOrCreateValueState(self, store_name: str, codec: Codec) -> ValueState: + store = self.getOrCreateKVStore(store_name) + return ValueState(store, codec) + + def getOrCreateValueStateAutoCodec(self, store_name: str) -> ValueState: + store = self.getOrCreateKVStore(store_name) + return ValueState(store, PickleCodec()) + + def getOrCreateMapState(self, store_name: str, key_codec: Codec, value_codec: Codec) -> MapState: + store = self.getOrCreateKVStore(store_name) + return MapState(store, key_codec, value_codec) + + def getOrCreateMapStateAutoKeyCodec(self, store_name: str, value_codec: Codec) -> MapState: + store = self.getOrCreateKVStore(store_name) + return MapState(store, BytesCodec(), value_codec) + + def getOrCreateListState(self, store_name: str, codec: Codec) -> ListState: + store = self.getOrCreateKVStore(store_name) + return ListState(store, codec) + + def getOrCreateListStateAutoCodec(self, store_name: str) -> ListState: + store = self.getOrCreateKVStore(store_name) + return ListState(store, PickleCodec()) + + def getOrCreatePriorityQueueState(self, store_name: str, codec: Codec) -> PriorityQueueState: + store = self.getOrCreateKVStore(store_name) + return PriorityQueueState(store, codec) + + def getOrCreatePriorityQueueStateAutoCodec(self, store_name: str) -> PriorityQueueState: + store = self.getOrCreateKVStore(store_name) + return PriorityQueueState(store, OrderedInt64Codec()) + + def getOrCreateAggregatingState( + self, store_name: str, acc_codec: Codec, agg_func: object + ) -> AggregatingState: + store = self.getOrCreateKVStore(store_name) + return AggregatingState(store, acc_codec, agg_func) + + def getOrCreateAggregatingStateAutoCodec( + self, store_name: str, agg_func: object + ) -> AggregatingState: + store = self.getOrCreateKVStore(store_name) + return AggregatingState(store, PickleCodec(), agg_func) + + def getOrCreateReducingState( + self, store_name: str, value_codec: Codec, reduce_func: object + ) -> ReducingState: + store = self.getOrCreateKVStore(store_name) + return ReducingState(store, value_codec, reduce_func) + + def getOrCreateReducingStateAutoCodec( + self, store_name: str, reduce_func: object + ) -> ReducingState: + store = self.getOrCreateKVStore(store_name) + return ReducingState(store, PickleCodec(), reduce_func) + + def getOrCreateKeyedListStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedListStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedListStateFactory(store, namespace, key_group, value_codec) + + def getOrCreateKeyedListStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_type=None + ) -> KeyedListStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return KeyedListStateFactory(store, namespace, key_group, codec) + + def getOrCreateKeyedValueStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedValueStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedValueStateFactory(store, namespace, key_group, value_codec) + + def getOrCreateKeyedValueStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_type=None + ) -> KeyedValueStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return KeyedValueStateFactory(store, namespace, key_group, codec) + + def getOrCreateKeyedMapStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, key_codec: Codec, value_codec: Codec + ) -> KeyedMapStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedMapStateFactory(store, namespace, key_group, key_codec, value_codec) + + def getOrCreateKeyedMapStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec + ) -> KeyedMapStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedMapStateFactory(store, namespace, key_group, BytesCodec(), value_codec) + + def getOrCreateKeyedPriorityQueueStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, item_codec: Codec + ) -> KeyedPriorityQueueStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedPriorityQueueStateFactory(store, namespace, key_group, item_codec) + + def getOrCreateKeyedPriorityQueueStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, item_type=None + ) -> KeyedPriorityQueueStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(item_type) if item_type is not None else OrderedInt64Codec() + return KeyedPriorityQueueStateFactory(store, namespace, key_group, codec) + + def getOrCreateKeyedAggregatingStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, acc_codec: Codec, agg_func: object + ) -> KeyedAggregatingStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedAggregatingStateFactory(store, namespace, key_group, acc_codec, agg_func) + + def getOrCreateKeyedAggregatingStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, agg_func: object, acc_type=None + ) -> KeyedAggregatingStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(acc_type) if acc_type is not None else PickleCodec() + return KeyedAggregatingStateFactory(store, namespace, key_group, codec, agg_func) + + def getOrCreateKeyedReducingStateFactory( + self, store_name: str, namespace: bytes, key_group: bytes, value_codec: Codec, reduce_func: object + ) -> KeyedReducingStateFactory: + store = self.getOrCreateKVStore(store_name) + return KeyedReducingStateFactory(store, namespace, key_group, value_codec, reduce_func) + + def getOrCreateKeyedReducingStateFactoryAutoCodec( + self, store_name: str, namespace: bytes, key_group: bytes, reduce_func: object, value_type=None + ) -> KeyedReducingStateFactory: + store = self.getOrCreateKVStore(store_name) + codec = default_codec_for(value_type) if value_type is not None else PickleCodec() + return KeyedReducingStateFactory(store, namespace, key_group, codec, reduce_func) + __all__ = ['WitContext', 'convert_config_to_dict']