From cb3ddd63819f74a05a4130a4fae7d03d89d1bf0e Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Fri, 20 Mar 2026 16:35:02 +1100 Subject: [PATCH 01/22] different model --- server/pkg/model/node/node.go | 94 ++++++++++++++ server/pkg/model/raw/bigint.go | 28 +++++ server/pkg/model/raw/check.go | 20 +++ server/pkg/model/raw/enc.go | 114 +++++++++++++++++ server/pkg/model/raw/enc_test.go | 118 +++++++++++++++++ server/pkg/model/raw/gumnut.go | 191 ++++++++++++++++++++++++++++ server/pkg/model/raw/merge.go | 196 +++++++++++++++++++++++++++++ server/pkg/model/raw/merge_test.go | 86 +++++++++++++ server/pkg/model/raw/op.go | 75 +++++++++++ server/pkg/model/raw/prior.go | 180 ++++++++++++++++++++++++++ server/pkg/model/raw/prior_test.go | 57 +++++++++ 11 files changed, 1159 insertions(+) create mode 100644 server/pkg/model/node/node.go create mode 100644 server/pkg/model/raw/bigint.go create mode 100644 server/pkg/model/raw/check.go create mode 100644 server/pkg/model/raw/enc.go create mode 100644 server/pkg/model/raw/enc_test.go create mode 100644 server/pkg/model/raw/gumnut.go create mode 100644 server/pkg/model/raw/merge.go create mode 100644 server/pkg/model/raw/merge_test.go create mode 100644 server/pkg/model/raw/op.go create mode 100644 server/pkg/model/raw/prior.go create mode 100644 server/pkg/model/raw/prior_test.go diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go new file mode 100644 index 0000000..9029166 --- /dev/null +++ b/server/pkg/model/node/node.go @@ -0,0 +1,94 @@ +package node + +import ( + "sort" + + "github.com/gumnutdev/foiled/server/pkg/model/raw" +) + +// Node inside Gumnut is a named inode. +// +// It comprises of various parts: +// - structural history +// - any number of named arrays of data in this inode +// +// Nodes do not actually have a specific length. +// They are effectively an infinite ticker tape. +// +// It knows about the handle who performed history operations, so it is exposed to end-clients. +type Node struct { + set SetMap // data here + readAt int // when we last read/flattened data + history []historyEntry +} + +type SetMap map[string]raw.GumnutSetArray + +func (sm *SetMap) tidy() { + for key, arr := range *sm { + if len(arr) == 0 { + delete(*sm, key) + } + } +} + +type historyEntry struct { + run []raw.PriorOp // how does this xform our structure + set SetMap // what to set after xform + handle int // author (zero for server/none) + version int // version this creates +} + +// ClearHandle removes all known use of the given +ve handle here. +func (n *Node) ClearHandle(handle int) (change bool) { + if handle <= 0 { + return + } + + for i, e := range n.history { + if e.handle == handle { + n.history[i].handle = 0 + change = true + } + } + // TODO: we could purge old history (if saved to disk) + + return +} + +func (n *Node) Read(docVersion int) (sm SetMap) { + if n.readAt >= docVersion { + return n.set + } + + idx := sort.Search(len(n.history), func(i int) (match bool) { + return n.history[i].version > n.readAt + }) + tail := n.history[idx:] + + // FIXME: xform current data - can't distinguish from _new_ data + + // apply self structural changes + for _, entry := range tail { + pops := make([]raw.PriorOpSelf, len(entry.run)) + for i, each := range entry.run { + pops[i].PriorOp = each + } + + for key, arr := range entry.set { + if !arr.Merge(pops) { + panic("couldn't merge in read()") + } + entry.set[key] = arr + } + + // FIXME: xform new data to end + for key, extra := range entry.set { + n.set[key] = append(n.set[key], extra...) + } + } + n.set.tidy() // remove blank + + n.readAt = docVersion + return n.set +} diff --git a/server/pkg/model/raw/bigint.go b/server/pkg/model/raw/bigint.go new file mode 100644 index 0000000..e929d4d --- /dev/null +++ b/server/pkg/model/raw/bigint.go @@ -0,0 +1,28 @@ +package raw + +func isBigInt(v string) (ok bool) { + length := len(v) + if length == 0 { + return false + } + + if length == 1 && v[0] == '0' { + return true + } + + // check leading 1-9 with optional "-" + if v[0] == '-' { + if length == 1 || v[1] < '1' || v[1] > '9' { + return false + } + } else if v[0] < '1' || v[0] > '9' { + return false + } + + for i := 1; i < length; i++ { + if v[i] < '0' || v[i] > '9' { + return false + } + } + return true +} diff --git a/server/pkg/model/raw/check.go b/server/pkg/model/raw/check.go new file mode 100644 index 0000000..f611a4f --- /dev/null +++ b/server/pkg/model/raw/check.go @@ -0,0 +1,20 @@ +package raw + +import ( + "slices" + "unsafe" +) + +func init() { + var i int + if unsafe.Sizeof(i) != 8 { + panic("must run where int is 8 bytes") + } +} + +func extendSlice[X any](slice []X, length int) (out []X) { + if len(slice) >= length { + return slice + } + return append(slice, slices.Repeat(make([]X, 1), length-len(slice))...) +} diff --git a/server/pkg/model/raw/enc.go b/server/pkg/model/raw/enc.go new file mode 100644 index 0000000..8d8511e --- /dev/null +++ b/server/pkg/model/raw/enc.go @@ -0,0 +1,114 @@ +package raw + +import ( + "encoding/json" + "math" + + "github.com/gumnutdev/foiled/server/pkg/jsonhelp" +) + +type GumnutSet struct { + Before int // apply before this location (maybe -ve for user) + + String []uint16 // if set, prefer - only has uint16's + Data []GumnutData // otherwise use Data as fallback +} + +func (gs *GumnutSet) asData() (gd []GumnutData) { + if gs.String != nil { + gd = make([]GumnutData, len(gs.String)) + for i, r := range gs.String { + gd[i].Type = DataTypeNumber + gd[i].Data = math.Float64bits(float64(r)) + } + return gd + } + + return gs.Data +} + +// GumnutSetArray is a sequence of set operations which will be applied one after another. +type GumnutSetArray []GumnutSet + +func (arr GumnutSetArray) MarshalJSON() (b []byte, err error) { + out := []any{} + + for _, each := range arr { + if each.length() == 0 { + continue + } + + b, err := each.internalEncode() + if err != nil { + return nil, err + } + out = append(out, each.Before, json.RawMessage(b)) + } + + return json.Marshal(out) +} + +func (arr *GumnutSetArray) UnmarshalJSON(b []byte) (err error) { + var tmp []json.RawMessage + err = json.Unmarshal(b, &tmp) + if err != nil { + return err + } + + if len(tmp)%2 != 0 { + return ErrEncoding + } + + update := make([]GumnutSet, 0, len(tmp)/2) + + for i := 0; i < len(tmp); i += 2 { + var s GumnutSet + + err = json.Unmarshal(tmp[i], &s.Before) + if err != nil { + return err + } + + err = s.internalDecode(tmp[i+1]) + if err != nil { + return err + } + + update = append(update, s) + } + + *arr = update + return nil +} + +func (s GumnutSet) internalEncode() (b []byte, err error) { + if s.String != nil { + s := jsonhelp.DecodeFromUTF16(s.String) + return json.Marshal(s) + } + return json.Marshal(s.Data) +} + +func (s *GumnutSet) internalDecode(b []byte) (err error) { + if len(b) == 0 { + return ErrEncoding + } + + if b[0] == '"' { + x := jsonhelp.UnmarshalToUTF16(b) + if x == nil { + return ErrEncoding + } + s.String = x + return nil + } + + return json.Unmarshal(b, &s.Data) +} + +func (s *GumnutSet) length() (length int) { + if s.String != nil { + return len(s.String) + } + return len(s.Data) +} diff --git a/server/pkg/model/raw/enc_test.go b/server/pkg/model/raw/enc_test.go new file mode 100644 index 0000000..a488156 --- /dev/null +++ b/server/pkg/model/raw/enc_test.go @@ -0,0 +1,118 @@ +package raw + +import ( + "encoding/json" + "math" + "reflect" + "testing" +) + +func TestGumnutSetArray_EncodeDecode(t *testing.T) { + cases := []struct { + name string + in GumnutSetArray + }{ + { + name: "empty", + in: GumnutSetArray{}, + }, + { + name: "fast-path uint16 string", + in: GumnutSetArray{ + { + Before: 10, + String: []uint16{'h', 'e', 'l', 'l', 'o'}, + }, + { + Before: 20, + String: []uint16{'w', 'o', 'r', 'l', 'd'}, + }, + }, + }, + { + name: "fallback data array", + in: GumnutSetArray{ + { + Before: 5, + Data: []GumnutData{ + {Type: DataTypeNumber, Data: math.Float64bits(42)}, + {Type: DataTypeNumber, Data: math.Float64bits(-1.5)}, + }, + }, + }, + }, + { + name: "mixed fast-path and data", + in: GumnutSetArray{ + { + Before: 1, + String: []uint16{'A'}, + }, + { + Before: 2, + Data: []GumnutData{ + {Type: DataTypeNumber, Data: math.Float64bits(1.23)}, + }, + }, + }, + }, + { + name: "skips empty sets", + in: GumnutSetArray{ + {Before: 100, String: []uint16{}}, + {Before: 101, Data: []GumnutData{}}, + {Before: 102, String: []uint16{'x'}}, // only this should be serialized + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b, err := json.Marshal(tc.in) + if err != nil { + t.Fatalf("MarshalJSON failed: %v", err) + } + + var out GumnutSetArray + err = json.Unmarshal(b, &out) + if err != nil { + t.Fatalf("UnmarshalJSON failed: %v", err) + } + + // Generate expected values (since empty sets are dropped on encode) + var expected GumnutSetArray + for _, s := range tc.in { + if s.length() > 0 { + expected = append(expected, s) + } + } + + if len(expected) == 0 && len(out) == 0 { + return // both are empty, matches + } + + if !reflect.DeepEqual(expected, out) { + t.Errorf("got %#v, want %#v", out, expected) + } + }) + } +} + +func TestGumnutSet_AsData(t *testing.T) { + s := GumnutSet{ + Before: 1, + String: []uint16{'H', 'i'}, + } + + gd := s.asData() + if len(gd) != 2 { + t.Fatalf("expected 2 elements, got %d", len(gd)) + } + + if gd[0].Type != DataTypeNumber || math.Float64frombits(gd[0].Data) != 'H' { + t.Errorf("unexpected gd[0]: %+v", gd[0]) + } + if gd[1].Type != DataTypeNumber || math.Float64frombits(gd[1].Data) != 'i' { + t.Errorf("unexpected gd[1]: %+v", gd[1]) + } +} diff --git a/server/pkg/model/raw/gumnut.go b/server/pkg/model/raw/gumnut.go new file mode 100644 index 0000000..d54874f --- /dev/null +++ b/server/pkg/model/raw/gumnut.go @@ -0,0 +1,191 @@ +package raw + +import ( + "encoding/json" + "errors" + "math" +) + +type DataType int + +const ( + DataTypeEmpty DataType = iota // unset/void - distinct from undefined + DataTypeNumber // JS number (float64 in bits) + DataTypeBigInt // JS bigint (string) + DataTypeKnown // JS known: true, false, null, undefined + DataTypeString // immutable string + DataTypePosRef // ref to inode with pos (str=>inode, pos=>data) +) + +// GumnutData represents all possible values inside Gumnut. +// Some inodes may simply store data as arrays of uint16 (e.g., if that's all that has been seen). +type GumnutData struct { + Type DataType // type + Data uint64 // contains number, float bits, ... + String string // string option +} + +// JSRune returns the uint16 for this GumnutData, if possible. +func (gd *GumnutData) JSRune() (r uint16, ok bool) { + if gd.Type != DataTypeNumber { + return + } + + f64 := math.Float64frombits(gd.Data) + r = uint16(f64) + if float64(r) != f64 { + return + } + + return r, true +} + +// IsZero returns whether this is the explicitly uninitialized data. +func (gd *GumnutData) IsZero() (is bool) { + return gd.Type == DataTypeEmpty +} + +// MarshalJSON returns the base-case of marshalling a single value to JSON. +// Ideally, this is wrapped in GumnutDataArray, which handles faster run-length encoding. +func (gd GumnutData) MarshalJSON() (b []byte, err error) { + switch gd.Type { + case DataTypeEmpty: + // nb. explicitly cannot encode + + case DataTypeNumber: + f64 := math.Float64frombits(gd.Data) + + if math.IsInf(f64, 0) { + if f64 < 0 { + return []byte(`{"inf":-1}`), nil + } else { + return []byte(`{"inf":1}`), nil + } + } else if math.IsNaN(f64) { + return []byte(`{"inf":0}`), nil + } + + return json.Marshal(f64) + + case DataTypeBigInt: + var out struct { + BigInt string `json:"b"` + } + out.BigInt = gd.String + return json.Marshal(out) + + case DataTypeKnown: + if gd.Data != 0 { + switch gd.String { + case "null", "true", "false": + return []byte(gd.String), nil + } + return nil, ErrEncoding + } + + var out struct { + Known string `json:"x"` + } + out.Known = gd.String + return json.Marshal(out) + + case DataTypeString: + var out struct { + String string `json:"s"` + } + out.String = gd.String + return json.Marshal(out) + + case DataTypePosRef: + var out struct { + Inode string `json:"i"` + Pos uint64 `json:"p"` + } + out.Inode = gd.String + out.Pos = uint64(gd.Data) + return json.Marshal(out) + } + + return nil, ErrEncoding +} + +// UnmarshalJSON is the base-case of unmarshalling a single value from JSON. +func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { + s := string(b) + + switch s { + case "": + return ErrEncoding + + case "null", "true", "false": + g.Type = DataTypeKnown + g.Data = 1 // mark as known literal + g.String = s + return + } + + if s[0] != '{' { + // try number + var num float64 + err = json.Unmarshal(b, &num) + if err != nil { + return + } + g.Type = DataTypeNumber + g.Data = math.Float64bits(num) + return nil + } + + var tmp struct { + String *string `json:"s"` // immutable string + Known *string `json:"x"` + BigInt string `json:"b"` // bigint value + Inf *int `json:"inf"` + Inode string `json:"i"` + Pos *uint64 `json:"p"` + } + err = json.Unmarshal(b, &tmp) + if err != nil { + return + } + + switch { + case tmp.String != nil: + g.Type = DataTypeString + g.String = *tmp.String + + case tmp.Known != nil: + g.Type = DataTypeKnown + g.String = *tmp.Known + + case tmp.BigInt != "": + if !isBigInt(tmp.BigInt) { + return ErrEncoding + } + g.Type = DataTypeBigInt + g.String = tmp.BigInt + return + + case tmp.Inf != nil: + g.Type = DataTypeNumber + if *tmp.Inf == 0 { + g.Data = math.Float64bits(math.NaN()) + } else { + g.Data = math.Float64bits(math.Inf(int(*tmp.Inf))) + } + + case tmp.Pos != nil: + g.Type = DataTypePosRef + g.String = tmp.Inode + g.Data = *tmp.Pos + + default: + return ErrEncoding + } + + return +} + +var ( + ErrEncoding = errors.New("data encoding error") +) diff --git a/server/pkg/model/raw/merge.go b/server/pkg/model/raw/merge.go new file mode 100644 index 0000000..bc7d0d4 --- /dev/null +++ b/server/pkg/model/raw/merge.go @@ -0,0 +1,196 @@ +package raw + +import ( + "iter" + "log" + "slices" +) + +// IterData returns an iterator which yields all GumnutData here. +// This will not yield fast-path string data. +func (s GumnutSetArray) IterData() (it iter.Seq[*GumnutData]) { + return func(yield func(*GumnutData) (more bool)) { + for i := range s { + d := s[i].Data + + for j := range d { + if !yield(&d[j]) { + return + } + } + } + } +} + +// SetMap is a map from key to GumnutSetArray. +type SetMap map[string]GumnutSetArray + +// SetEntry describes how to modify a SetMap over time. +// The first SetEntry in any array is effectively the base state. +type SetEntry struct { + Prior []PriorOp + Version int + Set SetMap +} + +type TransformFunc func(fromVersion int, gd *GumnutData) + +type setEntry struct { + version int + str bool // was this encoded as String + gd GumnutData +} + +func (se *setEntry) isStr() (is bool) { + is = se.str + if is { + _, is = se.gd.JSRune() + } + return +} + +// MoveForward helps to move sets forward over structural ops. +// This transforms it structurally and wrt its data. +// The SetEntry array must have an incrementing version number. +func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { + // this effectively flattens history to a single entry + + // confirm we always go > version + var version int + for _, e := range hist { + if e.Version <= version { + return + } + version = e.Version + } + + work := make(map[string][]setEntry) + priors := []PriorOp{} + + // work backwards + for _, each := range slices.Backward(hist) { + + // for each keyed set + for key, update := range each.Set { + + // apply each part of the keyed set + for _, s := range update { + target := s.Before + + str := s.String != nil + + for _, gd := range slices.Backward(s.asData()) { + pos := target + if len(priors) > 0 { + pos = transformOverLocal(target, priors) + log.Printf("transformed pos=%v -> %v", target, pos) + } + target-- + + if pos == 0 { + continue // xformed away + } + index := pos - 1 + + slice := work[key] + if len(slice) < pos { + // off end, always exists + slice = extendSlice(slice, pos) + work[key] = slice + slice[index] = setEntry{each.Version, str, gd} + + } else if slice[index].version == 0 { + // check nothing already here + slice[index] = setEntry{each.Version, str, gd} + + } else { + continue + } + + r, _ := gd.JSRune() + log.Printf("ver=%v placing char=%c at index=%+v", each.Version, r, index) + } + } + } + + // if we have more priors, put it at the start of the array + if len(each.Prior) != 0 { + tmp := make([]PriorOp, len(each.Prior)+len(priors)) + copy(tmp, each.Prior) + copy(tmp[len(each.Prior):], priors) + priors = tmp + } + } + + // finally, xform all data _forward_ + if tr != nil { + for _, data := range work { + for i := range data { + if data[i].version == 0 { + continue + } + tr(data[i].version, &data[i].gd) + } + } + } + + // create minimal sets + out = make(SetMap, len(work)) + + for key, data := range work { + length := len(data) + for i := 0; i < length; { + if data[i].version == 0 { + i++ + continue + } + + str := data[i].isStr() + + // consume matching (non-zero version, str == base) + j := i + 1 + for j < length { + jstr := data[j].isStr() + if jstr != str || data[j].version == 0 { + break + } + j++ + } + + // got slice [i,j], convert to String or Data + set := GumnutSet{Before: j} + if str { + set.String = make([]uint16, j-i) + for k := i; k < j; k++ { + set.String[k-i], _ = data[k].gd.JSRune() + } + } else { + set.Data = make([]GumnutData, j-i) + for k := i; k < j; k++ { + set.Data[k-i] = data[k].gd + } + } + out[key] = append(out[key], set) + + i = j + } + } + + return out +} + +func transformOverLocal(v int, over []PriorOp) (out int) { + for _, each := range over { + if v == 0 { + return 0 + } + var innerGone bool + v, innerGone = each.TransformPos(v) + + if innerGone || v == 0 { + return 0 + } + } + + return v +} diff --git a/server/pkg/model/raw/merge_test.go b/server/pkg/model/raw/merge_test.go new file mode 100644 index 0000000..39a2da4 --- /dev/null +++ b/server/pkg/model/raw/merge_test.go @@ -0,0 +1,86 @@ +package raw + +import ( + "reflect" + "testing" + + "github.com/gumnutdev/foiled/server/pkg/jsonhelp" +) + +func TestMoveForward(t *testing.T) { + tests := []struct { + name string + src []SetEntry + out SetMap + tr TransformFunc + }{ + { + "zero", + []SetEntry{}, + SetMap{}, + nil, + }, + { + "basic", + []SetEntry{ + { + Version: 1, + Set: SetMap{ + "abc": GumnutSetArray{ + {Before: 16, String: jsonhelp.EncodeToUTF16("hello there")}, + }, + }, + }, + { + Version: 2, + Prior: []PriorOp{ + &priorOpRemove{lo: 9, hi: 10}, + &priorOpInsert{after: 9, length: 1}, + }, + Set: SetMap{ + "abc": GumnutSetArray{ + {Before: 10, String: jsonhelp.EncodeToUTF16("!")}, + }, + }, + }, + }, + SetMap{ + "abc": GumnutSetArray{ + {Before: 16, String: jsonhelp.EncodeToUTF16("hell! there")}, + }, + }, + nil, + }, + { + "merged", + []SetEntry{ + { + Version: 100, + Set: SetMap{ + "abc": GumnutSetArray{ + {Before: 80, String: jsonhelp.EncodeToUTF16("hello")}, + {Before: 86, String: jsonhelp.EncodeToUTF16("? there")}, + {Before: 87, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, + }, + }, + }, + }, + SetMap{ + "abc": GumnutSetArray{ + {Before: 86, String: jsonhelp.EncodeToUTF16("hello there")}, + {Before: 87, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, + }, + }, + nil, + }, + } + + for _, each := range tests { + t.Run(each.name, func(t *testing.T) { + got := MoveForward(each.src, each.tr) + if !reflect.DeepEqual(got, each.out) { + t.Errorf("got bad out=%+v expected=%+v", got, each.out) + } + }) + } +} diff --git a/server/pkg/model/raw/op.go b/server/pkg/model/raw/op.go new file mode 100644 index 0000000..d74f51b --- /dev/null +++ b/server/pkg/model/raw/op.go @@ -0,0 +1,75 @@ +package raw + +import ( + "slices" +) + +// Op is an operation, applied in-order, which modifies an array-like. +// +// It is different based on the length of Range: +// 1. insert zero data of Length at the position +// 2. remove between lo,hi (in any order) +// 3. flip around this lo/mid/hi point (in any order) +// +// Any other length is invalid/zero. +type Op struct { + Range []int + Length int +} + +// TransformOver transforms the receiver Op over the prior ops. +// This simply nukes the length in an insert if the position is removed. +func (op *Op) TransformOver(prior []PriorOpSelf) { + for i, v := range op.Range { + var gone bool + op.Range[i], gone = TransformOver(v, prior) + if gone { + op.Length = 0 + } + } +} + +// IsEmpty returns whether this Op, when applied successfully, is actually a no-op. +func (op *Op) IsEmpty() (ok bool) { + prior, ok := op.Apply() + return prior == nil && ok +} + +// Apply enacts this Op, converting to a PriorOp, or nil if this is a no-op. +// This returns not-ok if any Op values are negative. +func (op *Op) Apply() (prior PriorOp, ok bool) { + for _, v := range op.Range { + if v < 0 { + return // out of range + } + } + + r := make([]int, len(op.Range)) + copy(r, op.Range) + slices.Sort(r) + + switch len(r) { + default: + return + + case 1: + if op.Length <= 0 { + break // ok but no-op + } + prior = &priorOpInsert{after: r[0], length: op.Length} + + case 2: + if r[0] == r[1] { + break // ok but no-op + } + prior = &priorOpRemove{lo: r[0], hi: r[1]} + + case 3: + if r[0] == r[1] || r[1] == r[2] { + break // ok but no-op + } + prior = &priorOpFlip{lo: r[0], mid: r[1], hi: r[2]} + } + + return prior, true +} diff --git a/server/pkg/model/raw/prior.go b/server/pkg/model/raw/prior.go new file mode 100644 index 0000000..4133a60 --- /dev/null +++ b/server/pkg/model/raw/prior.go @@ -0,0 +1,180 @@ +package raw + +// TransformOver transforms a single value over a run of PriorOpSelf. +func TransformOver(v int, over []PriorOpSelf) (out int, gone bool) { + // match -ve focus + if v < 0 { + for i := len(over) - 1; i >= 0; i-- { + if !over[i].Self { + continue + } + + rop, _ := over[i].PriorOp.(ResolveOp) + if rop == nil { + continue + } + + v = rop.ResolvePos(v) + if v >= 0 { + over = over[i+1:] + break + } + } + if v < 0 { + return v, false + } + } + + // now do actual transform + for _, each := range over { + if v == 0 { + break + } + var innerGone bool + v, innerGone = each.TransformPos(v) + gone = gone || innerGone + } + return v, gone +} + +// PriorOpSelf must be passed to TransformOver, to denote whether the prior op was initated by this session. +// Required for transforming inserts. +type PriorOpSelf struct { + PriorOp + Self bool // is this Op created by the same session +} + +// PriorOp describes how to transform a position over time. +// Note that pos is 1-indexed. +type PriorOp interface { + TransformPos(pos int) (out int, gone bool) + Delta() (delta int) +} + +// ApplyPriorOpSlice applies this PriorOp to a generic slice. +func ApplyPriorOpSlice[X any](target *[]X, pop PriorOp) { + switch pop := pop.(type) { + case *priorOpInsert: + if len(*target) <= pop.after { + return + } + + tmp := (*target)[:pop.after:pop.after] + tmp = append(tmp, make([]X, pop.length)...) + tmp = append(tmp, (*target)[pop.after:]...) + *target = tmp + + case *priorOpRemove: + if len(*target) <= pop.lo { + return + } + + if len(*target) <= pop.hi { + *target = (*target)[:pop.lo] + return + } + + tmp := (*target)[:pop.lo:pop.lo] + tmp = append(tmp, (*target)[pop.hi:]...) + *target = tmp + + case *priorOpFlip: + panic("TODO: implement flip") + } +} + +// ResolveOp is run backwards, and is used to match -ve IDs for user focus. +// Note that pos is 1-indexed, or, likely, -ve indexed when passed here (-1 is in front of last user data inserted). +type ResolveOp interface { + ResolvePos(pos int) (out int) +} + +type priorOpInsert struct { + after, length int +} + +func (p *priorOpInsert) TransformPos(pos int) (out int, gone bool) { + if pos <= p.after { + // before or equal to insert (later ops win) + return pos, false + } else { + // after insert + return pos + p.length, false + } +} + +func (p *priorOpInsert) Delta() (delta int) { + return p.length +} + +func (p *priorOpInsert) ResolvePos(pos int) (out int) { + // pos will be -1 and higher for us + if pos >= 0 { + return pos // no resolve here + } + + into := -(pos + 1) + + if into < p.length { + out = p.after + p.length - into + if out < 0 { + panic("bad out") + } + return + } + + pos += p.length + if pos > -1 { + panic("bad pos") + } + return pos +} + +type priorOpRemove struct { + lo, hi int +} + +func (p *priorOpRemove) TransformPos(pos int) (out int, gone bool) { + if pos <= p.lo { + // before range + return pos, false + } else if pos <= p.hi { + // within range + return p.lo, true + } else { + // after range + return pos - (p.hi - p.lo), false + } +} + +func (p *priorOpRemove) Delta() (delta int) { + return p.hi - p.lo +} + +type priorOpFlip struct { + lo, mid, hi int +} + +func (p *priorOpFlip) TransformPos(pos int) (out int, gone bool) { + switch { + case pos <= p.lo: + // before + + case pos <= p.mid: + // lo-mid + return pos + (p.hi - p.mid), false + + case pos <= p.hi: + // mid-hi + return pos - (p.mid - p.lo), false + + default: + // after + } + + return pos, false +} + +func (p *priorOpFlip) Delta() (delta int) { + return 0 +} diff --git a/server/pkg/model/raw/prior_test.go b/server/pkg/model/raw/prior_test.go new file mode 100644 index 0000000..ee67241 --- /dev/null +++ b/server/pkg/model/raw/prior_test.go @@ -0,0 +1,57 @@ +package raw + +import ( + "testing" +) + +func TestPriorFlip(t *testing.T) { + p := &priorOpFlip{lo: 2, mid: 5, hi: 10} + + tests := []struct { + pos int + want int + }{ + {0, 0}, + {1, 1}, + {2, 2}, + {3, 8}, + {4, 9}, + {5, 10}, + {6, 3}, + {7, 4}, + {8, 5}, + {9, 6}, + {10, 7}, + {11, 11}, + } + + for _, tt := range tests { + got, _ := p.TransformPos(tt.pos) + if got != tt.want { + t.Errorf("TransformPos(%d) = %d, want %d", tt.pos, got, tt.want) + } + } +} + +func TestPriorRemove(t *testing.T) { + p := &priorOpRemove{lo: 2, hi: 5} + tests := []struct { + pos int + want int + wantGone bool + }{ + {0, 0, false}, + {1, 1, false}, + {2, 2, false}, + {3, 2, true}, + {4, 2, true}, + {5, 2, true}, + {6, 3, false}, + } + for _, tt := range tests { + got, gotGone := p.TransformPos(tt.pos) + if got != tt.want || gotGone != tt.wantGone { + t.Errorf("TransformPos(%d) = (%d, %v), want (%d, %v)", tt.pos, got, gotGone, tt.want, tt.wantGone) + } + } +} From 1f381353190a01117f6c3272af3b21e24331dddd Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Fri, 20 Mar 2026 19:04:44 +1100 Subject: [PATCH 02/22] node and raw fixes --- server/pkg/model/doc/const.go | 13 ++ server/pkg/model/doc/doc.go | 182 ++++++++++++++++++++++++++++ server/pkg/model/doc/types.go | 37 ++++++ server/pkg/model/node/const.go | 10 ++ server/pkg/model/node/node.go | 151 +++++++++++++++++------ server/pkg/model/node/node_test.go | 58 +++++++++ server/pkg/model/node/patch.go | 14 +++ server/pkg/model/raw/bigint_test.go | 38 ++++++ server/pkg/model/raw/gumnut_test.go | 160 ++++++++++++++++++++++++ server/pkg/model/raw/merge.go | 90 ++++++++++++-- server/pkg/model/raw/merge_test.go | 97 +++++++++++++++ server/pkg/model/raw/op_test.go | 141 +++++++++++++++++++++ server/pkg/model/raw/prior.go | 32 ----- server/pkg/model/raw/prior_test.go | 84 +++++++++++++ 14 files changed, 1030 insertions(+), 77 deletions(-) create mode 100644 server/pkg/model/doc/const.go create mode 100644 server/pkg/model/doc/doc.go create mode 100644 server/pkg/model/doc/types.go create mode 100644 server/pkg/model/node/const.go create mode 100644 server/pkg/model/node/node_test.go create mode 100644 server/pkg/model/node/patch.go create mode 100644 server/pkg/model/raw/bigint_test.go create mode 100644 server/pkg/model/raw/gumnut_test.go create mode 100644 server/pkg/model/raw/op_test.go diff --git a/server/pkg/model/doc/const.go b/server/pkg/model/doc/const.go new file mode 100644 index 0000000..4852230 --- /dev/null +++ b/server/pkg/model/doc/const.go @@ -0,0 +1,13 @@ +package doc + +import ( + "errors" +) + +var ( + ErrIno = errors.New("bad ino") + ErrVersion = errors.New("bad target version") + ErrRunOp = errors.New("bad run op, couldn't apply (bad transform?)") + ErrSetOp = errors.New("bad set op, couldn't transform") + ErrRead = errors.New("internal error preparing data") +) diff --git a/server/pkg/model/doc/doc.go b/server/pkg/model/doc/doc.go new file mode 100644 index 0000000..914a7ab --- /dev/null +++ b/server/pkg/model/doc/doc.go @@ -0,0 +1,182 @@ +package doc + +import ( + "github.com/gumnutdev/foiled/server/pkg/model/node" +) + +func NewDoc() (d Doc) { + return &docImpl{ + byIno: map[string]*node.Node{}, + active: map[int]int{}, + } +} + +type docImpl struct { + version int // head version (will be max of byIno version) + byIno map[string]*node.Node // core inode map + active map[int]int // handle ID to last version ID +} + +// lazyHolder fetches the nodeHolder for the given string ino. +// This may be nil for an invalid string type. +func (d *docImpl) lazyHolder(ino string) (nh *node.Node) { + if ino == "" { + return nil + } + + nh = d.byIno[ino] + if nh == nil { + nh = &node.Node{} + d.byIno[ino] = nh + } + return +} + +func (d *docImpl) Apply(work *DocWork) (err error) { + // ensure not too far ahead + if work.TargetVersion > d.version { + return ErrVersion + } + + // fix state for handle'd users + if work.Handle > 0 { + lastActive := d.active[work.Handle] + if work.TargetVersion < lastActive { + err = ErrVersion + return // for some reason the user is targeting in the past for them + } + } + + arg := node.PrepareArg{ + Handle: work.Handle, + TargetVersion: work.TargetVersion, + NewVersion: d.version + 1, + } + out := make(map[string]*node.PrepareToken, len(work.Update)) + + // transform and check all patches + for ino, patch := range work.Update { + nh := d.lazyHolder(ino) + if nh == nil { + return ErrIno + } + + res, err := nh.Prepare(arg, patch) + if err != nil { + return err + } else if res != nil { + out[ino] = res + } + } + + // --- at this point we are going to apply or panic --- + + // record last active version for this handle + if work.Handle > 0 { + handleVersion := work.TargetVersion + if handleVersion <= 0 { + handleVersion = d.version // handle doesn't know about anything before this + } + d.active[work.Handle] = handleVersion + } + + // for some reason we got a no-op + if len(out) == 0 { + work.TargetVersion = d.version + work.Update = nil + return + } + + // set up DocWork as our output location + d.version = arg.NewVersion + work.TargetVersion = d.version + work.Update = make(map[string]node.Patch, len(out)) + + // if all have passed, blat history + for ino, res := range out { + nh := d.byIno[ino] + work.Update[ino] = nh.Enact(res) + } + + return nil +} + +func (d *docImpl) Version() (version int) { + return d.version +} + +func (d *docImpl) Read() (dw *DocWork) { + dw = &DocWork{ + TargetVersion: d.version, + Update: make(map[string]node.Patch, len(d.byIno)), + } + + for ino, nh := range d.byIno { + set := nh.Read(d.version) + if len(set) == 0 { + continue + } + dw.Update[ino] = node.Patch{Set: set} + } + + return +} + +// inoToVersion returns the high version for the specific inode. +// This is for the caller to check whether work can be applied (check barrier). +func (d *docImpl) inoToVersion(ino string) (version int) { + nh := d.byIno[ino] + if nh == nil { + return 0 + } + _, version = nh.VersionRange() + return +} + +func (d *docImpl) Barrier(targetVersion int, inos []string) (ok bool) { + if targetVersion <= 0 { + return true + } + for _, ino := range inos { + if d.inoToVersion(ino) > targetVersion { + return false + } + } + return true +} + +func (d *docImpl) ClearHandle(handleID int) (change bool) { + _, ok := d.active[handleID] + if !ok { + return false + } + delete(d.active, handleID) + + for _, nh := range d.byIno { + local := nh.ClearHandle(handleID) + change = change || local + } + return change +} + +// func (d *docImpl) moveForward(gd *GumnutData, from, handle int) { +// if gd.Type != DataTypePosRef { +// return +// } + +// if gd.Data == 0 { +// return // can't transform, already at zero +// } + +// pos := int(gd.Data) + +// hist := d.byIno[gd.String].historyFrom(from, handle) +// pos, _ = ymodel.TransformOver(pos, hist) + +// if pos < 0 { +// // FIXME: the client gave us bad data; just reset to zero for other clients +// pos = 0 +// } + +// gd.Data = uint64(pos) +// } diff --git a/server/pkg/model/doc/types.go b/server/pkg/model/doc/types.go new file mode 100644 index 0000000..918ac0d --- /dev/null +++ b/server/pkg/model/doc/types.go @@ -0,0 +1,37 @@ +package doc + +import ( + "github.com/gumnutdev/foiled/server/pkg/model/node" +) + +type Doc interface { + // Apply transforms and applies the given DocWork here. + // This checks that all work can be applied before making changes. + // If an error is returned here, the Doc will be unchanged (but the DocWork may be). + // + // On a non-error return, the DocWork will contain ops ready-for-broadcast. + // The map entries will match the input, but may not have length. + // + // This does not care about barrier ops, the caller must fail before this based on its own version checks. + Apply(work *DocWork) (err error) + + // Version returns the high version here. + Version() (version int) + + // Read reads the current status of this Doc. + // Internally, this may apply ops forward: we don't always keep a "live" copy around. + Read() (dw *DocWork) + + // Barrier returns true if the given client can pass the barrier with the given inos and their known target version. + // This is effectively checking all the target inos have not changed since ... + Barrier(targetVersion int, inos []string) (ok bool) + + // ClearHandle removes the given handle from known history. + ClearHandle(handleID int) (change bool) +} + +type DocWork struct { + TargetVersion int // target version of doc - transform > this (zero => no transform) + Handle int // globally unique author of work + Update map[string]node.Patch // mods to be made +} diff --git a/server/pkg/model/node/const.go b/server/pkg/model/node/const.go new file mode 100644 index 0000000..01ab389 --- /dev/null +++ b/server/pkg/model/node/const.go @@ -0,0 +1,10 @@ +package node + +import ( + "errors" +) + +var ( + ErrNodeVersion = errors.New("bad node target version") + ErrTransform = errors.New("couldn't transform all -ve refs") +) diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index 9029166..491d87f 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -1,6 +1,7 @@ package node import ( + "log" "sort" "github.com/gumnutdev/foiled/server/pkg/model/raw" @@ -17,26 +18,16 @@ import ( // // It knows about the handle who performed history operations, so it is exposed to end-clients. type Node struct { - set SetMap // data here - readAt int // when we last read/flattened data + set raw.SetMap // data here + readAt int // when we last read/flattened data history []historyEntry -} - -type SetMap map[string]raw.GumnutSetArray -func (sm *SetMap) tidy() { - for key, arr := range *sm { - if len(arr) == 0 { - delete(*sm, key) - } - } + Transform raw.TransformFunc // must be set to xform data } type historyEntry struct { - run []raw.PriorOp // how does this xform our structure - set SetMap // what to set after xform - handle int // author (zero for server/none) - version int // version this creates + entry raw.SetEntry // how does this modify us + handle int // author (zero for server/none) } // ClearHandle removes all known use of the given +ve handle here. @@ -56,39 +47,131 @@ func (n *Node) ClearHandle(handle int) (change bool) { return } -func (n *Node) Read(docVersion int) (sm SetMap) { +func (n *Node) VersionRange() (lo, hi int) { + if len(n.history) != 0 { + lo = n.history[0].entry.Version + hi = n.history[len(n.history)-1].entry.Version + } + return +} + +// Read reads the content of this node. +// This will transform data forward as needed. +func (n *Node) Read(docVersion int) (sm raw.SetMap) { if n.readAt >= docVersion { return n.set } idx := sort.Search(len(n.history), func(i int) (match bool) { - return n.history[i].version > n.readAt + return n.history[i].entry.Version > n.readAt }) tail := n.history[idx:] - // FIXME: xform current data - can't distinguish from _new_ data + prep := make([]raw.SetEntry, len(tail)) + for i, he := range tail { + prep[i] = he.entry + } + + log.Printf("got prep: %+v", prep) // FIXME + + n.set = raw.MoveForward(prep, n.Transform) + if n.set == nil { + panic("got out of order versions in history") + } + + n.readAt = docVersion + return n.set +} + +// PrepareArg just contains some values for Prepare. +type PrepareArg struct { + Handle int + TargetVersion int + NewVersion int +} + +// PrepareToken is an opaque result token passed between Prepare and Enact. +type PrepareToken struct { + patch Patch + he historyEntry +} + +// Prepare accepts the user's work on this node and confirms it is valid. +// It returns a PrepareToken which can be later passed to Enact. +// The PrepareToken can be nil if this is a no-op. +func (n *Node) Prepare(arg PrepareArg, p Patch) (tok *PrepareToken, err error) { + lo, hi := n.VersionRange() + + if len(n.history) != 0 { + if arg.TargetVersion > 0 && arg.TargetVersion < lo { + return nil, ErrNodeVersion + } + } + if arg.NewVersion <= max(hi, n.readAt) { + return nil, ErrNodeVersion + } + + hist := n.historyFrom(arg.TargetVersion, arg.Handle) + he := historyEntry{ + entry: raw.SetEntry{Version: arg.NewVersion}, + handle: arg.Handle, + } + + var out Patch + + // ensure that all of Run is valid + for _, op := range p.Run { + op.TransformOver(hist) - // apply self structural changes - for _, entry := range tail { - pops := make([]raw.PriorOpSelf, len(entry.run)) - for i, each := range entry.run { - pops[i].PriorOp = each + prior, ok := op.Apply() + if !ok { + return nil, ErrTransform + } else if prior == nil { + continue } - for key, arr := range entry.set { - if !arr.Merge(pops) { - panic("couldn't merge in read()") - } - entry.set[key] = arr + hist = append(hist, raw.PriorOpSelf{PriorOp: prior, Self: true}) + he.entry.Prior = append(he.entry.Prior, prior) + out.Run = append(out.Run, op) + } + + // leverage MoveForward to xform so sets are normalized + + set := raw.TransformSetOver(p.Set, hist) + if set == nil { + return nil, ErrTransform + } + out.Set = set + he.entry.Set = out.Set + + if len(he.entry.Prior) == 0 && len(he.entry.Set) == 0 { + return + } + return &PrepareToken{out, he}, nil +} + +// Enact can enact a previous PrepareToken. +func (n *Node) Enact(tok *PrepareToken) (out Patch) { + n.history = append(n.history, tok.he) + return tok.patch +} + +func (n *Node) historyFrom(version, handle int) (out []raw.PriorOpSelf) { + // look for the first entry that is > the known version + idx := sort.Search(len(n.history), func(i int) (match bool) { + return n.history[i].entry.Version > version + }) + + for _, entry := range n.history[idx:] { + self := handle > 0 && handle == entry.handle + if version < 0 && !self { + continue // can only xform against self anyway } - // FIXME: xform new data to end - for key, extra := range entry.set { - n.set[key] = append(n.set[key], extra...) + for _, pop := range entry.entry.Prior { + out = append(out, raw.PriorOpSelf{PriorOp: pop, Self: self}) } } - n.set.tidy() // remove blank - n.readAt = docVersion - return n.set + return } diff --git a/server/pkg/model/node/node_test.go b/server/pkg/model/node/node_test.go new file mode 100644 index 0000000..6238961 --- /dev/null +++ b/server/pkg/model/node/node_test.go @@ -0,0 +1,58 @@ +package node + +import ( + "reflect" + "testing" + + "github.com/gumnutdev/foiled/server/pkg/jsonhelp" + "github.com/gumnutdev/foiled/server/pkg/model/raw" +) + +func TestNode(t *testing.T) { + n := &Node{} + + mustEnact := func(handle, targetVersion int, patch Patch) (out Patch) { + _, hi := n.VersionRange() + + tok, err := n.Prepare(PrepareArg{ + Handle: handle, + TargetVersion: targetVersion, + NewVersion: hi + 1, + }, patch) + if err != nil { + t.Fatalf("couldn't prepare: %+v", err) + } + if tok != nil { + return n.Enact(tok) + } + return + } + + mustEnact(1, 0, Patch{ + Set: raw.SetMap{ + "": []raw.GumnutSet{ + {Before: 10, String: jsonhelp.EncodeToUTF16("hello")}, + }, + }, + }) + + mustEnact(1, 0, Patch{ + Run: []raw.Op{ + {Range: []int{10}, Length: 6}, + }, + Set: raw.SetMap{ + "": []raw.GumnutSet{ + {Before: -1, String: jsonhelp.EncodeToUTF16(" there")}, + }, + }, + }) + + out := n.Read(2) + if !reflect.DeepEqual(out, raw.SetMap{ + "": []raw.GumnutSet{ + {Before: 16, String: jsonhelp.EncodeToUTF16("hello there")}, + }, + }) { + t.Errorf("got bad state: %+v", out) + } +} diff --git a/server/pkg/model/node/patch.go b/server/pkg/model/node/patch.go new file mode 100644 index 0000000..1e47039 --- /dev/null +++ b/server/pkg/model/node/patch.go @@ -0,0 +1,14 @@ +package node + +import ( + "github.com/gumnutdev/foiled/server/pkg/model/raw" +) + +type Patch struct { + Run []raw.Op // structural changes + Set raw.SetMap // sets per keyed list +} + +func (p *Patch) IsZero() (is bool) { + return len(p.Run) == 0 && len(p.Set) == 0 +} diff --git a/server/pkg/model/raw/bigint_test.go b/server/pkg/model/raw/bigint_test.go new file mode 100644 index 0000000..bea0b44 --- /dev/null +++ b/server/pkg/model/raw/bigint_test.go @@ -0,0 +1,38 @@ +package raw + +import ( + "testing" +) + +func TestIsBigInt(t *testing.T) { + cases := []struct { + name string + in string + out bool + }{ + {"empty", "", false}, + {"zero", "0", true}, + {"single digit", "5", true}, + {"multiple digits", "1234567890", true}, + {"negative zero", "-0", false}, + {"negative", "-123", true}, + {"negative single digit", "-9", true}, + {"leading zero", "0123", false}, + {"negative leading zero", "-0123", false}, + {"letters", "123a", false}, + {"all letters", "abc", false}, + {"just dash", "-", false}, + {"dash inside", "12-3", false}, + {"spaces", " 123", false}, + {"float", "123.4", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := isBigInt(tc.in) + if got != tc.out { + t.Errorf("isBigInt(%q) = %v, want %v", tc.in, got, tc.out) + } + }) + } +} diff --git a/server/pkg/model/raw/gumnut_test.go b/server/pkg/model/raw/gumnut_test.go new file mode 100644 index 0000000..0bdc5e9 --- /dev/null +++ b/server/pkg/model/raw/gumnut_test.go @@ -0,0 +1,160 @@ +package raw + +import ( + "encoding/json" + "math" + "testing" +) + +func TestGumnutData_IsZero(t *testing.T) { + gd1 := GumnutData{Type: DataTypeEmpty} + if !gd1.IsZero() { + t.Errorf("expected IsZero to be true for DataTypeEmpty") + } + + gd2 := GumnutData{Type: DataTypeNumber, Data: 0} + if gd2.IsZero() { + t.Errorf("expected IsZero to be false for DataTypeNumber") + } +} + +func TestGumnutData_JSRune(t *testing.T) { + gd := GumnutData{Type: DataTypeNumber, Data: math.Float64bits(65)} // 'A' + r, ok := gd.JSRune() + if !ok || r != 65 { + t.Errorf("expected r=65, ok=true; got r=%d, ok=%v", r, ok) + } + + gd2 := GumnutData{Type: DataTypeNumber, Data: math.Float64bits(65.5)} + _, ok2 := gd2.JSRune() + if ok2 { + t.Errorf("expected ok=false for non-integer number") + } + + gd3 := GumnutData{Type: DataTypeString, String: "A"} + _, ok3 := gd3.JSRune() + if ok3 { + t.Errorf("expected ok=false for non-number type") + } + + gd4 := GumnutData{Type: DataTypeNumber, Data: math.Float64bits(float64(math.MaxUint16) + 1)} + _, ok4 := gd4.JSRune() + if ok4 { + t.Errorf("expected ok=false for out-of-range uint16 value") + } +} + +func TestGumnutData_JSON(t *testing.T) { + cases := []struct { + name string + in GumnutData + expected string // Expected JSON string + }{ + { + "number", + GumnutData{Type: DataTypeNumber, Data: math.Float64bits(42.5)}, + `42.5`, + }, + { + "number int", + GumnutData{Type: DataTypeNumber, Data: math.Float64bits(100)}, + `100`, + }, + { + "positive infinity", + GumnutData{Type: DataTypeNumber, Data: math.Float64bits(math.Inf(1))}, + `{"inf":1}`, + }, + { + "negative infinity", + GumnutData{Type: DataTypeNumber, Data: math.Float64bits(math.Inf(-1))}, + `{"inf":-1}`, + }, + { + "NaN", + GumnutData{Type: DataTypeNumber, Data: math.Float64bits(math.NaN())}, + `{"inf":0}`, + }, + { + "bigint", + GumnutData{Type: DataTypeBigInt, String: "12345678901234567890"}, + `{"b":"12345678901234567890"}`, + }, + { + "known null", + GumnutData{Type: DataTypeKnown, Data: 1, String: "null"}, + `null`, + }, + { + "known true", + GumnutData{Type: DataTypeKnown, Data: 1, String: "true"}, + `true`, + }, + { + "known false", + GumnutData{Type: DataTypeKnown, Data: 1, String: "false"}, + `false`, + }, + { + "known other", + GumnutData{Type: DataTypeKnown, String: "undefined"}, + `{"x":"undefined"}`, + }, + { + "string", + GumnutData{Type: DataTypeString, String: "hello"}, + `{"s":"hello"}`, + }, + { + "posref", + GumnutData{Type: DataTypePosRef, String: "node123", Data: 456}, + `{"i":"node123","p":456}`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b, err := json.Marshal(tc.in) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + if string(b) != tc.expected { + t.Errorf("got %s, want %s", string(b), tc.expected) + } + + // Empty cannot be encoded in base case, so no roundtrip testing for it + // Similarly, test if unmarshaling returns the same type and data + var out GumnutData + err = json.Unmarshal(b, &out) + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + if tc.in.Type != out.Type || tc.in.String != out.String || tc.in.Data != out.Data { + t.Errorf("roundtrip mismatch: in=%+v, out=%+v", tc.in, out) + } + }) + } +} + +func TestGumnutData_JSONUnmarshal_Errors(t *testing.T) { + cases := []struct { + name string + in string + }{ + {"empty string", `""`}, + {"invalid bigint", `{"b":"123a"}`}, + {"unknown type object", `{"unknown":true}`}, + {"malformed json", `{`}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var out GumnutData + err := json.Unmarshal([]byte(tc.in), &out) + if err == nil { + t.Errorf("expected error for input %q, got nil", tc.in) + } + }) + } +} diff --git a/server/pkg/model/raw/merge.go b/server/pkg/model/raw/merge.go index bc7d0d4..8c9afba 100644 --- a/server/pkg/model/raw/merge.go +++ b/server/pkg/model/raw/merge.go @@ -2,7 +2,6 @@ package raw import ( "iter" - "log" "slices" ) @@ -25,6 +24,15 @@ func (s GumnutSetArray) IterData() (it iter.Seq[*GumnutData]) { // SetMap is a map from key to GumnutSetArray. type SetMap map[string]GumnutSetArray +// Tidy removes superfluous items from SetMap. +func (sm *SetMap) Tidy() { + for key, arr := range *sm { + if len(arr) == 0 { + delete(*sm, key) + } + } +} + // SetEntry describes how to modify a SetMap over time. // The first SetEntry in any array is effectively the base state. type SetEntry struct { @@ -35,13 +43,13 @@ type SetEntry struct { type TransformFunc func(fromVersion int, gd *GumnutData) -type setEntry struct { +type sliceSet struct { version int str bool // was this encoded as String gd GumnutData } -func (se *setEntry) isStr() (is bool) { +func (se *sliceSet) isStr() (is bool) { is = se.str if is { _, is = se.gd.JSRune() @@ -49,7 +57,7 @@ func (se *setEntry) isStr() (is bool) { return } -// MoveForward helps to move sets forward over structural ops. +// MoveForward helps to move already "normalized" sets forward over structural ops. // This transforms it structurally and wrt its data. // The SetEntry array must have an incrementing version number. func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { @@ -64,7 +72,7 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { version = e.Version } - work := make(map[string][]setEntry) + work := make(map[string][]sliceSet) priors := []PriorOp{} // work backwards @@ -83,7 +91,6 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { pos := target if len(priors) > 0 { pos = transformOverLocal(target, priors) - log.Printf("transformed pos=%v -> %v", target, pos) } target-- @@ -97,18 +104,15 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { // off end, always exists slice = extendSlice(slice, pos) work[key] = slice - slice[index] = setEntry{each.Version, str, gd} + slice[index] = sliceSet{each.Version, str, gd} } else if slice[index].version == 0 { // check nothing already here - slice[index] = setEntry{each.Version, str, gd} + slice[index] = sliceSet{each.Version, str, gd} } else { continue } - - r, _ := gd.JSRune() - log.Printf("ver=%v placing char=%c at index=%+v", each.Version, r, index) } } } @@ -194,3 +198,67 @@ func transformOverLocal(v int, over []PriorOp) (out int) { return v } + +func TransformSetOver(set SetMap, prior []PriorOpSelf) (out SetMap) { + acc := SetMap{} + + // accumulate into single entries + for key, arr := range set { + var tmp []GumnutSet + + for _, x := range arr { + + targets := make([]int, x.length()) + for i := range targets { + pos := x.Before - len(targets) + i + 1 + + var gone bool + pos, gone = TransformOver(pos, prior) + if gone { + pos = 0 + } + if pos < 0 { + return nil + // log.Printf("couldn't transform (%v -> %v) over=%+v", was, pos, prior) + // for _, p := range prior { + // log.Printf("prior=%+v", p.PriorOp) + // } + // panic("BAD POS") + } + + targets[i] = pos + } + + if x.String != nil { + for i, r := range x.String { + pos := targets[i] + if pos == 0 { + continue + } + tmp = append(tmp, GumnutSet{Before: pos, String: []uint16{r}}) + } + continue + } + + for i, r := range x.Data { + pos := targets[i] + if pos == 0 { + continue + } + tmp = append(tmp, GumnutSet{Before: pos, Data: []GumnutData{r}}) + } + continue + } + + acc[key] = tmp + } + + // run regular MoveForward ... + + return MoveForward([]SetEntry{ + { + Version: 1, + Set: acc, + }, + }, nil) +} diff --git a/server/pkg/model/raw/merge_test.go b/server/pkg/model/raw/merge_test.go index 39a2da4..5db91fe 100644 --- a/server/pkg/model/raw/merge_test.go +++ b/server/pkg/model/raw/merge_test.go @@ -73,6 +73,73 @@ func TestMoveForward(t *testing.T) { }, nil, }, + { + "multiple_versions", + []SetEntry{ + { + Version: 1, + Set: SetMap{ + "foo": GumnutSetArray{ + {Before: 5, String: jsonhelp.EncodeToUTF16("ABCD")}, + }, + }, + }, + { + Version: 2, + Set: SetMap{ + "foo": GumnutSetArray{ + {Before: 4, String: jsonhelp.EncodeToUTF16("x")}, + }, + }, + }, + }, + SetMap{ + "foo": GumnutSetArray{ + {Before: 5, String: jsonhelp.EncodeToUTF16("ABxD")}, + }, + }, + nil, + }, + { + "transform_func", + []SetEntry{ + { + Version: 1, + Set: SetMap{ + "bar": GumnutSetArray{ + {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 100}}}, + }, + }, + }, + }, + SetMap{ + "bar": GumnutSetArray{ + {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 200}}}, + }, + }, + func(ver int, gd *GumnutData) { + if gd.Type == DataTypeNumber { + gd.Data += 100 + } + }, + }, + { + "multiple_keys", + []SetEntry{ + { + Version: 1, + Set: SetMap{ + "k1": GumnutSetArray{{Before: 1, String: jsonhelp.EncodeToUTF16("1")}}, + "k2": GumnutSetArray{{Before: 2, String: jsonhelp.EncodeToUTF16("22")}}, + }, + }, + }, + SetMap{ + "k1": GumnutSetArray{{Before: 1, String: jsonhelp.EncodeToUTF16("1")}}, + "k2": GumnutSetArray{{Before: 2, String: jsonhelp.EncodeToUTF16("22")}}, + }, + nil, + }, } for _, each := range tests { @@ -84,3 +151,33 @@ func TestMoveForward(t *testing.T) { }) } } + +func TestIterData(t *testing.T) { + arr := GumnutSetArray{ + {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 42}, {Type: DataTypeNumber, Data: 43}}}, + {Before: 20, String: []uint16{'a', 'b'}}, + {Before: 30, Data: []GumnutData{{Type: DataTypeString, String: "foo"}}}, + } + + var got []GumnutData + for gd := range arr.IterData() { + got = append(got, *gd) + } + + if len(got) != 3 { + t.Fatalf("expected 3 data items, got %d", len(got)) + } + if got[0].Data != 42 || got[1].Data != 43 || got[2].String != "foo" { + t.Errorf("unexpected elements in iter: %+v", got) + } + + // test early exit + count := 0 + for range arr.IterData() { + count++ + break + } + if count != 1 { + t.Errorf("expected early exit to yield 1 item, got %d", count) + } +} diff --git a/server/pkg/model/raw/op_test.go b/server/pkg/model/raw/op_test.go new file mode 100644 index 0000000..6b85955 --- /dev/null +++ b/server/pkg/model/raw/op_test.go @@ -0,0 +1,141 @@ +package raw + +import ( + "reflect" + "testing" +) + +func TestOp_Apply(t *testing.T) { + cases := []struct { + name string + op Op + wantPrior PriorOp + wantOk bool + }{ + { + "negative range invalid", + Op{Range: []int{-1, 5}}, + nil, + false, + }, + { + "zero elements invalid length", + Op{Range: []int{}}, + nil, + false, + }, + { + "1 element zero length is no-op", + Op{Range: []int{5}, Length: 0}, + nil, + true, + }, + { + "1 element insert", + Op{Range: []int{5}, Length: 3}, + &priorOpInsert{after: 5, length: 3}, + true, + }, + { + "2 element remove identical is no-op", + Op{Range: []int{5, 5}}, + nil, + true, + }, + { + "2 element remove", + Op{Range: []int{5, 10}}, + &priorOpRemove{lo: 5, hi: 10}, + true, + }, + { + "2 element remove unordered", + Op{Range: []int{10, 5}}, + &priorOpRemove{lo: 5, hi: 10}, + true, + }, + { + "3 element flip identical is no-op", + Op{Range: []int{5, 5, 10}}, + nil, + true, + }, + { + "3 element flip", + Op{Range: []int{2, 5, 10}}, + &priorOpFlip{lo: 2, mid: 5, hi: 10}, + true, + }, + { + "3 element flip unordered", + Op{Range: []int{10, 2, 5}}, + &priorOpFlip{lo: 2, mid: 5, hi: 10}, + true, + }, + { + "4 element invalid length", + Op{Range: []int{1, 2, 3, 4}}, + nil, + false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + gotPrior, gotOk := tc.op.Apply() + if gotOk != tc.wantOk { + t.Errorf("Apply() gotOk = %v, want %v", gotOk, tc.wantOk) + } + if !reflect.DeepEqual(gotPrior, tc.wantPrior) { + t.Errorf("Apply() gotPrior = %#v, want %#v", gotPrior, tc.wantPrior) + } + }) + } +} + +func TestOp_TransformOver(t *testing.T) { + // op starting with Range [3] being transformed over an insert at 1 of length 2 + op := Op{Range: []int{3}, Length: 1} + + prior := []PriorOpSelf{ + {PriorOp: &priorOpInsert{after: 1, length: 2}}, + } + + op.TransformOver(prior) + + if len(op.Range) != 1 || op.Range[0] != 5 { + t.Errorf("expected Range [5], got %v", op.Range) + } + if op.Length != 1 { + t.Errorf("expected Length 1, got %v", op.Length) + } + + // op starting with Range [3] being transformed over a remove covering 3 + op2 := Op{Range: []int{3}, Length: 1} + prior2 := []PriorOpSelf{ + {PriorOp: &priorOpRemove{lo: 2, hi: 5}}, + } + op2.TransformOver(prior2) + + if len(op2.Range) != 1 || op2.Range[0] != 2 { + t.Errorf("expected Range [2] due to remove collapsing point to start, got %v", op2.Range) + } + if op2.Length != 0 { + t.Errorf("expected Length 0 (nuked because gone), got %v", op2.Length) + } +} + +func TestOp_IsEmpty(t *testing.T) { + if !(&Op{Range: []int{5}, Length: 0}).IsEmpty() { + t.Errorf("expected true for empty insert") + } + if (&Op{Range: []int{5}, Length: 2}).IsEmpty() { + t.Errorf("expected false for active insert") + } + if !(&Op{Range: []int{5, 5}}).IsEmpty() { + t.Errorf("expected true for empty remove") + } + if (&Op{Range: []int{}}).IsEmpty() { + t.Errorf("expected false for zero length range (invalid op)") + } +} diff --git a/server/pkg/model/raw/prior.go b/server/pkg/model/raw/prior.go index 4133a60..ebff213 100644 --- a/server/pkg/model/raw/prior.go +++ b/server/pkg/model/raw/prior.go @@ -51,38 +51,6 @@ type PriorOp interface { Delta() (delta int) } -// ApplyPriorOpSlice applies this PriorOp to a generic slice. -func ApplyPriorOpSlice[X any](target *[]X, pop PriorOp) { - switch pop := pop.(type) { - case *priorOpInsert: - if len(*target) <= pop.after { - return - } - - tmp := (*target)[:pop.after:pop.after] - tmp = append(tmp, make([]X, pop.length)...) - tmp = append(tmp, (*target)[pop.after:]...) - *target = tmp - - case *priorOpRemove: - if len(*target) <= pop.lo { - return - } - - if len(*target) <= pop.hi { - *target = (*target)[:pop.lo] - return - } - - tmp := (*target)[:pop.lo:pop.lo] - tmp = append(tmp, (*target)[pop.hi:]...) - *target = tmp - - case *priorOpFlip: - panic("TODO: implement flip") - } -} - // ResolveOp is run backwards, and is used to match -ve IDs for user focus. // Note that pos is 1-indexed, or, likely, -ve indexed when passed here (-1 is in front of last user data inserted). type ResolveOp interface { diff --git a/server/pkg/model/raw/prior_test.go b/server/pkg/model/raw/prior_test.go index ee67241..d25e612 100644 --- a/server/pkg/model/raw/prior_test.go +++ b/server/pkg/model/raw/prior_test.go @@ -55,3 +55,87 @@ func TestPriorRemove(t *testing.T) { } } } + +func TestPriorInsert(t *testing.T) { + p := &priorOpInsert{after: 2, length: 3} + tests := []struct { + pos int + want int + wantGone bool + }{ + {0, 0, false}, + {1, 1, false}, + {2, 2, false}, + {3, 6, false}, + {4, 7, false}, + } + for _, tt := range tests { + got, gotGone := p.TransformPos(tt.pos) + if got != tt.want || gotGone != tt.wantGone { + t.Errorf("TransformPos(%d) = (%d, %v), want (%d, %v)", tt.pos, got, gotGone, tt.want, tt.wantGone) + } + } + + if p.Delta() != 3 { + t.Errorf("Delta() = %d, want 3", p.Delta()) + } + + // ResolvePos + resolveTests := []struct { + pos int + want int + }{ + {5, 5}, + {-1, 5}, + {-2, 4}, + {-3, 3}, + {-4, -1}, // shifted because length is 3, so -4 was at -1 before insert + {-5, -2}, + } + for _, tt := range resolveTests { + got := p.ResolvePos(tt.pos) + if got != tt.want { + t.Errorf("ResolvePos(%d) = %d, want %d", tt.pos, got, tt.want) + } + } +} + +func TestTransformOver_NegativePos(t *testing.T) { + // A negative pos resolves forwards based on self-inserts. + prior := []PriorOpSelf{ + {PriorOp: &priorOpInsert{after: 10, length: 2}, Self: true}, + } + + got, gone := TransformOver(-2, prior) // inside the length 2 insert: -( (-2) + 1 ) = 1 < 2 => 10+2-1 = 11 + if gone || got != 11 { + t.Errorf("TransformOver(-2) = (%d, %v), want (11, false)", got, gone) + } + + got2, gone2 := TransformOver(-3, prior) // past the length 2 insert, into before: -3+2 = -1 + if gone2 || got2 != -1 { + t.Errorf("TransformOver(-3) = (%d, %v), want (-1, false)", got2, gone2) + } +} + +func TestTransformOver_PositivePos(t *testing.T) { + prior := []PriorOpSelf{ + {PriorOp: &priorOpInsert{after: 5, length: 3}, Self: false}, + {PriorOp: &priorOpRemove{lo: 2, hi: 10}, Self: false}, + } + + // pos 8: + // after insert: 8 + 3 = 11 + // after remove: 11 - (10 - 2) = 11 - 8 = 3 + got, gone := TransformOver(8, prior) + if gone || got != 3 { + t.Errorf("TransformOver(8) = (%d, %v), want (3, false)", got, gone) + } + + // pos 4: + // after insert: 4 (before insert) + // after remove: falls inside remove 2..10 -> out=2, gone=true + got2, gone2 := TransformOver(4, prior) + if !gone2 || got2 != 2 { + t.Errorf("TransformOver(4) = (%d, %v), want (2, true)", got2, gone2) + } +} From 56ec3e934b8549c17e07b08ad15c821c70bcc496 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Fri, 20 Mar 2026 21:42:43 +1100 Subject: [PATCH 03/22] xform self including inoref --- server/pkg/model/doc/doc.go | 62 +++++++++------ server/pkg/model/node/const.go | 1 + server/pkg/model/node/node.go | 118 +++++++++++++++++++++++------ server/pkg/model/node/node_test.go | 20 ++++- server/pkg/model/raw/const.go | 10 +++ server/pkg/model/raw/gumnut.go | 5 -- server/pkg/model/raw/merge.go | 37 +++++---- server/pkg/model/raw/merge_test.go | 8 +- 8 files changed, 185 insertions(+), 76 deletions(-) create mode 100644 server/pkg/model/raw/const.go diff --git a/server/pkg/model/doc/doc.go b/server/pkg/model/doc/doc.go index 914a7ab..24707c7 100644 --- a/server/pkg/model/doc/doc.go +++ b/server/pkg/model/doc/doc.go @@ -2,6 +2,7 @@ package doc import ( "github.com/gumnutdev/foiled/server/pkg/model/node" + "github.com/gumnutdev/foiled/server/pkg/model/raw" ) func NewDoc() (d Doc) { @@ -26,7 +27,7 @@ func (d *docImpl) lazyHolder(ino string) (nh *node.Node) { nh = d.byIno[ino] if nh == nil { - nh = &node.Node{} + nh = &node.Node{Transform: d.moveForward} d.byIno[ino] = nh } return @@ -69,8 +70,6 @@ func (d *docImpl) Apply(work *DocWork) (err error) { } } - // --- at this point we are going to apply or panic --- - // record last active version for this handle if work.Handle > 0 { handleVersion := work.TargetVersion @@ -94,8 +93,22 @@ func (d *docImpl) Apply(work *DocWork) (err error) { // if all have passed, blat history for ino, res := range out { - nh := d.byIno[ino] - work.Update[ino] = nh.Enact(res) + err = res.Finalize() + if err != nil { + break + } + work.Update[ino] = res.Patch() + } + if err != nil { + if err == node.ErrFinalize { + // this is a structural error + panic("should not ErrFinalize") + } + // otherwise it is returned from TransformFunc + for _, res := range out { + res.Rollback() + } + return err } return nil @@ -159,24 +172,27 @@ func (d *docImpl) ClearHandle(handleID int) (change bool) { return change } -// func (d *docImpl) moveForward(gd *GumnutData, from, handle int) { -// if gd.Type != DataTypePosRef { -// return -// } - -// if gd.Data == 0 { -// return // can't transform, already at zero -// } - -// pos := int(gd.Data) +// moveForward is passed to all Node instances so they can transform DataTypePosRef. +func (d *docImpl) moveForward(handle, fromVersion int, gd *raw.GumnutData) (err error) { + if gd.Type != raw.DataTypePosRef { + return + } -// hist := d.byIno[gd.String].historyFrom(from, handle) -// pos, _ = ymodel.TransformOver(pos, hist) + if gd.Data == 0 { + return // can't transform, already at zero + } -// if pos < 0 { -// // FIXME: the client gave us bad data; just reset to zero for other clients -// pos = 0 -// } + pos := int(gd.Data) + node := d.byIno[gd.String] + if node == nil { + return // bad node + } -// gd.Data = uint64(pos) -// } + update, _ := node.TransformPos(handle, fromVersion, pos) + if update < 0 { + // we should only see this on _initial_ transform + return ErrSetOp + } + gd.Data = uint64(update) + return nil +} diff --git a/server/pkg/model/node/const.go b/server/pkg/model/node/const.go index 01ab389..7f455c2 100644 --- a/server/pkg/model/node/const.go +++ b/server/pkg/model/node/const.go @@ -7,4 +7,5 @@ import ( var ( ErrNodeVersion = errors.New("bad node target version") ErrTransform = errors.New("couldn't transform all -ve refs") + ErrFinalize = errors.New("couldn't finalize") ) diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index 491d87f..f6f03e9 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -1,7 +1,7 @@ package node import ( - "log" + "fmt" "sort" "github.com/gumnutdev/foiled/server/pkg/model/raw" @@ -22,9 +22,12 @@ type Node struct { readAt int // when we last read/flattened data history []historyEntry - Transform raw.TransformFunc // must be set to xform data + Transform TransformHandleFunc // must be set to xform data } +// TransformHandleFunc is set on Node to transform GumnutData forward. +type TransformHandleFunc func(handle, fromVersion int, gd *raw.GumnutData) (err error) + type historyEntry struct { entry raw.SetEntry // how does this modify us handle int // author (zero for server/none) @@ -58,6 +61,9 @@ func (n *Node) VersionRange() (lo, hi int) { // Read reads the content of this node. // This will transform data forward as needed. func (n *Node) Read(docVersion int) (sm raw.SetMap) { + _, hi := n.VersionRange() + docVersion = max(hi, docVersion) + if n.readAt >= docVersion { return n.set } @@ -72,11 +78,19 @@ func (n *Node) Read(docVersion int) (sm raw.SetMap) { prep[i] = he.entry } - log.Printf("got prep: %+v", prep) // FIXME + // setup transform fn + var fn raw.TransformFunc + if n.Transform != nil { + fn = func(fromVersion int, gd *raw.GumnutData) (err error) { + return n.Transform(0, fromVersion, gd) + } + } - n.set = raw.MoveForward(prep, n.Transform) - if n.set == nil { - panic("got out of order versions in history") + // read and move forward + var err error + n.set, err = raw.MoveForward(prep, fn) + if err != nil { + panic(fmt.Errorf("couldn't MoveForward on read: %+v", err)) } n.readAt = docVersion @@ -90,12 +104,6 @@ type PrepareArg struct { NewVersion int } -// PrepareToken is an opaque result token passed between Prepare and Enact. -type PrepareToken struct { - patch Patch - he historyEntry -} - // Prepare accepts the user's work on this node and confirms it is valid. // It returns a PrepareToken which can be later passed to Enact. // The PrepareToken can be nil if this is a no-op. @@ -116,7 +124,6 @@ func (n *Node) Prepare(arg PrepareArg, p Patch) (tok *PrepareToken, err error) { entry: raw.SetEntry{Version: arg.NewVersion}, handle: arg.Handle, } - var out Patch // ensure that all of Run is valid @@ -136,24 +143,82 @@ func (n *Node) Prepare(arg PrepareArg, p Patch) (tok *PrepareToken, err error) { } // leverage MoveForward to xform so sets are normalized + out.Set, err = raw.TransformSetOver(p.Set, hist) + if err != nil { + return nil, err + } - set := raw.TransformSetOver(p.Set, hist) - if set == nil { - return nil, ErrTransform + // is this a no-op? + if len(he.entry.Prior) == 0 && len(out.Set) == 0 { + return } - out.Set = set - he.entry.Set = out.Set - if len(he.entry.Prior) == 0 && len(he.entry.Set) == 0 { + // we actually enact this here, but it does not have its Set yet + n.history = append(n.history, he) + + return &PrepareToken{ + arg: arg, + patch: out, + node: n, + he: &n.history[len(n.history)-1], + }, nil +} + +// PrepareToken is an opaque result token passed between Prepare and Enact. +type PrepareToken struct { + arg PrepareArg + patch Patch + node *Node + he *historyEntry +} + +func (pt *PrepareToken) Patch() (p Patch) { + return pt.patch +} + +func (pt *PrepareToken) Finalize() (err error) { + if pt.node.lastHist() != pt.he { + return ErrFinalize // not top of stack + } + + if pt.he.entry.Set != nil { + return ErrFinalize // already finalized + } + + var tr raw.TransformFunc + if pt.node.Transform != nil { + tr = func(fromVersion int, gd *raw.GumnutData) (err error) { + return pt.node.Transform(pt.arg.Handle, fromVersion, gd) + } + } + + // transform xrefs forward + set, err := raw.MoveForward([]raw.SetEntry{ + {Version: pt.arg.TargetVersion, Set: pt.patch.Set}, + }, tr) + if err != nil { + return err + } + pt.he.entry.Set = set + + return nil +} + +// Rollback removes this work from the Node. +func (pt *PrepareToken) Rollback() (ok bool) { + if pt.node.lastHist() != pt.he { return } - return &PrepareToken{out, he}, nil + + pt.node.history = pt.node.history[:len(pt.node.history)-1] + return true } -// Enact can enact a previous PrepareToken. -func (n *Node) Enact(tok *PrepareToken) (out Patch) { - n.history = append(n.history, tok.he) - return tok.patch +func (n *Node) lastHist() (he *historyEntry) { + if len(n.history) != 0 { + return &n.history[len(n.history)-1] + } + return } func (n *Node) historyFrom(version, handle int) (out []raw.PriorOpSelf) { @@ -175,3 +240,8 @@ func (n *Node) historyFrom(version, handle int) (out []raw.PriorOpSelf) { return } + +func (n *Node) TransformPos(handle, fromVersion, pos int) (out int, gone bool) { + pop := n.historyFrom(fromVersion, handle) + return raw.TransformOver(pos, pop) +} diff --git a/server/pkg/model/node/node_test.go b/server/pkg/model/node/node_test.go index 6238961..03a2b04 100644 --- a/server/pkg/model/node/node_test.go +++ b/server/pkg/model/node/node_test.go @@ -23,7 +23,10 @@ func TestNode(t *testing.T) { t.Fatalf("couldn't prepare: %+v", err) } if tok != nil { - return n.Enact(tok) + err = tok.Finalize() + if err != nil { + t.Fatalf("couldn't Finalize: %+v", err) + } } return } @@ -47,10 +50,21 @@ func TestNode(t *testing.T) { }, }) - out := n.Read(2) + mustEnact(2, 1, Patch{ + Run: []raw.Op{ + {Range: []int{10}, Length: 4}, + }, + Set: raw.SetMap{ + "": []raw.GumnutSet{ + {Before: -1, String: jsonhelp.EncodeToUTF16(" bob")}, + }, + }, + }) + + out := n.Read(3) if !reflect.DeepEqual(out, raw.SetMap{ "": []raw.GumnutSet{ - {Before: 16, String: jsonhelp.EncodeToUTF16("hello there")}, + {Before: 20, String: jsonhelp.EncodeToUTF16("hello bob there")}, }, }) { t.Errorf("got bad state: %+v", out) diff --git a/server/pkg/model/raw/const.go b/server/pkg/model/raw/const.go new file mode 100644 index 0000000..a2f3224 --- /dev/null +++ b/server/pkg/model/raw/const.go @@ -0,0 +1,10 @@ +package raw + +import ( + "errors" +) + +var ( + ErrEncoding = errors.New("data encoding error") + ErrVersion = errors.New("version must always increase") +) diff --git a/server/pkg/model/raw/gumnut.go b/server/pkg/model/raw/gumnut.go index d54874f..cb5d272 100644 --- a/server/pkg/model/raw/gumnut.go +++ b/server/pkg/model/raw/gumnut.go @@ -2,7 +2,6 @@ package raw import ( "encoding/json" - "errors" "math" ) @@ -185,7 +184,3 @@ func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { return } - -var ( - ErrEncoding = errors.New("data encoding error") -) diff --git a/server/pkg/model/raw/merge.go b/server/pkg/model/raw/merge.go index 8c9afba..efab4bd 100644 --- a/server/pkg/model/raw/merge.go +++ b/server/pkg/model/raw/merge.go @@ -41,7 +41,8 @@ type SetEntry struct { Set SetMap } -type TransformFunc func(fromVersion int, gd *GumnutData) +// TransformFunc describes how we modify a GumnutData forward. +type TransformFunc func(fromVersion int, gd *GumnutData) (err error) type sliceSet struct { version int @@ -60,14 +61,15 @@ func (se *sliceSet) isStr() (is bool) { // MoveForward helps to move already "normalized" sets forward over structural ops. // This transforms it structurally and wrt its data. // The SetEntry array must have an incrementing version number. -func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { +func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap, err error) { // this effectively flattens history to a single entry // confirm we always go > version - var version int + version := -1 for _, e := range hist { + // log.Printf("got eVersion=%v vs version=%v", e.Version, version) if e.Version <= version { - return + return nil, ErrVersion } version = e.Version } @@ -130,10 +132,13 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { if tr != nil { for _, data := range work { for i := range data { - if data[i].version == 0 { + if data[i].gd.IsZero() { continue } - tr(data[i].version, &data[i].gd) + err = tr(data[i].version, &data[i].gd) + if err != nil { + return nil, err + } } } } @@ -144,7 +149,7 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { for key, data := range work { length := len(data) for i := 0; i < length; { - if data[i].version == 0 { + if data[i].gd.IsZero() { i++ continue } @@ -155,7 +160,7 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { j := i + 1 for j < length { jstr := data[j].isStr() - if jstr != str || data[j].version == 0 { + if jstr != str || data[j].gd.IsZero() { break } j++ @@ -180,7 +185,7 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap) { } } - return out + return out, nil } func transformOverLocal(v int, over []PriorOp) (out int) { @@ -199,7 +204,7 @@ func transformOverLocal(v int, over []PriorOp) (out int) { return v } -func TransformSetOver(set SetMap, prior []PriorOpSelf) (out SetMap) { +func TransformSetOver(set SetMap, prior []PriorOpSelf) (out SetMap, err error) { acc := SetMap{} // accumulate into single entries @@ -218,7 +223,7 @@ func TransformSetOver(set SetMap, prior []PriorOpSelf) (out SetMap) { pos = 0 } if pos < 0 { - return nil + return nil, ErrVersion // log.Printf("couldn't transform (%v -> %v) over=%+v", was, pos, prior) // for _, p := range prior { // log.Printf("prior=%+v", p.PriorOp) @@ -253,12 +258,6 @@ func TransformSetOver(set SetMap, prior []PriorOpSelf) (out SetMap) { acc[key] = tmp } - // run regular MoveForward ... - - return MoveForward([]SetEntry{ - { - Version: 1, - Set: acc, - }, - }, nil) + // run regular MoveForward to flatten data + return MoveForward([]SetEntry{{Version: 1, Set: acc}}, nil) } diff --git a/server/pkg/model/raw/merge_test.go b/server/pkg/model/raw/merge_test.go index 5db91fe..2e87840 100644 --- a/server/pkg/model/raw/merge_test.go +++ b/server/pkg/model/raw/merge_test.go @@ -117,10 +117,11 @@ func TestMoveForward(t *testing.T) { {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 200}}}, }, }, - func(ver int, gd *GumnutData) { + func(ver int, gd *GumnutData) (err error) { if gd.Type == DataTypeNumber { gd.Data += 100 } + return nil }, }, { @@ -144,7 +145,10 @@ func TestMoveForward(t *testing.T) { for _, each := range tests { t.Run(each.name, func(t *testing.T) { - got := MoveForward(each.src, each.tr) + got, err := MoveForward(each.src, each.tr) + if err != nil { + t.Errorf("expected non-nil err: %+v", err) + } if !reflect.DeepEqual(got, each.out) { t.Errorf("got bad out=%+v expected=%+v", got, each.out) } From 98823aa378fc3924a722224131232cb2f5f97403 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Fri, 20 Mar 2026 23:00:17 +1100 Subject: [PATCH 04/22] actually include prior data in read --- server/pkg/model/doc/const.go | 2 - server/pkg/model/doc/doc.go | 6 +- server/pkg/model/doc/doc_test.go | 95 ++++++++++++++++++++++++++++++++ server/pkg/model/node/node.go | 13 +++-- 4 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 server/pkg/model/doc/doc_test.go diff --git a/server/pkg/model/doc/const.go b/server/pkg/model/doc/const.go index 4852230..278eaa1 100644 --- a/server/pkg/model/doc/const.go +++ b/server/pkg/model/doc/const.go @@ -7,7 +7,5 @@ import ( var ( ErrIno = errors.New("bad ino") ErrVersion = errors.New("bad target version") - ErrRunOp = errors.New("bad run op, couldn't apply (bad transform?)") ErrSetOp = errors.New("bad set op, couldn't transform") - ErrRead = errors.New("internal error preparing data") ) diff --git a/server/pkg/model/doc/doc.go b/server/pkg/model/doc/doc.go index 24707c7..7bc6197 100644 --- a/server/pkg/model/doc/doc.go +++ b/server/pkg/model/doc/doc.go @@ -5,7 +5,7 @@ import ( "github.com/gumnutdev/foiled/server/pkg/model/raw" ) -func NewDoc() (d Doc) { +func New() (d Doc) { return &docImpl{ byIno: map[string]*node.Node{}, active: map[int]int{}, @@ -106,7 +106,9 @@ func (d *docImpl) Apply(work *DocWork) (err error) { } // otherwise it is returned from TransformFunc for _, res := range out { - res.Rollback() + if !res.Rollback() { + panic("could not Rollback") + } } return err } diff --git a/server/pkg/model/doc/doc_test.go b/server/pkg/model/doc/doc_test.go new file mode 100644 index 0000000..fe44294 --- /dev/null +++ b/server/pkg/model/doc/doc_test.go @@ -0,0 +1,95 @@ +package doc + +import ( + "reflect" + "testing" + + "github.com/gumnutdev/foiled/server/pkg/model/node" + "github.com/gumnutdev/foiled/server/pkg/model/raw" +) + +func intToUint(x int) (out uint64) { + return uint64(x) +} + +func TestPosRef(t *testing.T) { + d := New() + + err := d.Apply(&DocWork{ + TargetVersion: 0, + Handle: 1, + Update: map[string]node.Patch{ + "inode_abc": { + Run: []raw.Op{{Range: []int{0}, Length: 4}}, + Set: raw.SetMap{ + "": raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + {Type: raw.DataTypePosRef, String: "inode_abc", Data: intToUint(-2)}, + }}}, + }, + }, + }, + }) + if err != nil { + t.Errorf("could not apply PosRef") + } + + // read and confirm pos + out := d.Read() + p := out.Update["inode_abc"] + if !reflect.DeepEqual(p.Set[""], raw.GumnutSetArray{ + {Before: 4, Data: []raw.GumnutData{{Type: raw.DataTypePosRef, String: "inode_abc", Data: 3}}}, + }) { + t.Errorf("bad set: %+v", p.Set[""]) + } + + // move by inserting new, confirm update + err = d.Apply(&DocWork{ + TargetVersion: 1, + Handle: 2, + Update: map[string]node.Patch{ + "inode_abc": { + Run: []raw.Op{{Range: []int{1}, Length: 1}}, + Set: raw.SetMap{ + "": raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + {Type: raw.DataTypeString, String: "whatever"}, + }}}, + }, + }, + }, + }) + if err != nil { + t.Errorf("could not apply normal insert") + } + + out = d.Read() + p = out.Update["inode_abc"] + if !reflect.DeepEqual(p.Set[""], raw.GumnutSetArray{ + {Before: 2, Data: []raw.GumnutData{{Type: raw.DataTypeString, String: "whatever"}}}, + {Before: 5, Data: []raw.GumnutData{{Type: raw.DataTypePosRef, String: "inode_abc", Data: 4}}}, + }) { + t.Errorf("bad set: %+v", p.Set[""]) + } +} + +func TestInvalidPosRef(t *testing.T) { + d := New() + + err := d.Apply(&DocWork{ + TargetVersion: 0, + Handle: 1, + Update: map[string]node.Patch{ + "inode_abc": { + Run: []raw.Op{{Range: []int{0}, Length: 4}}, + Set: raw.SetMap{ + "": raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + {Type: raw.DataTypePosRef, String: "inode_abc", Data: intToUint(-5)}, + }}}, + }, + }, + }, + }) + if err != ErrSetOp { + t.Errorf("should have failed to apply, was: %v", err) + } + +} diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index f6f03e9..5e7eaef 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -64,6 +64,8 @@ func (n *Node) Read(docVersion int) (sm raw.SetMap) { _, hi := n.VersionRange() docVersion = max(hi, docVersion) + // log.Printf("reading %v -> %v", n.readAt, docVersion) + if n.readAt >= docVersion { return n.set } @@ -73,9 +75,10 @@ func (n *Node) Read(docVersion int) (sm raw.SetMap) { }) tail := n.history[idx:] - prep := make([]raw.SetEntry, len(tail)) - for i, he := range tail { - prep[i] = he.entry + prep := make([]raw.SetEntry, 1, len(tail)+1) + prep[0] = raw.SetEntry{Version: n.readAt, Set: n.set} + for _, he := range tail { + prep = append(prep, he.entry) } // setup transform fn @@ -104,8 +107,8 @@ type PrepareArg struct { NewVersion int } -// Prepare accepts the user's work on this node and confirms it is valid. -// It returns a PrepareToken which can be later passed to Enact. +// Prepare accepts the user's work on this node and applies it to the history. +// It returns a PrepareToken which must later be finalized or rolled back. // The PrepareToken can be nil if this is a no-op. func (n *Node) Prepare(arg PrepareArg, p Patch) (tok *PrepareToken, err error) { lo, hi := n.VersionRange() From dd45fe66b09da1587ed0973a933bf82b877b511c Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Fri, 20 Mar 2026 23:06:48 +1100 Subject: [PATCH 05/22] comment --- server/pkg/model/node/node.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index 5e7eaef..71ae38c 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -145,7 +145,8 @@ func (n *Node) Prepare(arg PrepareArg, p Patch) (tok *PrepareToken, err error) { out.Run = append(out.Run, op) } - // leverage MoveForward to xform so sets are normalized + // transform Set _locations_ + // this doesn't call our Transform function; we can't do that until all priors are in-place out.Set, err = raw.TransformSetOver(p.Set, hist) if err != nil { return nil, err From 1176274eba3361db2e7091567fa2d0b24d57e4d2 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sun, 22 Mar 2026 08:44:30 +1100 Subject: [PATCH 06/22] Inode type --- server/pkg/model/raw/gumnut.go | 22 +++++++++++++++++----- server/pkg/model/raw/merge.go | 4 ++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/server/pkg/model/raw/gumnut.go b/server/pkg/model/raw/gumnut.go index cb5d272..5921e4e 100644 --- a/server/pkg/model/raw/gumnut.go +++ b/server/pkg/model/raw/gumnut.go @@ -10,10 +10,11 @@ type DataType int const ( DataTypeEmpty DataType = iota // unset/void - distinct from undefined DataTypeNumber // JS number (float64 in bits) + DataTypePosRef // ref to inode with pos (str=>inode, pos=>data) DataTypeBigInt // JS bigint (string) - DataTypeKnown // JS known: true, false, null, undefined + DataTypeKnown // fixed string knowns (including JS null, true, false -> enc to real) DataTypeString // immutable string - DataTypePosRef // ref to inode with pos (str=>inode, pos=>data) + DataTypeInode // as string, but specifically denotes another inode ) // GumnutData represents all possible values inside Gumnut. @@ -95,6 +96,13 @@ func (gd GumnutData) MarshalJSON() (b []byte, err error) { out.String = gd.String return json.Marshal(out) + case DataTypeInode: + var out struct { + Inode string `json:"i"` + } + out.Inode = gd.String + return json.Marshal(out) + case DataTypePosRef: var out struct { Inode string `json:"i"` @@ -140,7 +148,7 @@ func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { Known *string `json:"x"` BigInt string `json:"b"` // bigint value Inf *int `json:"inf"` - Inode string `json:"i"` + Inode *string `json:"i"` Pos *uint64 `json:"p"` } err = json.Unmarshal(b, &tmp) @@ -173,11 +181,15 @@ func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { g.Data = math.Float64bits(math.Inf(int(*tmp.Inf))) } - case tmp.Pos != nil: + case tmp.Pos != nil && tmp.Inode != nil: g.Type = DataTypePosRef - g.String = tmp.Inode + g.String = *tmp.Inode g.Data = *tmp.Pos + case tmp.Inode != nil: + g.Type = DataTypeInode + g.String = *tmp.Inode + default: return ErrEncoding } diff --git a/server/pkg/model/raw/merge.go b/server/pkg/model/raw/merge.go index efab4bd..5c64474 100644 --- a/server/pkg/model/raw/merge.go +++ b/server/pkg/model/raw/merge.go @@ -22,6 +22,7 @@ func (s GumnutSetArray) IterData() (it iter.Seq[*GumnutData]) { } // SetMap is a map from key to GumnutSetArray. +// This allows sets on different keyed arrays for one node. type SetMap map[string]GumnutSetArray // Tidy removes superfluous items from SetMap. @@ -204,6 +205,9 @@ func transformOverLocal(v int, over []PriorOp) (out int) { return v } +// TransformSetOver is used to transform Set locations for a given node and "normalize" them. +// Basically, this maps a user setting at "-1" to the real location, or to fix a server-space location which has changed. +// This happens after a patch's structural changes have been applied and the PriorOpSelf will reflect that. func TransformSetOver(set SetMap, prior []PriorOpSelf) (out SetMap, err error) { acc := SetMap{} From c5b98f34f057c24b49331302b51feb13dec06848 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Fri, 27 Mar 2026 12:39:12 +1100 Subject: [PATCH 07/22] WIP --- client/lib/model/raw.ts | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 client/lib/model/raw.ts diff --git a/client/lib/model/raw.ts b/client/lib/model/raw.ts new file mode 100644 index 0000000..0e8f91f --- /dev/null +++ b/client/lib/model/raw.ts @@ -0,0 +1,69 @@ +import { AATree, Rope } from 'thorish'; +import { Op } from '../ymodel/types.ts'; + +/** + * This is an new rope state class. + * + * This holds the _canonical server state_, which is a number of keyed arrays with effectively infinite length. + * + * Unlike the server, this is ID-based so that end-users don't have to "update" their IDs to fetch new locations. + */ +export class NewRopeState { + private readonly rope: Rope; + private readonly tree: AATree; + private readonly keyedData: Record = {}; + + public static new(zeroValue: X) { + return new NewRopeState(zeroValue); + } + + public clone(): NewRopeState { + return new NewRopeState(this.zeroValue); + } + + private constructor(private readonly zeroValue: X) { + this.rope = new Rope(0); + this.tree = new AATree((a, b) => a - b); + } + + private eachData(fn: (arg: X[] | Uint16Array) => void) { + for (const data of Object.values(this.keyedData)) { + fn(data); + } + } + + /** + * Does the thing + */ + applyOp(op: Op) { + switch (op.r.length) { + case 1: + if (!op.length || op.length <= 0) { + return false; + } + + this.eachData((data) => {}); + } + } +} + +export class DataState { + private data: X[] | Uint16Array = []; + + /** + * Does the thing + */ + applyOp(op: Op) { + switch (op.r.length) { + case 1: + if (!op.length || op.length <= 0) { + return false; + } + + if (Array.isArray(this.data)) { + // FIXME: we cannot splice Uint16Array; must be chunked because the zero value is not 0 + this.data.splice(op.r[0], 0, op.length); + } + } + } +} From 6f358374a560978e8bc7c683c71e75bde223f2ba Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Fri, 27 Mar 2026 14:18:46 +1100 Subject: [PATCH 08/22] basic layout --- client/lib/model/layout.test.ts | 21 +++ client/lib/model/layout.ts | 248 ++++++++++++++++++++++++++++++++ client/lib/model/raw.ts | 69 --------- 3 files changed, 269 insertions(+), 69 deletions(-) create mode 100644 client/lib/model/layout.test.ts create mode 100644 client/lib/model/layout.ts delete mode 100644 client/lib/model/raw.ts diff --git a/client/lib/model/layout.test.ts b/client/lib/model/layout.test.ts new file mode 100644 index 0000000..b3bfa96 --- /dev/null +++ b/client/lib/model/layout.test.ts @@ -0,0 +1,21 @@ +import test from 'node:test'; +import * as assert from 'node:assert'; +import { RopeLayout } from './layout.ts'; + +test('layout', () => { + const r = RopeLayout.new(); + + const idForHigh = r.posToId(100); + assert.deepStrictEqual(r.idToPos(idForHigh), { pos: 100, gone: false }); + + r.applyOp(r.coerceOp({ r: [0], length: 5 })); + assert.deepStrictEqual(r.idToPos(idForHigh), { pos: 105, gone: false }); + + const deleteOp = r.coerceOp({ r: [40, 105] }); + + r.applyOp(r.coerceOp({ r: [100, 110] })); + assert.deepStrictEqual(r.idToPos(idForHigh), { pos: 100, gone: true }); + + // confirm deletion is now _lower_, because we nuked the target range + assert.deepStrictEqual(r.applyOp(deleteOp), { lo: 40, hi: 100, length: 0 }); +}); diff --git a/client/lib/model/layout.ts b/client/lib/model/layout.ts new file mode 100644 index 0000000..048a9a2 --- /dev/null +++ b/client/lib/model/layout.ts @@ -0,0 +1,248 @@ +import { AATree, Rope } from 'thorish'; +import type { Op } from '../ymodel/types.ts'; +import type { RangeUpdate } from '../ymodel/internal/range.ts'; + +export type InternalOp = { + r: number[]; + length: number; + id: number; +}; + +export type IdLookup = { + pos: number; + gone: boolean; +}; + +let globalLocalId = 20_000; + +function allocLocalId(length?: number): number { + if (length && length > 0) { + globalLocalId += length; + return globalLocalId; + } + return 0; +} + +/** + * @returns The last allocated ID, for use in tests. + */ +export function lastLocalId(): number { + return globalLocalId; +} + +/** + * Holds a layout with custom IDs. + * This has effectively infinite length and helps track positions, but no actual data. + * + * Unlike the server, this is ID-based so that end-users don't have to "update" their IDs to fetch new locations. + */ +export class RopeLayout { + private constructor( + /** + * Points ID to length. + * However, the actual length inside the {@link Rope} might be zero to indicate deletion. + */ + private readonly rope: Rope = new Rope(0, 0), + + /** + * Tree for ID lookup only. + */ + private readonly tree: AATree = new AATree((a, b) => a - b), + ) {} + + /** + * Creeate a new instance. + */ + public static new() { + return new RopeLayout(); + } + + /** + * Clones this instance. + */ + public clone(): RopeLayout { + return new RopeLayout(this.rope.clone(), this.tree.clone()); + } + + /** + * Ensures an edge in the underlying {@link Rope} at the given {@link Id}. + * + * Returns whether the target ID exists (true) or is deleted (false). + */ + private ensureIdEdge(id: number): boolean { + if (id === 0) { + return true; + } + const highId = this.tree.equalAfter(id); + if (!highId) { + throw new Error(`can't find ID: ${id}`); + } + + // find host, ensure we're slicing "in the middle" + const host = this.rope.lookup(highId); + const exists = host.length !== 0; + if (highId === id) { + return exists; + } + + // adjust existing to be right side + const right = host.id - id; + this.rope.adjust(host.id, right, exists ? right : 0); + + // insert new left side + const left = host.data - right; + this.rope.insertAfter(host.prevId, id, exists ? left : 0, left); + + // insert new left side ID + this.tree.insert(id); + + return exists; + } + + /** + * Coerce this public {@link Op} at the current state to a repeatable {@link InternalOp}. + */ + coerceOp(op: Op): InternalOp { + return { + r: op.r.map((r) => this.posToId(r)), + length: op.length ?? 0, + id: allocLocalId(op.length), + }; + } + + /** + * Converts a >=0 position to an internal {@link Id}. + * There is no limit on the length here. + */ + public posToId(pos: number): number { + if (pos < 0) { + throw new Error(`can't posToId negative: ${pos}`); + } else if (pos === 0) { + return 0; + } + + const extra = pos - this.rope.length(); + if (extra > 0) { + const lastId = this.rope.last(); + + // TODO: would create a lot of cruft if a user simply calls posToId one-by-one ... + + const id = allocLocalId(extra); + this.rope.insertAfter(lastId, id, extra, extra); + this.tree.insert(id); + return id; + } + + const lookup = this.rope.byPosition(pos); // O(logn-ish) + return lookup.id - lookup.offset; + } + + /** + * Resolves this ID, including its most valid >=0 pos and whether it is gone. + * + * If the ID is not valid here, returns `undefined`. + */ + public idToPos(id: number): IdLookup | undefined { + if (id === 0) { + return { pos: 0, gone: false }; + } + const highId = this.tree.equalAfter(id); + if (!highId) { + return; + } + + // find high pos + const pos = this.rope.find(highId); + const lookup = this.rope.lookup(highId); + + const fromEnd = highId - id; + if (fromEnd > lookup.data) { + return; // out of range + } + + if (lookup.length === 0) { + return { pos, gone: true }; + } + return { pos: pos - fromEnd, gone: false }; + } + + /** + * Applies a previously coerced {@link InternalOp}. + */ + applyOp(int: InternalOp): RangeUpdate | undefined { + switch (int.r.length) { + case 1: + return this.internalInsert(int.r[0], int.id, int.length); + + case 2: + return this.internalDelete(int.r[0], int.r[1]); + } + + // TODO: support flip + throw new Error(`unsupported op: ${int.r.length}`); + } + + private internalInsert(at: number, newId: number, length: number): RangeUpdate | undefined { + if (length <= 0) { + return; + } + + // remember: the rope doesn't really care that we think long entries take up "rope space" + const prevHostId = this.tree.equalAfter(newId); + if (prevHostId !== undefined) { + const offset = prevHostId - newId; + const host = this.rope.lookup(prevHostId); + if (host.length > offset) { + throw new Error(`can't insert over prior id: insert=${newId} prev=${prevHostId}`); + } + } + + const exists = this.ensureIdEdge(at); + + // we always insert (even into deleted space) + this.rope.insertAfter(at, newId, exists ? length : 0, length); + this.tree.insert(newId); + + if (exists) { + const pos = this.rope.find(at); + return { lo: pos, hi: pos, length }; + } + } + + private internalDelete(lo: number, hi: number): RangeUpdate | undefined { + if (lo === hi) { + return; + } + + this.ensureIdEdge(lo); + this.ensureIdEdge(hi); + + if (!this.rope.before(lo, hi)) { + [lo, hi] = [hi, lo]; + } + + // walk and delete undeleted bits + let cleared = 0; + for (const prev of this.rope.iter(lo, hi)) { + if (prev.length) { + this.rope.adjust(prev.id, prev.data, 0); + cleared += prev.length; + } + } + + if (cleared !== 0) { + const pos = this.rope.find(lo); + return { lo: pos, hi: pos + cleared, length: 0 }; + } + } +} + +/** + * Generates the inverse operation required to remove this {@link InternalOp}. + * Just returns a delete for an insert. + */ +export function rollbackInternalOp(int: InternalOp): InternalOp | undefined { + if (int.r.length === 1) { + return { r: [int.r[0], int.id], length: 0, id: 0 }; + } +} diff --git a/client/lib/model/raw.ts b/client/lib/model/raw.ts deleted file mode 100644 index 0e8f91f..0000000 --- a/client/lib/model/raw.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { AATree, Rope } from 'thorish'; -import { Op } from '../ymodel/types.ts'; - -/** - * This is an new rope state class. - * - * This holds the _canonical server state_, which is a number of keyed arrays with effectively infinite length. - * - * Unlike the server, this is ID-based so that end-users don't have to "update" their IDs to fetch new locations. - */ -export class NewRopeState { - private readonly rope: Rope; - private readonly tree: AATree; - private readonly keyedData: Record = {}; - - public static new(zeroValue: X) { - return new NewRopeState(zeroValue); - } - - public clone(): NewRopeState { - return new NewRopeState(this.zeroValue); - } - - private constructor(private readonly zeroValue: X) { - this.rope = new Rope(0); - this.tree = new AATree((a, b) => a - b); - } - - private eachData(fn: (arg: X[] | Uint16Array) => void) { - for (const data of Object.values(this.keyedData)) { - fn(data); - } - } - - /** - * Does the thing - */ - applyOp(op: Op) { - switch (op.r.length) { - case 1: - if (!op.length || op.length <= 0) { - return false; - } - - this.eachData((data) => {}); - } - } -} - -export class DataState { - private data: X[] | Uint16Array = []; - - /** - * Does the thing - */ - applyOp(op: Op) { - switch (op.r.length) { - case 1: - if (!op.length || op.length <= 0) { - return false; - } - - if (Array.isArray(this.data)) { - // FIXME: we cannot splice Uint16Array; must be chunked because the zero value is not 0 - this.data.splice(op.r[0], 0, op.length); - } - } - } -} From 0d4c189f5bd214aa45e1d94c873482dabe5f0161 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Fri, 27 Mar 2026 15:18:16 +1100 Subject: [PATCH 09/22] remove SetMap --- server/pkg/model/doc/doc_test.go | 32 ++-- server/pkg/model/node/node.go | 6 +- server/pkg/model/node/node_test.go | 24 +-- server/pkg/model/node/patch.go | 4 +- server/pkg/model/raw/merge.go | 230 +++++++++++++---------------- server/pkg/model/raw/merge_test.go | 87 ++++------- 6 files changed, 151 insertions(+), 232 deletions(-) diff --git a/server/pkg/model/doc/doc_test.go b/server/pkg/model/doc/doc_test.go index fe44294..18455e4 100644 --- a/server/pkg/model/doc/doc_test.go +++ b/server/pkg/model/doc/doc_test.go @@ -21,11 +21,9 @@ func TestPosRef(t *testing.T) { Update: map[string]node.Patch{ "inode_abc": { Run: []raw.Op{{Range: []int{0}, Length: 4}}, - Set: raw.SetMap{ - "": raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ - {Type: raw.DataTypePosRef, String: "inode_abc", Data: intToUint(-2)}, - }}}, - }, + Set: raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + {Type: raw.DataTypePosRef, String: "inode_abc", Data: intToUint(-2)}, + }}}, }, }, }) @@ -36,10 +34,10 @@ func TestPosRef(t *testing.T) { // read and confirm pos out := d.Read() p := out.Update["inode_abc"] - if !reflect.DeepEqual(p.Set[""], raw.GumnutSetArray{ + if !reflect.DeepEqual(p.Set, raw.GumnutSetArray{ {Before: 4, Data: []raw.GumnutData{{Type: raw.DataTypePosRef, String: "inode_abc", Data: 3}}}, }) { - t.Errorf("bad set: %+v", p.Set[""]) + t.Errorf("bad set: %+v", p.Set) } // move by inserting new, confirm update @@ -49,11 +47,9 @@ func TestPosRef(t *testing.T) { Update: map[string]node.Patch{ "inode_abc": { Run: []raw.Op{{Range: []int{1}, Length: 1}}, - Set: raw.SetMap{ - "": raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ - {Type: raw.DataTypeString, String: "whatever"}, - }}}, - }, + Set: raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + {Type: raw.DataTypeString, String: "whatever"}, + }}}, }, }, }) @@ -63,11 +59,11 @@ func TestPosRef(t *testing.T) { out = d.Read() p = out.Update["inode_abc"] - if !reflect.DeepEqual(p.Set[""], raw.GumnutSetArray{ + if !reflect.DeepEqual(p.Set, raw.GumnutSetArray{ {Before: 2, Data: []raw.GumnutData{{Type: raw.DataTypeString, String: "whatever"}}}, {Before: 5, Data: []raw.GumnutData{{Type: raw.DataTypePosRef, String: "inode_abc", Data: 4}}}, }) { - t.Errorf("bad set: %+v", p.Set[""]) + t.Errorf("bad set: %+v", p.Set) } } @@ -80,11 +76,9 @@ func TestInvalidPosRef(t *testing.T) { Update: map[string]node.Patch{ "inode_abc": { Run: []raw.Op{{Range: []int{0}, Length: 4}}, - Set: raw.SetMap{ - "": raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ - {Type: raw.DataTypePosRef, String: "inode_abc", Data: intToUint(-5)}, - }}}, - }, + Set: raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + {Type: raw.DataTypePosRef, String: "inode_abc", Data: intToUint(-5)}, + }}}, }, }, }) diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index 71ae38c..aa5cb9d 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -18,8 +18,8 @@ import ( // // It knows about the handle who performed history operations, so it is exposed to end-clients. type Node struct { - set raw.SetMap // data here - readAt int // when we last read/flattened data + set raw.GumnutSetArray // data here + readAt int // when we last read/flattened data history []historyEntry Transform TransformHandleFunc // must be set to xform data @@ -60,7 +60,7 @@ func (n *Node) VersionRange() (lo, hi int) { // Read reads the content of this node. // This will transform data forward as needed. -func (n *Node) Read(docVersion int) (sm raw.SetMap) { +func (n *Node) Read(docVersion int) (sm raw.GumnutSetArray) { _, hi := n.VersionRange() docVersion = max(hi, docVersion) diff --git a/server/pkg/model/node/node_test.go b/server/pkg/model/node/node_test.go index 03a2b04..521b82c 100644 --- a/server/pkg/model/node/node_test.go +++ b/server/pkg/model/node/node_test.go @@ -32,10 +32,8 @@ func TestNode(t *testing.T) { } mustEnact(1, 0, Patch{ - Set: raw.SetMap{ - "": []raw.GumnutSet{ - {Before: 10, String: jsonhelp.EncodeToUTF16("hello")}, - }, + Set: raw.GumnutSetArray{ + {Before: 10, String: jsonhelp.EncodeToUTF16("hello")}, }, }) @@ -43,10 +41,8 @@ func TestNode(t *testing.T) { Run: []raw.Op{ {Range: []int{10}, Length: 6}, }, - Set: raw.SetMap{ - "": []raw.GumnutSet{ - {Before: -1, String: jsonhelp.EncodeToUTF16(" there")}, - }, + Set: raw.GumnutSetArray{ + {Before: -1, String: jsonhelp.EncodeToUTF16(" there")}, }, }) @@ -54,18 +50,14 @@ func TestNode(t *testing.T) { Run: []raw.Op{ {Range: []int{10}, Length: 4}, }, - Set: raw.SetMap{ - "": []raw.GumnutSet{ - {Before: -1, String: jsonhelp.EncodeToUTF16(" bob")}, - }, + Set: raw.GumnutSetArray{ + {Before: -1, String: jsonhelp.EncodeToUTF16(" bob")}, }, }) out := n.Read(3) - if !reflect.DeepEqual(out, raw.SetMap{ - "": []raw.GumnutSet{ - {Before: 20, String: jsonhelp.EncodeToUTF16("hello bob there")}, - }, + if !reflect.DeepEqual(out, raw.GumnutSetArray{ + {Before: 20, String: jsonhelp.EncodeToUTF16("hello bob there")}, }) { t.Errorf("got bad state: %+v", out) } diff --git a/server/pkg/model/node/patch.go b/server/pkg/model/node/patch.go index 1e47039..70a14da 100644 --- a/server/pkg/model/node/patch.go +++ b/server/pkg/model/node/patch.go @@ -5,8 +5,8 @@ import ( ) type Patch struct { - Run []raw.Op // structural changes - Set raw.SetMap // sets per keyed list + Run []raw.Op // structural changes + Set raw.GumnutSetArray // sets for data } func (p *Patch) IsZero() (is bool) { diff --git a/server/pkg/model/raw/merge.go b/server/pkg/model/raw/merge.go index 5c64474..6b3fd3a 100644 --- a/server/pkg/model/raw/merge.go +++ b/server/pkg/model/raw/merge.go @@ -21,25 +21,12 @@ func (s GumnutSetArray) IterData() (it iter.Seq[*GumnutData]) { } } -// SetMap is a map from key to GumnutSetArray. -// This allows sets on different keyed arrays for one node. -type SetMap map[string]GumnutSetArray - -// Tidy removes superfluous items from SetMap. -func (sm *SetMap) Tidy() { - for key, arr := range *sm { - if len(arr) == 0 { - delete(*sm, key) - } - } -} - -// SetEntry describes how to modify a SetMap over time. +// SetEntry describes how to modify data over time. // The first SetEntry in any array is effectively the base state. type SetEntry struct { Prior []PriorOp Version int - Set SetMap + Set GumnutSetArray } // TransformFunc describes how we modify a GumnutData forward. @@ -62,7 +49,7 @@ func (se *sliceSet) isStr() (is bool) { // MoveForward helps to move already "normalized" sets forward over structural ops. // This transforms it structurally and wrt its data. // The SetEntry array must have an incrementing version number. -func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap, err error) { +func MoveForward(hist []SetEntry, tr TransformFunc) (out GumnutSetArray, err error) { // this effectively flattens history to a single entry // confirm we always go > version @@ -75,47 +62,40 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap, err error) { version = e.Version } - work := make(map[string][]sliceSet) + var slice []sliceSet priors := []PriorOp{} // work backwards for _, each := range slices.Backward(hist) { // for each keyed set - for key, update := range each.Set { - - // apply each part of the keyed set - for _, s := range update { - target := s.Before - - str := s.String != nil - - for _, gd := range slices.Backward(s.asData()) { - pos := target - if len(priors) > 0 { - pos = transformOverLocal(target, priors) - } - target-- - - if pos == 0 { - continue // xformed away - } - index := pos - 1 - - slice := work[key] - if len(slice) < pos { - // off end, always exists - slice = extendSlice(slice, pos) - work[key] = slice - slice[index] = sliceSet{each.Version, str, gd} - - } else if slice[index].version == 0 { - // check nothing already here - slice[index] = sliceSet{each.Version, str, gd} - - } else { - continue - } + for _, s := range each.Set { + target := s.Before + str := s.String != nil + + for _, gd := range slices.Backward(s.asData()) { + pos := target + if len(priors) > 0 { + pos = transformOverLocal(target, priors) + } + target-- + + if pos == 0 { + continue // xformed away + } + index := pos - 1 + + if len(slice) < pos { + // off end, always exists + slice = extendSlice(slice, pos) + slice[index] = sliceSet{each.Version, str, gd} + + } else if slice[index].version == 0 { + // check nothing already here + slice[index] = sliceSet{each.Version, str, gd} + + } else { + continue } } } @@ -131,59 +111,55 @@ func MoveForward(hist []SetEntry, tr TransformFunc) (out SetMap, err error) { // finally, xform all data _forward_ if tr != nil { - for _, data := range work { - for i := range data { - if data[i].gd.IsZero() { - continue - } - err = tr(data[i].version, &data[i].gd) - if err != nil { - return nil, err - } + for i := range slice { + if slice[i].gd.IsZero() { + continue + } + err = tr(slice[i].version, &slice[i].gd) + if err != nil { + return nil, err } } } // create minimal sets - out = make(SetMap, len(work)) + out = GumnutSetArray{} - for key, data := range work { - length := len(data) - for i := 0; i < length; { - if data[i].gd.IsZero() { - i++ - continue - } + length := len(slice) + for i := 0; i < length; { + if slice[i].gd.IsZero() { + i++ + continue + } - str := data[i].isStr() + str := slice[i].isStr() - // consume matching (non-zero version, str == base) - j := i + 1 - for j < length { - jstr := data[j].isStr() - if jstr != str || data[j].gd.IsZero() { - break - } - j++ + // consume matching (non-zero version, str == base) + j := i + 1 + for j < length { + jstr := slice[j].isStr() + if jstr != str || slice[j].gd.IsZero() { + break } + j++ + } - // got slice [i,j], convert to String or Data - set := GumnutSet{Before: j} - if str { - set.String = make([]uint16, j-i) - for k := i; k < j; k++ { - set.String[k-i], _ = data[k].gd.JSRune() - } - } else { - set.Data = make([]GumnutData, j-i) - for k := i; k < j; k++ { - set.Data[k-i] = data[k].gd - } + // got slice [i,j], convert to String or Data + set := GumnutSet{Before: j} + if str { + set.String = make([]uint16, j-i) + for k := i; k < j; k++ { + set.String[k-i], _ = slice[k].gd.JSRune() + } + } else { + set.Data = make([]GumnutData, j-i) + for k := i; k < j; k++ { + set.Data[k-i] = slice[k].gd } - out[key] = append(out[key], set) - - i = j } + out = append(out, set) + + i = j } return out, nil @@ -208,60 +184,54 @@ func transformOverLocal(v int, over []PriorOp) (out int) { // TransformSetOver is used to transform Set locations for a given node and "normalize" them. // Basically, this maps a user setting at "-1" to the real location, or to fix a server-space location which has changed. // This happens after a patch's structural changes have been applied and the PriorOpSelf will reflect that. -func TransformSetOver(set SetMap, prior []PriorOpSelf) (out SetMap, err error) { - acc := SetMap{} - +func TransformSetOver(arr GumnutSetArray, prior []PriorOpSelf) (out GumnutSetArray, err error) { // accumulate into single entries - for key, arr := range set { - var tmp []GumnutSet - - for _, x := range arr { + var tmp []GumnutSet - targets := make([]int, x.length()) - for i := range targets { - pos := x.Before - len(targets) + i + 1 + for _, x := range arr { - var gone bool - pos, gone = TransformOver(pos, prior) - if gone { - pos = 0 - } - if pos < 0 { - return nil, ErrVersion - // log.Printf("couldn't transform (%v -> %v) over=%+v", was, pos, prior) - // for _, p := range prior { - // log.Printf("prior=%+v", p.PriorOp) - // } - // panic("BAD POS") - } + targets := make([]int, x.length()) + for i := range targets { + pos := x.Before - len(targets) + i + 1 - targets[i] = pos + var gone bool + pos, gone = TransformOver(pos, prior) + if gone { + pos = 0 } - - if x.String != nil { - for i, r := range x.String { - pos := targets[i] - if pos == 0 { - continue - } - tmp = append(tmp, GumnutSet{Before: pos, String: []uint16{r}}) - } - continue + if pos < 0 { + return nil, ErrVersion + // log.Printf("couldn't transform (%v -> %v) over=%+v", was, pos, prior) + // for _, p := range prior { + // log.Printf("prior=%+v", p.PriorOp) + // } + // panic("BAD POS") } - for i, r := range x.Data { + targets[i] = pos + } + + if x.String != nil { + for i, r := range x.String { pos := targets[i] if pos == 0 { continue } - tmp = append(tmp, GumnutSet{Before: pos, Data: []GumnutData{r}}) + tmp = append(tmp, GumnutSet{Before: pos, String: []uint16{r}}) } continue } - acc[key] = tmp + for i, r := range x.Data { + pos := targets[i] + if pos == 0 { + continue + } + tmp = append(tmp, GumnutSet{Before: pos, Data: []GumnutData{r}}) + } + continue } // run regular MoveForward to flatten data - return MoveForward([]SetEntry{{Version: 1, Set: acc}}, nil) + return MoveForward([]SetEntry{{Version: 1, Set: tmp}}, nil) } diff --git a/server/pkg/model/raw/merge_test.go b/server/pkg/model/raw/merge_test.go index 2e87840..8fd0b7d 100644 --- a/server/pkg/model/raw/merge_test.go +++ b/server/pkg/model/raw/merge_test.go @@ -11,13 +11,13 @@ func TestMoveForward(t *testing.T) { tests := []struct { name string src []SetEntry - out SetMap + out GumnutSetArray tr TransformFunc }{ { "zero", []SetEntry{}, - SetMap{}, + GumnutSetArray{}, nil, }, { @@ -25,10 +25,8 @@ func TestMoveForward(t *testing.T) { []SetEntry{ { Version: 1, - Set: SetMap{ - "abc": GumnutSetArray{ - {Before: 16, String: jsonhelp.EncodeToUTF16("hello there")}, - }, + Set: GumnutSetArray{ + {Before: 16, String: jsonhelp.EncodeToUTF16("hello there")}, }, }, { @@ -37,17 +35,13 @@ func TestMoveForward(t *testing.T) { &priorOpRemove{lo: 9, hi: 10}, &priorOpInsert{after: 9, length: 1}, }, - Set: SetMap{ - "abc": GumnutSetArray{ - {Before: 10, String: jsonhelp.EncodeToUTF16("!")}, - }, + Set: GumnutSetArray{ + {Before: 10, String: jsonhelp.EncodeToUTF16("!")}, }, }, }, - SetMap{ - "abc": GumnutSetArray{ - {Before: 16, String: jsonhelp.EncodeToUTF16("hell! there")}, - }, + GumnutSetArray{ + {Before: 16, String: jsonhelp.EncodeToUTF16("hell! there")}, }, nil, }, @@ -56,20 +50,16 @@ func TestMoveForward(t *testing.T) { []SetEntry{ { Version: 100, - Set: SetMap{ - "abc": GumnutSetArray{ - {Before: 80, String: jsonhelp.EncodeToUTF16("hello")}, - {Before: 86, String: jsonhelp.EncodeToUTF16("? there")}, - {Before: 87, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, - }, + Set: GumnutSetArray{ + {Before: 80, String: jsonhelp.EncodeToUTF16("hello")}, + {Before: 86, String: jsonhelp.EncodeToUTF16("? there")}, + {Before: 87, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, }, }, }, - SetMap{ - "abc": GumnutSetArray{ - {Before: 86, String: jsonhelp.EncodeToUTF16("hello there")}, - {Before: 87, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, - }, + GumnutSetArray{ + {Before: 86, String: jsonhelp.EncodeToUTF16("hello there")}, + {Before: 87, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, }, nil, }, @@ -78,25 +68,19 @@ func TestMoveForward(t *testing.T) { []SetEntry{ { Version: 1, - Set: SetMap{ - "foo": GumnutSetArray{ - {Before: 5, String: jsonhelp.EncodeToUTF16("ABCD")}, - }, + Set: GumnutSetArray{ + {Before: 5, String: jsonhelp.EncodeToUTF16("ABCD")}, }, }, { Version: 2, - Set: SetMap{ - "foo": GumnutSetArray{ - {Before: 4, String: jsonhelp.EncodeToUTF16("x")}, - }, + Set: GumnutSetArray{ + {Before: 4, String: jsonhelp.EncodeToUTF16("x")}, }, }, }, - SetMap{ - "foo": GumnutSetArray{ - {Before: 5, String: jsonhelp.EncodeToUTF16("ABxD")}, - }, + GumnutSetArray{ + {Before: 5, String: jsonhelp.EncodeToUTF16("ABxD")}, }, nil, }, @@ -105,17 +89,13 @@ func TestMoveForward(t *testing.T) { []SetEntry{ { Version: 1, - Set: SetMap{ - "bar": GumnutSetArray{ - {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 100}}}, - }, + Set: GumnutSetArray{ + {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 100}}}, }, }, }, - SetMap{ - "bar": GumnutSetArray{ - {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 200}}}, - }, + GumnutSetArray{ + {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 200}}}, }, func(ver int, gd *GumnutData) (err error) { if gd.Type == DataTypeNumber { @@ -124,23 +104,6 @@ func TestMoveForward(t *testing.T) { return nil }, }, - { - "multiple_keys", - []SetEntry{ - { - Version: 1, - Set: SetMap{ - "k1": GumnutSetArray{{Before: 1, String: jsonhelp.EncodeToUTF16("1")}}, - "k2": GumnutSetArray{{Before: 2, String: jsonhelp.EncodeToUTF16("22")}}, - }, - }, - }, - SetMap{ - "k1": GumnutSetArray{{Before: 1, String: jsonhelp.EncodeToUTF16("1")}}, - "k2": GumnutSetArray{{Before: 2, String: jsonhelp.EncodeToUTF16("22")}}, - }, - nil, - }, } for _, each := range tests { From 4912f1a4ad2c4372aa072764b1635f474cc326d2 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sun, 29 Mar 2026 08:25:57 +1100 Subject: [PATCH 10/22] WIP remove ymodel --- client/lib/gumnut/floor.ts | 80 ++++---- client/lib/model/layout.ts | 3 +- client/lib/ymodel/api.ts | 16 +- client/lib/ymodel/types.ts | 32 ++-- demo-encode.ts | 34 ++++ server/pkg/model/doc/doc.go | 11 +- server/pkg/model/doc/doc_test.go | 39 +++- server/pkg/model/doc/types.go | 16 +- server/pkg/model/node/node.go | 2 + server/pkg/model/node/patch.go | 57 +++++- server/pkg/model/node/patch_test.go | 115 ++++++++++++ server/pkg/model/raw/enc.go | 31 +++- server/pkg/model/raw/op.go | 74 ++++++++ server/pkg/model/raw/op_test.go | 64 +++++++ server/pkg/server/server.go | 16 +- server/pkg/server/wire.go | 14 +- server/pkg/ylayer/const.go | 13 -- server/pkg/ylayer/doc.go | 198 -------------------- server/pkg/ylayer/doc_test.go | 223 ---------------------- server/pkg/ylayer/gumnut.go | 278 ---------------------------- server/pkg/ylayer/gumnut_test.go | 150 --------------- server/pkg/ylayer/node.go | 217 ---------------------- server/pkg/ylayer/patch.go | 147 --------------- server/pkg/ylayer/patch_test.go | 122 ------------ server/pkg/ylayer/types.go | 33 ---- server/pkg/ymodel/op.go | 182 ------------------ server/pkg/ymodel/prior.go | 91 --------- server/pkg/ymodel/prior_test.go | 57 ------ server/pkg/ymodel/sets.go | 128 ------------- server/pkg/ymodel/sets_test.go | 173 ----------------- 30 files changed, 498 insertions(+), 2118 deletions(-) create mode 100644 demo-encode.ts create mode 100644 server/pkg/model/node/patch_test.go delete mode 100644 server/pkg/ylayer/const.go delete mode 100644 server/pkg/ylayer/doc.go delete mode 100644 server/pkg/ylayer/doc_test.go delete mode 100644 server/pkg/ylayer/gumnut.go delete mode 100644 server/pkg/ylayer/gumnut_test.go delete mode 100644 server/pkg/ylayer/node.go delete mode 100644 server/pkg/ylayer/patch.go delete mode 100644 server/pkg/ylayer/patch_test.go delete mode 100644 server/pkg/ylayer/types.go delete mode 100644 server/pkg/ymodel/op.go delete mode 100644 server/pkg/ymodel/prior.go delete mode 100644 server/pkg/ymodel/prior_test.go delete mode 100644 server/pkg/ymodel/sets.go delete mode 100644 server/pkg/ymodel/sets_test.go diff --git a/client/lib/gumnut/floor.ts b/client/lib/gumnut/floor.ts index 9bae07f..5e31b85 100644 --- a/client/lib/gumnut/floor.ts +++ b/client/lib/gumnut/floor.ts @@ -1,15 +1,7 @@ import { mapRecord, newChannel } from 'thorish'; import { GumnutLow } from '../ymodel/api.ts'; import type { Id } from '../ymodel/internal/shared.ts'; -import { - decodeOpRun, - decodeSetOp, - encodePatch, - type Op, - type SetPart, - type Patch, - type WirePatch, -} from '../ymodel/types.ts'; +import { encodePatch, type Patch, type WirePatch, decodePatch } from '../ymodel/types.ts'; import { emptySymbol, InoPosRef, InoPosRefImpl, type GumnutData } from './types.ts'; import type { PatchApi } from '../ymodel/work-holder.ts'; import type { RangeUpdate } from '../ymodel/internal/range.ts'; @@ -27,22 +19,22 @@ class GumnutFloorApi extends GumnutLow { return data; case 'bigint': - return { p: `b:${data.toString()}` }; + return { b: data.toString() }; case 'number': if (!isFinite(data)) { if (data === +Infinity) { - return { p: '+Inf' }; + return { inf: 1 }; } else if (data === -Infinity) { - return { p: '-Inf' }; + return { inf: -1 }; } else { - return { p: 'NaN' }; + return { inf: 0 }; } } return data; case 'undefined': - return { p: 'undefined' }; + return { x: 'undefined' }; case 'object': if (data === null) { @@ -57,12 +49,14 @@ class GumnutFloorApi extends GumnutLow { throw new Error(`unexpected data: ${JSON.stringify(data)}`); } + // TODO: explicit inode ref + if (data instanceof InoPosRefImpl) { // we need to deref the ID here - on the way up to the server const pos = lookupId(data.target, data.id); - return { s: data.target, t: pos }; + return { i: data.target, p: pos }; } else if (data === emptySymbol) { - return { e: true }; + throw new Error(`can't encode emptySymbol`); } throw new Error(`unexpected data: ${JSON.stringify(data)}`); @@ -85,38 +79,32 @@ class GumnutFloorApi extends GumnutLow { throw new Error(`unexpected raw: ${JSON.stringify(raw)}`); } - if ('e' in raw) { - return emptySymbol; - } else if ('p' in raw) { - switch (raw.p) { - case '+Inf': - return +Infinity; - case '-Inf': - return -Infinity; - case 'NaN': - return NaN; - case 'undefined': - return undefined; - } - if (raw.p.startsWith('b:')) { - return BigInt(raw.p.slice(2)); + if ('inf' in raw) { + if (raw.inf > 0) { + return Infinity; + } else if (raw.inf < 0) { + return -Infinity; + } else { + return NaN; } - - // TODO: unknown special - return emptySymbol; - } else if ('s' in raw) { - if ('t' in raw) { - // we got a real-space pos, convert to ID - const id = resolvePos(raw.s, raw.t); - return new InoPosRefImpl(raw.s, id); + } else if ('b' in raw) { + return BigInt(raw.b); + } else if ('x' in raw) { + if (raw.x === 'undefined') { + return undefined; } - return raw.s; + throw new Error(`unhandled known: ${raw.x}`); + } else if ('i' in raw && 'p' in raw) { + // we got a real-space pos, convert to ID + const id = resolvePos(raw.s, raw.t); + return new InoPosRefImpl(raw.s, id); } + // TODO: explicit inode ref throw new Error(`unexpected raw: ${JSON.stringify(raw)}`); } - protected patchToServer( + protected patchToWire( barrier: string[], update: Record>, lookupId: (inode: string, id: Id) => number, @@ -131,16 +119,12 @@ class GumnutFloorApi extends GumnutLow { return out; } - protected serverToRun(w: WireType): Record { - return mapRecord(w.p, (id, patch) => decodeOpRun(patch.r ?? [])); - } - - protected serverToSet( + protected wireToPatch( w: WireType, resolvePos: (inode: string, pos: number) => Id, - ): Record[]> { + ): Record> { const decodeGumnutData = this.decodeGumnutData.bind(this, resolvePos); - return mapRecord(w.p, (id, patch) => decodeSetOp(patch.s ?? [], decodeGumnutData)); + return mapRecord(w.p, (id, patch) => decodePatch(patch, decodeGumnutData)); } constructor() { diff --git a/client/lib/model/layout.ts b/client/lib/model/layout.ts index 048a9a2..ba92e67 100644 --- a/client/lib/model/layout.ts +++ b/client/lib/model/layout.ts @@ -35,6 +35,7 @@ export function lastLocalId(): number { * This has effectively infinite length and helps track positions, but no actual data. * * Unlike the server, this is ID-based so that end-users don't have to "update" their IDs to fetch new locations. + * Over time, the ID space will inevitably become fragmented. */ export class RopeLayout { private constructor( @@ -51,7 +52,7 @@ export class RopeLayout { ) {} /** - * Creeate a new instance. + * Create a new instance. */ public static new() { return new RopeLayout(); diff --git a/client/lib/ymodel/api.ts b/client/lib/ymodel/api.ts index b40c643..e85d4b9 100644 --- a/client/lib/ymodel/api.ts +++ b/client/lib/ymodel/api.ts @@ -37,18 +37,19 @@ export abstract class GumnutLow { * * This happens inline wrt. the {@link lookupId} function, which returns -ve (self) or +ve (real server). */ - protected abstract patchToServer( + protected abstract patchToWire( barrier: string[], update: Record>, lookupId: (inode: string, id: Id) => number, ): WireType; - protected abstract serverToRun(w: WireType): Record; - - protected abstract serverToSet( + /** + * Converts the wire format to changes. + */ + protected abstract wireToPatch( w: WireType, resolvePos: (inode: string, pos: number) => Id, - ): Record[]>; + ): Record>; private readonly _byIno = new SimpleCache>((ino) => { const state = RopeState.new(this.zeroValue); @@ -144,7 +145,7 @@ export abstract class GumnutLow { private internalPerform(handler: WorkFn): Map { const seq = ++this.transactionSeq; - const pending = new PendingWork(this.patchToServer.bind(this), handler); + const pending = new PendingWork(this.patchToWire.bind(this), handler); // run this _immediately_, since the state on the ino is up-to-date pending.resume(this.holderByIno, this.partialLookupId.bind(this, -1)); @@ -178,6 +179,9 @@ export abstract class GumnutLow { } }; + // FIXME: the reason wireToPatch was in two parts is because we could only resolve sets _once ready_... + const p = this.wireToPatch(update, this.partialLookupId.bind(this, -1)); + // #1: reset all inodes to server state for (const [ino, is] of this._byIno.entries()) { is.state = is.serverState; diff --git a/client/lib/ymodel/types.ts b/client/lib/ymodel/types.ts index 962db58..ad2befb 100644 --- a/client/lib/ymodel/types.ts +++ b/client/lib/ymodel/types.ts @@ -104,31 +104,33 @@ export type Patch = { set: SetPart[]; }; -export type WirePatch = { - r?: number[]; - s?: any[]; -}; +export type WirePatch = any[]; export function encodePatch(patch: Patch, encode: (x: X) => any): WirePatch { - const run = encodeOpRun(patch.run); - const set = encodeSetOp(patch.set, encode); - const out: WirePatch = {}; + const out = encodeSetOp(patch.set, encode); + const run = encodeOpRun(patch.run); if (run.length) { - out.r = run; - } - if (set.length) { - out.s = set; + out.unshift(run); } return out; } export function decodePatch(wire: WirePatch, decode: (raw: any) => X = (x) => x): Patch { - return { - run: wire.r ? decodeOpRun(wire.r) : [], - set: wire.s ? decodeSetOp(wire.s, decode) : [], - }; + const out: Patch = { run: [], set: [] }; + + if (!wire.length) { + return out; + } + if (Array.isArray(wire[0])) { + out.run = decodeOpRun(wire[0]); + wire = wire.slice(1); + } + if (wire.length) { + out.set = decodeSetOp(wire, decode); + } + return out; } /** diff --git a/demo-encode.ts b/demo-encode.ts new file mode 100644 index 0000000..53e77f9 --- /dev/null +++ b/demo-encode.ts @@ -0,0 +1,34 @@ +const lo = 0; +const hi = 1 << 16; // uint16 + +console.info({ lo, hi }); + +// TODO: is 2<<32, takes forever +for (let i = 0; i < hi; ++i) { + for (let j = 0; j < hi; ++j) { + const s = String.fromCharCode(i, j); + if (s.length !== 2) { + throw new Error(`lol`); + } + + if (s.charCodeAt(0) !== i) { + throw new Error(`couldn't store ${i}`); + } + if (s.charCodeAt(1) !== j) { + throw new Error(`couldn't store ${j}`); + } + + const enc = JSON.stringify(s); + const dec = JSON.parse(enc); + + if (s !== dec) { + throw new Error(`can't JSON ${i}`); + } + } + + if (i % 1000 === 0) { + console.info(i); + } +} + +console.info('ok'); diff --git a/server/pkg/model/doc/doc.go b/server/pkg/model/doc/doc.go index 7bc6197..ca1a854 100644 --- a/server/pkg/model/doc/doc.go +++ b/server/pkg/model/doc/doc.go @@ -33,7 +33,12 @@ func (d *docImpl) lazyHolder(ino string) (nh *node.Node) { return } -func (d *docImpl) Apply(work *DocWork) (err error) { +func (d *docImpl) RevVersion() (newVersion int) { + d.version++ + return d.version +} + +func (d *docImpl) Apply(work *Work) (err error) { // ensure not too far ahead if work.TargetVersion > d.version { return ErrVersion @@ -120,8 +125,8 @@ func (d *docImpl) Version() (version int) { return d.version } -func (d *docImpl) Read() (dw *DocWork) { - dw = &DocWork{ +func (d *docImpl) Read() (dw *Work) { + dw = &Work{ TargetVersion: d.version, Update: make(map[string]node.Patch, len(d.byIno)), } diff --git a/server/pkg/model/doc/doc_test.go b/server/pkg/model/doc/doc_test.go index 18455e4..69736d5 100644 --- a/server/pkg/model/doc/doc_test.go +++ b/server/pkg/model/doc/doc_test.go @@ -15,7 +15,7 @@ func intToUint(x int) (out uint64) { func TestPosRef(t *testing.T) { d := New() - err := d.Apply(&DocWork{ + err := d.Apply(&Work{ TargetVersion: 0, Handle: 1, Update: map[string]node.Patch{ @@ -41,7 +41,7 @@ func TestPosRef(t *testing.T) { } // move by inserting new, confirm update - err = d.Apply(&DocWork{ + err = d.Apply(&Work{ TargetVersion: 1, Handle: 2, Update: map[string]node.Patch{ @@ -70,7 +70,7 @@ func TestPosRef(t *testing.T) { func TestInvalidPosRef(t *testing.T) { d := New() - err := d.Apply(&DocWork{ + err := d.Apply(&Work{ TargetVersion: 0, Handle: 1, Update: map[string]node.Patch{ @@ -85,5 +85,38 @@ func TestInvalidPosRef(t *testing.T) { if err != ErrSetOp { t.Errorf("should have failed to apply, was: %v", err) } +} + +func TestTransform(t *testing.T) { + d := New() + d.RevVersion() + + // delete some state + err := d.Apply(&Work{ + TargetVersion: 1, + Handle: 1, + Update: map[string]node.Patch{ + "inode_abc": { + Run: []raw.Op{{Range: []int{0, 100}}}, + }, + }, + }) + if err != nil { + t.Fatalf("couldn't delete as part of xform: %+v", err) + } + + // set within that deleted state (but xformed away) + err = d.Apply(&Work{ + TargetVersion: 1, + Handle: 2, + Update: map[string]node.Patch{ + "inode_abc": { + Set: raw.GumnutSetArray{{Before: 4, String: []uint16{'h', 'i'}}}, + }, + }, + }) + if err != nil { + t.Fatalf("couldn't set xformed away: %+v", err) + } } diff --git a/server/pkg/model/doc/types.go b/server/pkg/model/doc/types.go index 918ac0d..0eb3947 100644 --- a/server/pkg/model/doc/types.go +++ b/server/pkg/model/doc/types.go @@ -10,28 +10,32 @@ type Doc interface { // If an error is returned here, the Doc will be unchanged (but the DocWork may be). // // On a non-error return, the DocWork will contain ops ready-for-broadcast. - // The map entries will match the input, but may not have length. + // This may be less than the input if the operations are transformed away. // // This does not care about barrier ops, the caller must fail before this based on its own version checks. - Apply(work *DocWork) (err error) + Apply(work *Work) (err error) + + // RevVersion bumps the version without doing any work. + // This is unlike Apply, which will not rev the version if nothing occurs. + RevVersion() (newVersion int) // Version returns the high version here. Version() (version int) // Read reads the current status of this Doc. // Internally, this may apply ops forward: we don't always keep a "live" copy around. - Read() (dw *DocWork) + Read() (dw *Work) // Barrier returns true if the given client can pass the barrier with the given inos and their known target version. - // This is effectively checking all the target inos have not changed since ... + // This is effectively checking all the target inos have not changed since the targetVersion. Barrier(targetVersion int, inos []string) (ok bool) // ClearHandle removes the given handle from known history. ClearHandle(handleID int) (change bool) } -type DocWork struct { +type Work struct { TargetVersion int // target version of doc - transform > this (zero => no transform) Handle int // globally unique author of work - Update map[string]node.Patch // mods to be made + Update map[string]node.Patch // mods to be made per-ino } diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index aa5cb9d..51dbe01 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -2,6 +2,7 @@ package node import ( "fmt" + "log" "sort" "github.com/gumnutdev/foiled/server/pkg/model/raw" @@ -115,6 +116,7 @@ func (n *Node) Prepare(arg PrepareArg, p Patch) (tok *PrepareToken, err error) { if len(n.history) != 0 { if arg.TargetVersion > 0 && arg.TargetVersion < lo { + log.Printf("failed to apply: arg.tv=%v lo=%v", arg.TargetVersion, lo) return nil, ErrNodeVersion } } diff --git a/server/pkg/model/node/patch.go b/server/pkg/model/node/patch.go index 70a14da..c21b960 100644 --- a/server/pkg/model/node/patch.go +++ b/server/pkg/model/node/patch.go @@ -1,14 +1,69 @@ package node import ( + "bytes" + "encoding/json" + "github.com/gumnutdev/foiled/server/pkg/model/raw" ) type Patch struct { - Run []raw.Op // structural changes + Run raw.OpRun // structural changes Set raw.GumnutSetArray // sets for data } func (p *Patch) IsZero() (is bool) { return len(p.Run) == 0 && len(p.Set) == 0 } + +func (p Patch) MarshalJSON() (b []byte, err error) { + out := []json.RawMessage{} + + // prefix with array if there's structural changes + if len(p.Run) != 0 { + enc, err := json.Marshal(p.Run) + if err != nil { + return nil, err + } + out = append(out, enc) + } + + // will always start with number + arr, err := p.Set.MarshalJSONArray() + if err != nil { + return nil, err + } + out = append(out, arr...) + + return json.Marshal(out) +} + +func (p *Patch) UnmarshalJSON(b []byte) (err error) { + var tmp []json.RawMessage + err = json.Unmarshal(b, &tmp) + if err != nil { + return err + } + + if len(tmp) == 0 { + *p = Patch{} + return nil + } + + // look for prefix item that is an array + if bytes.HasPrefix(tmp[0], []byte{'['}) { + err = p.Run.UnmarshalJSON(tmp[0]) + if err != nil { + return err + } + tmp = tmp[1:] + } + + // decode sets from remainder + err = p.Set.UnmarshalJSONArray(tmp) + if err != nil { + return err + } + + return +} diff --git a/server/pkg/model/node/patch_test.go b/server/pkg/model/node/patch_test.go new file mode 100644 index 0000000..f8813f6 --- /dev/null +++ b/server/pkg/model/node/patch_test.go @@ -0,0 +1,115 @@ +package node + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/gumnutdev/foiled/server/pkg/model/raw" +) + +func TestPatch_MarshalUnmarshal(t *testing.T) { + cases := []struct { + name string + in Patch + want string + }{ + { + "empty", + Patch{}, + `[]`, + }, + { + "only structural", + Patch{ + Run: raw.OpRun{{Range: []int{5}, Length: 3}}, + }, + `[[-3,5]]`, + }, + { + "only data string", + Patch{ + Set: raw.GumnutSetArray{{Before: 10, String: []uint16{'a', 'b'}}}, + }, + `[10,"ab"]`, + }, + { + "only data raw", + Patch{ + Set: raw.GumnutSetArray{{Before: 10, Data: []raw.GumnutData{{Type: raw.DataTypeNumber, Data: 0x4000000000000000}}}}, // 2.0 + }, + `[10,[2]]`, + }, + { + "both structural and data", + Patch{ + Run: raw.OpRun{{Range: []int{5}, Length: 3}}, + Set: raw.GumnutSetArray{{Before: 10, String: []uint16{'a', 'b'}}}, + }, + `[[-3,5],10,"ab"]`, + }, + { + "multiple data", + Patch{ + Set: raw.GumnutSetArray{ + {Before: 10, String: []uint16{'a'}}, + {Before: 20, String: []uint16{'b'}}, + }, + }, + `[10,"a",20,"b"]`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b, err := json.Marshal(tc.in) + if err != nil { + t.Fatalf("Marshal() err = %v", err) + } + if string(b) != tc.want { + t.Errorf("Marshal() got = %s, want %s", string(b), tc.want) + } + + var got Patch + err = json.Unmarshal(b, &got) + if err != nil { + t.Fatalf("Unmarshal() err = %v", err) + } + + got.Run = normalizeRun(got.Run) + tc.in.Run = normalizeRun(tc.in.Run) + got.Set = normalizeSet(got.Set) + tc.in.Set = normalizeSet(tc.in.Set) + + if !reflect.DeepEqual(got, tc.in) { + t.Errorf("Unmarshal() got = %#v, want %#v", got, tc.in) + } + }) + } +} + +func normalizeRun(r raw.OpRun) raw.OpRun { + if len(r) == 0 { + return nil + } + return r +} + +func normalizeSet(s raw.GumnutSetArray) raw.GumnutSetArray { + if len(s) == 0 { + return nil + } + return s +} + +func TestPatch_IsZero(t *testing.T) { + if !(&Patch{}).IsZero() { + t.Error("expected true for empty Patch") + } + if (&Patch{Run: raw.OpRun{{Range: []int{1}}}}).IsZero() { + t.Error("expected false for Patch with Run") + } + if (&Patch{Set: raw.GumnutSetArray{{Before: 1}}}).IsZero() { + t.Error("expected false for Patch with Set") + } +} diff --git a/server/pkg/model/raw/enc.go b/server/pkg/model/raw/enc.go index 8d8511e..67c32ff 100644 --- a/server/pkg/model/raw/enc.go +++ b/server/pkg/model/raw/enc.go @@ -31,21 +31,33 @@ func (gs *GumnutSet) asData() (gd []GumnutData) { type GumnutSetArray []GumnutSet func (arr GumnutSetArray) MarshalJSON() (b []byte, err error) { - out := []any{} + out, err := arr.MarshalJSONArray() + if err != nil { + return nil, err + } + return json.Marshal(out) +} +func (arr GumnutSetArray) MarshalJSONArray() (out []json.RawMessage, err error) { for _, each := range arr { if each.length() == 0 { continue } + num, err := json.Marshal(each.Before) + if err != nil { + return nil, err + } + b, err := each.internalEncode() if err != nil { return nil, err } - out = append(out, each.Before, json.RawMessage(b)) + + out = append(out, num, json.RawMessage(b)) } - return json.Marshal(out) + return out, nil } func (arr *GumnutSetArray) UnmarshalJSON(b []byte) (err error) { @@ -54,22 +66,25 @@ func (arr *GumnutSetArray) UnmarshalJSON(b []byte) (err error) { if err != nil { return err } + return arr.UnmarshalJSONArray(tmp) +} - if len(tmp)%2 != 0 { +func (arr *GumnutSetArray) UnmarshalJSONArray(raw []json.RawMessage) (err error) { + if len(raw)%2 != 0 { return ErrEncoding } - update := make([]GumnutSet, 0, len(tmp)/2) + update := make([]GumnutSet, 0, len(raw)/2) - for i := 0; i < len(tmp); i += 2 { + for i := 0; i < len(raw); i += 2 { var s GumnutSet - err = json.Unmarshal(tmp[i], &s.Before) + err = json.Unmarshal(raw[i], &s.Before) if err != nil { return err } - err = s.internalDecode(tmp[i+1]) + err = s.internalDecode(raw[i+1]) if err != nil { return err } diff --git a/server/pkg/model/raw/op.go b/server/pkg/model/raw/op.go index d74f51b..8693ba2 100644 --- a/server/pkg/model/raw/op.go +++ b/server/pkg/model/raw/op.go @@ -1,6 +1,7 @@ package raw import ( + "encoding/json" "slices" ) @@ -73,3 +74,76 @@ func (op *Op) Apply() (prior PriorOp, ok bool) { return prior, true } + +// OpRun is a sequence of Op which has a valid JSON encoding. +// Any invalid Op causes this whole encode to be invalid. +type OpRun []Op + +func (or OpRun) MarshalJSON() (b []byte, err error) { + out := make([]int, 0, len(or)*4) + + // encode Run + for _, op := range or { + switch len(op.Range) { + case 1: + if op.Length <= 0 { + continue + } + out = append(out, -op.Length, op.Range[0]) + case 2: + out = append(out, 0, op.Range[0], op.Range[1]) + case 3: + out = append(out, 1, op.Range[0], op.Range[1], op.Range[2]) + } + } + + return json.Marshal(out) +} + +func (or *OpRun) UnmarshalJSON(b []byte) (err error) { + var raw []int + err = json.Unmarshal(b, &raw) + if err != nil { + return + } + + out := make([]Op, 0, len(raw)/4) + + // decode Run + for i := 0; i < len(raw); { + if raw[i] < 0 { + // len(Range) == 1 + if i+1 >= len(raw) { + return ErrEncoding + } + out = append(out, Op{ + Length: -raw[i], + Range: []int{raw[i+1]}, + }) + i += 2 + } else if raw[i] == 0 { + // len(Range) == 2 + if i+2 >= len(raw) { + return ErrEncoding + } + out = append(out, Op{ + Range: []int{raw[i+1], raw[i+2]}, + }) + i += 3 + } else if raw[i] == 1 { + // len(Range) == 3 + if i+3 >= len(raw) { + return ErrEncoding + } + out = append(out, Op{ + Range: []int{raw[i+1], raw[i+2], raw[i+3]}, + }) + i += 4 + } else { + return ErrEncoding + } + } + + *or = out + return +} diff --git a/server/pkg/model/raw/op_test.go b/server/pkg/model/raw/op_test.go index 6b85955..080330e 100644 --- a/server/pkg/model/raw/op_test.go +++ b/server/pkg/model/raw/op_test.go @@ -139,3 +139,67 @@ func TestOp_IsEmpty(t *testing.T) { t.Errorf("expected false for zero length range (invalid op)") } } + +func TestOpRun_MarshalUnmarshal(t *testing.T) { + cases := []struct { + name string + run OpRun + want string + }{ + { + "empty", + OpRun{}, + `[]`, + }, + { + "insert", + OpRun{{Range: []int{5}, Length: 3}}, + `[-3,5]`, + }, + { + "remove", + OpRun{{Range: []int{5, 10}}}, + `[0,5,10]`, + }, + { + "flip", + OpRun{{Range: []int{2, 5, 10}}}, + `[1,2,5,10]`, + }, + { + "multiple", + OpRun{ + {Range: []int{5}, Length: 3}, + {Range: []int{10, 20}}, + {Range: []int{1, 2, 3}}, + }, + `[-3,5,0,10,20,1,1,2,3]`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b, err := tc.run.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON() err = %v", err) + } + if string(b) != tc.want { + t.Errorf("MarshalJSON() got = %s, want %s", string(b), tc.want) + } + + var got OpRun + err = got.UnmarshalJSON(b) + if err != nil { + t.Fatalf("UnmarshalJSON() err = %v", err) + } + + if !reflect.DeepEqual(got, tc.run) { + // Special case for empty because nil vs [] + if len(got) == 0 && len(tc.run) == 0 { + return + } + t.Errorf("UnmarshalJSON() got = %#v, want %#v", got, tc.run) + } + }) + } +} diff --git a/server/pkg/server/server.go b/server/pkg/server/server.go index 81177f6..1f07c97 100644 --- a/server/pkg/server/server.go +++ b/server/pkg/server/server.go @@ -9,22 +9,22 @@ import ( "github.com/gumnutdev/foiled/server/pkg/alloc" "github.com/gumnutdev/foiled/server/pkg/host" - "github.com/gumnutdev/foiled/server/pkg/ylayer" + "github.com/gumnutdev/foiled/server/pkg/model/doc" "github.com/samthor/thorgo/queue" "github.com/samthor/thorgo/transport" "golang.org/x/sync/errgroup" ) type opResult struct { - clientID int // clientID (from higher level) - handleID int // sourcer of ops - seq int // sequence number - dw ylayer.DocWork // work done here + clientID int // clientID (from higher level) + handleID int // sourcer of ops + seq int // sequence number + dw doc.Work // work done here } type safeDoc struct { lock sync.Mutex - doc ylayer.Doc + doc doc.Doc q queue.Queue[opResult] } @@ -39,7 +39,7 @@ func (sd *safeDoc) Perform(in InPacket, handleID, clientID int) (ok bool, err er } // do work normally - work := ylayer.DocWork{ + work := doc.Work{ Handle: handleID, TargetVersion: in.TargetVersion, Update: in.Update, @@ -82,7 +82,7 @@ func (s *Server) getDoc(id string) (sd *safeDoc) { } d := &safeDoc{ q: queue.New[opResult](), - doc: ylayer.NewDoc(), + doc: doc.New(), } s.docs[id] = d return d diff --git a/server/pkg/server/wire.go b/server/pkg/server/wire.go index c40e97b..13f50b6 100644 --- a/server/pkg/server/wire.go +++ b/server/pkg/server/wire.go @@ -1,14 +1,14 @@ package server import ( - "github.com/gumnutdev/foiled/server/pkg/ylayer" + "github.com/gumnutdev/foiled/server/pkg/model/node" ) type InPacket struct { - TargetVersion int `json:"v"` // target version to apply to - Barrier []string `json:"b"` // any inodes to barrier on - Update map[string]ylayer.Patch `json:"p"` // ops to apply - Seq int `json:"seq"` // incrementing seq no# + TargetVersion int `json:"v"` // target version to apply to + Barrier []string `json:"b"` // any inodes to barrier on + Update map[string]node.Patch `json:"p"` // patch to apply + Seq int `json:"seq"` // incrementing seq no# } type AckPacket struct { @@ -17,6 +17,6 @@ type AckPacket struct { } type AnnouncePacket struct { - Version int `json:"v"` // what version this brings us to - Update map[string]ylayer.Patch `json:"p"` // ops for client to apply + Version int `json:"v"` // what version this brings us to + Update map[string]node.Patch `json:"p"` // ops for client to apply } diff --git a/server/pkg/ylayer/const.go b/server/pkg/ylayer/const.go deleted file mode 100644 index c1d0c42..0000000 --- a/server/pkg/ylayer/const.go +++ /dev/null @@ -1,13 +0,0 @@ -package ylayer - -import ( - "errors" -) - -var ( - ErrIno = errors.New("bad ino") - ErrVersion = errors.New("bad target version") - ErrRunOp = errors.New("bad run op, couldn't apply (bad transform?)") - ErrSetOp = errors.New("bad set op, couldn't transform") - ErrRead = errors.New("internal error preparing data") -) diff --git a/server/pkg/ylayer/doc.go b/server/pkg/ylayer/doc.go deleted file mode 100644 index 6694e5c..0000000 --- a/server/pkg/ylayer/doc.go +++ /dev/null @@ -1,198 +0,0 @@ -package ylayer - -import ( - "github.com/gumnutdev/foiled/server/pkg/ymodel" -) - -func NewDoc() (d Doc) { - return &docImpl{ - byIno: map[string]*nodeHolder{}, - active: map[int]int{}, - } -} - -type docImpl struct { - version int // head version (will be max of byIno version) - byIno map[string]*nodeHolder // core inode map - active map[int]int // handle ID to last version ID -} - -// lazyHolder fetches the nodeHolder for the given string ino. -// This may be nil for an invalid string type. -func (d *docImpl) lazyHolder(ino string) (nh *nodeHolder) { - if ino == "" { - return nil - } - - nh = d.byIno[ino] - if nh == nil { - nh = &nodeHolder{doc: d} - d.byIno[ino] = nh - } - return -} - -func (d *docImpl) Apply(work *DocWork) (err error) { - // ensure not too far ahead - if work.TargetVersion > d.version { - return ErrVersion - } - - // fix state for handle'd users - if work.Handle > 0 { - lastActive := d.active[work.Handle] - if work.TargetVersion < lastActive { - err = ErrVersion - return // for some reason the user is targeting in the past for them - } - - // if we are targeting zero but exist here, move target forward - if work.TargetVersion <= 0 { - work.TargetVersion = d.active[work.Handle] - } - } - - arg := prepareArg{ - handle: work.Handle, - targetVersion: work.TargetVersion, - newVersion: d.version + 1, - } - out := make(map[string]prepareResult, len(work.Update)) - - // transform and check all patches - for ino, patch := range work.Update { - nh := d.lazyHolder(ino) - if nh == nil { - return ErrIno - } - - res, err := nh.prepare(arg, &patch) - if err != nil { - return err - } else if res.entry.p.IsEmpty() { - continue // no data here - } - - out[ino] = res - } - - // --- at this point we are going to apply or panic --- - - // record last active version for this handle - if work.Handle > 0 { - handleVersion := work.TargetVersion - if handleVersion <= 0 { - handleVersion = d.version // handle doesn't know about anything before this - } - d.active[work.Handle] = handleVersion - } - - // for some reason we got a no-op - if len(out) == 0 { - work.TargetVersion = d.version - work.Update = nil - return - } - - // set up DocWork as our output location - d.version = arg.newVersion - work.TargetVersion = d.version - work.Update = make(map[string]Patch, len(out)) - - // if all have passed, blat history - for ino, res := range out { - nh := d.byIno[ino] - nh.enact(res) - work.Update[ino] = res.entry.p - } - - // xform sets we did here - // TODO: leaky abstraction, we poke in/out of node - for ino := range out { - nh := d.byIno[ino] - last := &nh.history[len(nh.history)-1] - last.p.Set.IterAll(func(x *GumnutData) { - d.moveForward(x, arg.targetVersion, arg.handle) - }) - } - - return nil -} - -func (d *docImpl) Version() (version int) { - return d.version -} - -func (d *docImpl) Read() (dw *DocWork) { - dw = &DocWork{ - TargetVersion: d.version, - Update: make(map[string]Patch, len(d.byIno)), - } - - for ino, nh := range d.byIno { - gd, err := nh.read() - if err != nil { - panic("couldn't read data - inconsistent") - } - if len(gd) == 0 { - continue - } - - dw.Update[ino] = Patch{ - Run: []ymodel.Op{{Range: []int{0}, Length: len(gd)}}, - Set: ymodel.ManySet[GumnutData]{{Before: len(gd), Body: gd}}, - } - } - - return -} - -func (d *docImpl) Barrier(handle, targetVersion int, inos []string) (ok bool) { - for _, ino := range inos { - nh := d.byIno[ino] - if nh == nil { - continue - } - - if !nh.allowBarrier(handle, targetVersion) { - return false - } - } - return true -} - -func (d *docImpl) ClearHandle(handleID int) (change bool) { - _, ok := d.active[handleID] - if !ok { - return false - } - delete(d.active, handleID) - - for _, nh := range d.byIno { - local := nh.clearHandle(handleID) - change = change || local - } - return change -} - -func (d *docImpl) moveForward(gd *GumnutData, from, handle int) { - if gd.Type != DataTypePosRef { - return - } - - if gd.Data == 0 { - return // can't transform, already at zero - } - - pos := int(gd.Data) - - hist := d.byIno[gd.String].historyFrom(from, handle) - pos, _ = ymodel.TransformOver(pos, hist) - - if pos < 0 { - // FIXME: the client gave us bad data; just reset to zero for other clients - pos = 0 - } - - gd.Data = uint64(pos) -} diff --git a/server/pkg/ylayer/doc_test.go b/server/pkg/ylayer/doc_test.go deleted file mode 100644 index 6cd4826..0000000 --- a/server/pkg/ylayer/doc_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package ylayer - -import ( - "reflect" - "testing" - - "github.com/gumnutdev/foiled/server/pkg/ymodel" -) - -func TestDocApplyAndRead(t *testing.T) { - doc := NewDoc() - - // Insert 5 elements at position 0 - data := DataForString("hello") - - work := &DocWork{ - TargetVersion: 0, - Handle: 123, - Update: map[string]Patch{ - "inode_for_test": { - Run: []ymodel.Op{ - {Range: []int{0}, Length: 5}, - }, - Set: ymodel.ManySet[GumnutData]{ - { - Before: -1, - Body: data, - }, - }, - }, - }, - } - - err := doc.Apply(work) - if err != nil { - t.Fatalf("doc.Apply failed: %v", err) - } - - result := doc.Read() - - patch, ok := result.Update["inode_for_test"] - if !ok { - t.Fatalf("expected inode_for_test in Read() output") - } - - if len(patch.Run) != 1 || patch.Run[0].Length != 5 { - t.Errorf("expected Run to have 5 elements, got %+v", patch.Run) - } - - if len(patch.Set) != 1 { - t.Fatalf("expected 1 set, got %d", len(patch.Set)) - } - - if patch.Set[0].Before != 5 { - t.Errorf("expected Before to be 5, got %d", patch.Set[0].Before) - } - - if !reflect.DeepEqual(patch.Set[0].Body, data) { - t.Errorf("expected data %v, got %v", data, patch.Set[0].Body) - } -} - -func TestDocConcurrentApply(t *testing.T) { - doc := NewDoc() - ino := "concurrent_ino" - - // User A: Insert "A" at 0 - workA := &DocWork{ - TargetVersion: 0, - Handle: 1, - Update: map[string]Patch{ - ino: { - Run: []ymodel.Op{{Range: []int{0}, Length: 1}}, - Set: ymodel.ManySet[GumnutData]{{Before: -1, Body: DataForString("A")}}, - }, - }, - } - - // User B: Insert "B" at 0 - workB := &DocWork{ - TargetVersion: 0, - Handle: 2, - Update: map[string]Patch{ - ino: { - Run: []ymodel.Op{{Range: []int{0}, Length: 1}}, - Set: ymodel.ManySet[GumnutData]{{Before: -1, Body: DataForString("B")}}, - }, - }, - } - - if err := doc.Apply(workA); err != nil { - t.Fatalf("Apply(A) failed: %v", err) - } - if err := doc.Apply(workB); err != nil { - t.Fatalf("Apply(B) failed: %v", err) - } - - result := doc.Read() - patch := result.Update[ino] - - // Convert data to string - var data GumnutDataArray - if len(patch.Set) == 1 { - data = patch.Set[0].Body - } - s := "" - for _, d := range data { - if d.Type != DataTypeJSRune { - t.Fatalf("expected only JSRune, got: %+v", d) - } - s += string(rune(d.Data)) - } - if s != "BA" { - t.Fatalf("unexpected string: %q (expected \"BA\" due to OT)", s) - } -} - -func TestDocConcurrentDelete(t *testing.T) { - doc := NewDoc() - ino := "del_ino" - - // Initial state: "XY" - setup := &DocWork{ - TargetVersion: 0, - Handle: 0, - Update: map[string]Patch{ - ino: { - Run: []ymodel.Op{{Range: []int{0}, Length: 2}}, - Set: ymodel.ManySet[GumnutData]{{Before: -1, Body: DataForString("XY")}}, - }, - }, - } - if err := doc.Apply(setup); err != nil || doc.Version() != 1 { - t.Fatalf("couldn't apply setup op or version bad: err=%v version=%v", err, doc.Version()) - } - - // User A: Delete "X" at 0 - workA := &DocWork{ - TargetVersion: 1, - Handle: 1, - Update: map[string]Patch{ - ino: { - Run: []ymodel.Op{{Range: []int{0, 1}}}, - }, - }, - } - - // User B: Delete "Y" at 1 (thinking doc is "XY") - workB := &DocWork{ - TargetVersion: 1, - Handle: 2, - Update: map[string]Patch{ - ino: { - Run: []ymodel.Op{{Range: []int{1, 2}}}, - }, - }, - } - - if err := doc.Apply(workA); err != nil { - t.Fatalf("Apply(A) failed: %v", err) - } - if err := doc.Apply(workB); err != nil { - t.Fatalf("Apply(B) failed: %v", err) - } - - result := doc.Read() - if len(result.Update) != 0 { - t.Errorf("expected empty update") - } -} - -func TestDataRef(t *testing.T) { - doc := NewDoc() - ino := "demo" - - // Initial state: "XY" - selfPastPos := -2 // ref from back, between ref and 'X' - setup := &DocWork{ - TargetVersion: 0, - Handle: 1, - Update: map[string]Patch{ - ino: { - Run: []ymodel.Op{{Range: []int{0}, Length: 3}}, - Set: ymodel.ManySet[GumnutData]{{Before: -1, Body: []GumnutData{ - {Type: DataTypeJSRune, Data: uint64('X')}, - {Type: DataTypePosRef, Data: uint64(selfPastPos), String: ino}, - {Type: DataTypeJSRune, Data: uint64('Y')}, - }}}, - }, - }, - } - if err := doc.Apply(setup); err != nil || doc.Version() != 1 { - t.Fatalf("couldn't apply setup op or version bad: err=%v version=%v", err, doc.Version()) - } - - task := &DocWork{ - TargetVersion: 1, - Handle: 1, - Update: map[string]Patch{ - ino: { - Run: []ymodel.Op{{Range: []int{0, 1}}}, - }, - }, - } - if err := doc.Apply(task); err != nil { - t.Fatalf("Apply failed: %v", err) - } - - data := doc.Read() - - forIno := data.Update[ino] - if len(forIno.Run) != 1 || forIno.Run[0].Length != 2 { - t.Fatalf("bad forIno") - } - - body := forIno.Set[0].Body - if !reflect.DeepEqual(body, []GumnutData{ - {Type: DataTypePosRef, Data: uint64(1), String: ino}, - {Type: DataTypeJSRune, Data: uint64('Y')}, - }) { - t.Errorf("bad body, got: %+v", body) - } -} diff --git a/server/pkg/ylayer/gumnut.go b/server/pkg/ylayer/gumnut.go deleted file mode 100644 index 15f5847..0000000 --- a/server/pkg/ylayer/gumnut.go +++ /dev/null @@ -1,278 +0,0 @@ -package ylayer - -import ( - "encoding/json" - "errors" - "math" - "unsafe" - - "github.com/gumnutdev/foiled/server/pkg/jsonhelp" -) - -type DataType int - -const ( - DataTypeEmpty DataType = iota // unset/void - distinct from undefined - DataTypeNumber // JS number - DataTypeJSRune // JS UTF-16 part - DataTypeBoolean // JS boolean - DataTypeNull // JS null - DataTypeSpecial // JS 'special', encompasses e.g. NaN, Inf, bigint:... - must be non-zero - DataTypeString // immutable string - DataTypePosRef // ref to inode with pos (str=>inode, pos=>data) -) - -func init() { - var i int - if unsafe.Sizeof(i) != 8 { - panic("must run where int is 8 bytes") - } -} - -// GumnutData represents all possible values inside Gumnut. -type GumnutData struct { - Type DataType // type - Data uint64 // contains number, float bits, ... - String string // string option -} - -// DataForString constructs a slice of GumnutData for the given string. -// This is mostly for testing. -func DataForString(s string) (d []GumnutData) { - runes := jsonhelp.EncodeToUTF16(s) - d = make([]GumnutData, len(runes)) - for i, r := range runes { - d[i].Type = DataTypeJSRune - d[i].Data = uint64(r) - } - return d -} - -// GumnutDataArray contains a run of data inside Gumnut. -// This should be converted to/from JSON, as it includes behavior for runs of JS runes (i.e., strings). -// -// TODO: in the medium-future, we should have fast-paths for []uint16 (string), []float64 (JS numbers), etc. -// This is currently using 32 bytes per entry. -type GumnutDataArray []GumnutData - -func (arr GumnutDataArray) MarshalJSON() (b []byte, err error) { - panic("should not run MarshalJSON") -} - -func (arr GumnutDataArray) buildJSONParts() (out []json.RawMessage, err error) { - // consume runs of DataTypeJSRune _or_ anything else - rest := arr - for len(rest) > 0 { - first := rest[0] - - // generate rune run - if first.Type == DataTypeJSRune { - runes := []uint16{} - for i := 0; i < len(rest); i++ { - if rest[i].Type != DataTypeJSRune { - break - } - runes = append(runes, uint16(rest[i].Data)) - } - rest = rest[len(runes):] - - str := jsonhelp.DecodeFromUTF16(runes) - b, err := json.Marshal(str) - if err != nil { - return nil, err - } - out = append(out, b) - continue - } - - // generate normal data run - internal := []GumnutData{} - for i := 0; i < len(rest); i++ { - if rest[i].Type != first.Type { - break - } - internal = append(internal, rest[i]) - } - rest = rest[len(internal):] - - b, err := json.Marshal(internal) - if err != nil { - return nil, err - } - out = append(out, b) - } - - return out, nil -} - -func (arr *GumnutDataArray) UnmarshalJSON(b []byte) (err error) { - panic("should not run UnmarshalJSON") -} - -func decodeJSONParts(parts []json.RawMessage) (out GumnutDataArray, err error) { - for _, part := range parts { - var gd GumnutDataArray - gd, err = decodeJSONPart(part) - if err != nil { - return nil, err - } - out = append(out, gd...) - } - return out, nil -} - -func decodeJSONPart(b json.RawMessage) (out GumnutDataArray, err error) { - if len(b) == 0 { - return nil, ErrEncoding - } - - switch b[0] { - case '"': - runes := jsonhelp.UnmarshalToUTF16(b) - if runes == nil { - return nil, ErrEncoding - } - out = make([]GumnutData, 0, len(runes)) - for _, r := range runes { - out = append(out, GumnutData{Type: DataTypeJSRune, Data: uint64(r)}) - } - return out, nil - - case '[': - var tmp []GumnutData - err = json.Unmarshal(b, &tmp) - if err != nil { - return nil, err - } - return tmp, nil - } - - return nil, ErrEncoding -} - -func (gd GumnutData) MarshalJSON() (b []byte, err error) { - switch gd.Type { - case DataTypeEmpty: - return []byte(`{"e":true}`), nil - - case DataTypeNumber: - f64 := math.Float64frombits(gd.Data) - return json.Marshal(f64) - - case DataTypeNull: - return []byte("null"), nil - - case DataTypeBoolean: - if gd.Data != 0 { - return []byte("true"), nil - } - return []byte("false"), nil - - case DataTypeSpecial: - var out struct { - Special string `json:"p"` - } - out.Special = gd.String - return json.Marshal(out) - - case DataTypeString: - return json.Marshal(gd.String) - - case DataTypePosRef: - var out struct { - String string `json:"s"` - Target int64 `json:"t"` - } - out.String = gd.String - out.Target = int64(gd.Data) - return json.Marshal(out) - - case DataTypeJSRune: - // can't encode DataTypeJSRune directly - } - - return nil, ErrEncoding -} - -func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { - s := string(b) - - switch s { - case "": - return ErrEncoding - - case "null": - g.Type = DataTypeNull - return nil - - case "true": - g.Type = DataTypeBoolean - g.Data = 1 - return nil - - case "false": - g.Type = DataTypeBoolean - g.Data = 0 - return nil - } - - if s[0] == '"' { - // try string - var str string - err = json.Unmarshal(b, &str) - if err != nil { - return - } - g.Type = DataTypeString - g.String = str - return - } - - if s[0] != '{' { - // try number - var num float64 - err = json.Unmarshal(b, &num) - if err != nil { - return - } - g.Type = DataTypeNumber - g.Data = math.Float64bits(num) - return nil - } - - // otherwise, must be an object type - - var raw struct { - Empty bool `json:"e"` - Special string `json:"p"` - String string `json:"s,omitzero"` // really, inode ref - Target *int64 `json:"t,omitzero"` - } - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - - if raw.Empty { - g.Type = DataTypeEmpty - return nil - } - - if raw.Special != "" { - g.Type = DataTypeSpecial - g.String = raw.Special - return nil - } - - if raw.Target != nil { - g.Type = DataTypePosRef - g.String = raw.String - g.Data = uint64(*raw.Target) - return nil - } - - return ErrEncoding -} - -var ( - ErrEncoding = errors.New("data encoding error") -) diff --git a/server/pkg/ylayer/gumnut_test.go b/server/pkg/ylayer/gumnut_test.go deleted file mode 100644 index 1aad8b1..0000000 --- a/server/pkg/ylayer/gumnut_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package ylayer - -import ( - "encoding/json" - "math" - "reflect" - "testing" -) - -func TestGumnutDataArray_MarshalUnmarshal(t *testing.T) { - tests := []struct { - name string - arr GumnutDataArray - }{ - { - name: "empty", - arr: GumnutDataArray{}, - }, - { - name: "mixed types", - arr: GumnutDataArray{ - {Type: DataTypeNumber, Data: math.Float64bits(1.23)}, - {Type: DataTypeBoolean, Data: 1}, - {Type: DataTypeBoolean, Data: 0}, - {Type: DataTypeNull}, - {Type: DataTypeSpecial, String: "NaN"}, - {Type: DataTypeSpecial, String: "Infinity"}, - {Type: DataTypeString, String: "hello"}, - {Type: DataTypePosRef, String: "inode1", Data: 100}, - }, - }, - { - name: "runs of runes", - arr: GumnutDataArray{ - {Type: DataTypeJSRune, Data: uint64('a')}, - {Type: DataTypeJSRune, Data: uint64('b')}, - {Type: DataTypeJSRune, Data: uint64('c')}, - {Type: DataTypeNumber, Data: math.Float64bits(42)}, - {Type: DataTypeJSRune, Data: uint64('d')}, - {Type: DataTypeJSRune, Data: uint64('e')}, - }, - }, - { - name: "surrogate pairs", - arr: GumnutDataArray{ - {Type: DataTypeJSRune, Data: 0xD83D}, // 😂 (U+1F602) in UTF-16: high surrogate - {Type: DataTypeJSRune, Data: 0xDE02}, // low surrogate - {Type: DataTypeNumber, Data: math.Float64bits(0)}, - {Type: DataTypeJSRune, Data: 0xD83D}, - {Type: DataTypeJSRune, Data: 0xDE0D}, // 😍 - }, - }, - { - name: "single rune", - arr: GumnutDataArray{ - {Type: DataTypeJSRune, Data: uint64('x')}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - parts, err := tt.arr.buildJSONParts() - if err != nil { - t.Fatalf("buildJSONParts failed: %v", err) - } - - got, err := decodeJSONParts(parts) - if err != nil { - t.Fatalf("decodeJSONParts failed: %v", err) - } - - if !reflect.DeepEqual(tt.arr, got) { - // Special check for empty vs nil slice - if len(tt.arr) == 0 && len(got) == 0 { - return - } - t.Errorf("roundtrip failed\nexp: %+v\ngot: %+v", tt.arr, got) - } - }) - } -} - -func TestGumnutDataArray_MarshalRuns(t *testing.T) { - // Specifically test that DataTypeJSRune is marshaled as strings in JSON - arr := GumnutDataArray{ - {Type: DataTypeJSRune, Data: uint64('h')}, - {Type: DataTypeJSRune, Data: uint64('i')}, - {Type: DataTypeNumber, Data: math.Float64bits(1)}, - {Type: DataTypeJSRune, Data: uint64('o')}, - } - - parts, err := arr.buildJSONParts() - if err != nil { - t.Fatal(err) - } - - // render as single array - b, err := json.Marshal(parts) - if err != nil { - t.Fatal(err) - } - - // decode as single array we can check - var raw []any - if err := json.Unmarshal(b, &raw); err != nil { - t.Fatal(err) - } - - if len(raw) != 3 { - t.Errorf("expected 3 elements (string, number, string), got %d: %v", len(raw), raw) - } - - if raw[0] != "hi" { - t.Errorf("expected \"hi\", got %v", raw[0]) - } - if !reflect.DeepEqual(raw[1], []any{1.0}) { - t.Errorf("expected [1], got %+v", raw[1]) - } - if raw[2] != "o" { - t.Errorf("expected \"o\", got %v", raw[2]) - } -} - -func TestGumnutDataArray_UnmarshalStrings(t *testing.T) { - // Specifically test that strings in JSON are unmarshaled back to runs of DataTypeJSRune - input := `["abc", [123], "d"]` - - var parts []json.RawMessage - if err := json.Unmarshal([]byte(input), &parts); err != nil { - t.Fatal(err) - } - - arr, err := decodeJSONParts(parts) - if err != nil { - t.Fatal(err) - } - - expected := GumnutDataArray{ - {Type: DataTypeJSRune, Data: uint64('a')}, - {Type: DataTypeJSRune, Data: uint64('b')}, - {Type: DataTypeJSRune, Data: uint64('c')}, - {Type: DataTypeNumber, Data: math.Float64bits(123)}, - {Type: DataTypeJSRune, Data: uint64('d')}, - } - - if !reflect.DeepEqual(arr, expected) { - t.Errorf("unmarshal failed\nexp: %+v\ngot: %+v", expected, arr) - } -} diff --git a/server/pkg/ylayer/node.go b/server/pkg/ylayer/node.go deleted file mode 100644 index c11d339..0000000 --- a/server/pkg/ylayer/node.go +++ /dev/null @@ -1,217 +0,0 @@ -package ylayer - -import ( - "sort" - - "github.com/gumnutdev/foiled/server/pkg/ymodel" -) - -type historyEntry struct { - p Patch // the op to apply - prior []ymodel.PriorOp // how does that op transform others - handle int // by this user - version int // the version created here -} - -// nodeHolder stores information about a unique inode in Gumnut. -// It's tightly coupled to [docImpl] and is only used internally by that struct. -// -// Notably, nodeHolder really conflates two separate concepts. -// -// 1. The history of changes and transform logic over that -// 2. Rendering of that history, which typically is "behind" the history itself -// -// These could be trivially split out, and this might be helpful at some point. -// We only render history when we have to read a snapshot, _and_ this rendering is always zero state + all history ops. -// So it's functionally an optimization (albeit an incredibly obvious one). -type nodeHolder struct { - doc *docImpl // owner - - dataAt int // when was data last flattened - data GumnutDataArray // data at dataAt - - prune bool // if history is not exhaustive - history []historyEntry // history over time - length int // current length -} - -func (fn *nodeHolder) versionRange() (prune bool, lo, hi int) { - if len(fn.history) != 0 { - lo, hi = fn.history[0].version, fn.history[len(fn.history)-1].version - } - prune = fn.prune - return -} - -// read moves the current "live" version here forward to the given version number. -func (nh *nodeHolder) read() (gd GumnutDataArray, err error) { - if nh.dataAt == nh.doc.version { - return nh.data, nil - } - - idx := sort.Search(len(nh.history), func(i int) (match bool) { - return nh.history[i].version > nh.dataAt - }) - tail := nh.history[idx:] - - // xform all data here _right now_ (TODO: likely be overwritten below) - // we have to do this even without history IN CASE we have a ref - for i := range nh.data { - nh.doc.moveForward(&nh.data[i], nh.dataAt, 0) - } - - // apply self changes - move data forward (might not be anything) - for _, entry := range tail { - for _, op := range entry.p.Run { - update, _, ok := ymodel.ApplySlice(op, nh.data) - if !ok { - return nil, ErrRead - } - nh.data = update - } - - clone := entry.p.Set - - // xform sets from where they were valid to now - clone.IterAll(func(x *GumnutData) { - nh.doc.moveForward(x, entry.version, entry.handle) - }) - ok := clone.ApplySlice(nh.data) - if !ok { - return nil, ErrSetOp - } - } - - // hooray - nh.dataAt = nh.doc.version - return nh.data, nil -} - -func (nh *nodeHolder) clearHandle(handle int) (change bool) { - for i, e := range nh.history { - if e.handle == handle { - nh.history[i].handle = 0 - change = true - } - } - // TODO: we could purge old history (if saved to disk) - - return -} - -// allowBarrier checks that the handle with the given target is "as they see it". -// i.e., there are only ops past the target which the handle created. -func (n *nodeHolder) allowBarrier(handle, targetVersion int) (allow bool) { - prune, lo, _ := n.versionRange() - if prune && targetVersion < lo { - return false - } - - // look for the first entry that is > the known version - idx := sort.Search(len(n.history), func(i int) (match bool) { - return n.history[i].version > targetVersion - }) - - for _, entry := range n.history[idx:] { - if handle != entry.handle { - return false - } - } - return true -} - -func (nh *nodeHolder) historyFrom(version, handle int) (out []ymodel.PriorOpSelf) { - // look for the first entry that is > the known version - idx := sort.Search(len(nh.history), func(i int) (match bool) { - return nh.history[i].version > version - }) - - for _, entry := range nh.history[idx:] { - self := handle > 0 && handle == entry.handle - - for _, pop := range entry.prior { - out = append(out, ymodel.PriorOpSelf{PriorOp: pop, Self: self}) - } - } - return -} - -type prepareArg struct { - handle int - targetVersion int - newVersion int -} - -type prepareResult struct { - length int - targetVersion int - entry historyEntry -} - -// prepare generates the updated state we should apply to this nodeHolder after this Patch. -// This is so doc can apply ops on many nodes at once. -func (nh *nodeHolder) prepare(arg prepareArg, p *Patch) (res prepareResult, err error) { - prune, lo, hi := nh.versionRange() - if prune && arg.targetVersion > 0 && arg.targetVersion < lo { - err = ErrVersion - return // must be within known range - but we don't check hi, assume host has - } - if arg.newVersion <= hi || arg.newVersion <= arg.targetVersion { - err = ErrVersion - return // can't apply at same version - } - - hist := nh.historyFrom(arg.targetVersion, arg.handle) - length := nh.length - ready := &Patch{ - Run: make([]ymodel.Op, 0, len(p.Run)), - Set: p.Set, - } - var entryPrior []ymodel.PriorOp - - // ensure that all of Run is valid - for _, op := range p.Run { - op.TransformOver(hist) - - var prior ymodel.PriorOp - length, prior = ymodel.VirtApplySlice(op, length) - if length < 0 { - err = ErrRunOp - return - } else if prior == nil { - continue // no-op once transformed - } - - if op.IsEmpty() { - continue - } - ready.Run = append(ready.Run, op) - hist = append(hist, ymodel.PriorOpSelf{PriorOp: prior, Self: true}) - entryPrior = append(entryPrior, prior) - } - - // ensure that all of p.Set is valid - if !ready.Set.TransformOver(hist) { - err = ErrSetOp - return - } - - // nb. entry.Set might still contain -ve refs to own work, we fix "externally" (leaky abstraction) - - res = prepareResult{ - length: length, - targetVersion: arg.targetVersion, - entry: historyEntry{ - p: *ready, - handle: arg.handle, - prior: entryPrior, - version: arg.newVersion, - }, - } - return -} - -func (nh *nodeHolder) enact(res prepareResult) { - nh.length = res.length - nh.history = append(nh.history, res.entry) -} diff --git a/server/pkg/ylayer/patch.go b/server/pkg/ylayer/patch.go deleted file mode 100644 index fd6a9ea..0000000 --- a/server/pkg/ylayer/patch.go +++ /dev/null @@ -1,147 +0,0 @@ -package ylayer - -import ( - "encoding/json" - - "github.com/gumnutdev/foiled/server/pkg/ymodel" -) - -// Patch represents a unit of work that is performed on an inode. -// Its ops are either stacked on top of each other (incoming), or to be applied sequentially without transform (outgoing). -type Patch struct { - Run []ymodel.Op - Set ymodel.ManySet[GumnutData] -} - -func (p *Patch) IsEmpty() (is bool) { - return len(p.Run) == 0 && len(p.Set) == 0 -} - -func (p Patch) MarshalJSON() (b []byte, err error) { - var out struct { - Run []int `json:"r,omitzero"` - Set []json.RawMessage `json:"s,omitzero"` - } - - // encode Run - for _, op := range p.Run { - switch len(op.Range) { - case 1: - if op.Length <= 0 { - continue - } - out.Run = append(out.Run, -op.Length, op.Range[0]) - case 2: - out.Run = append(out.Run, 0, op.Range[0], op.Range[1]) - case 3: - out.Run = append(out.Run, 1, op.Range[0], op.Range[1], op.Range[2]) - } - } - - // encode Set - for _, set := range p.Set { - parts, err := GumnutDataArray(set.Body).buildJSONParts() - if err != nil { - return nil, err - } - if len(parts) == 0 { - continue - } - - // because "parts" is []json.RawMessage, encode Before to match - var beforeEnc json.RawMessage - beforeEnc, err = json.Marshal(set.Before) - if err != nil { - return nil, err - } - - out.Set = append(out.Set, beforeEnc) - out.Set = append(out.Set, parts...) - } - - return json.Marshal(out) -} - -func (p *Patch) UnmarshalJSON(b []byte) error { - var raw struct { - Run []int `json:"r"` - Set []json.RawMessage `json:"s"` - } - - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - - // decode Run - for i := 0; i < len(raw.Run); { - if raw.Run[i] < 0 { - // len(Range) == 1 - if i+1 >= len(raw.Run) { - return ErrEncoding - } - p.Run = append(p.Run, ymodel.Op{ - Length: -raw.Run[i], - Range: []int{raw.Run[i+1]}, - }) - i += 2 - } else if raw.Run[i] == 0 { - // len(Range) == 2 - if i+2 >= len(raw.Run) { - return ErrEncoding - } - p.Run = append(p.Run, ymodel.Op{ - Range: []int{raw.Run[i+1], raw.Run[i+2]}, - }) - i += 3 - } else if raw.Run[i] == 1 { - // len(Range) == 3 - if i+3 >= len(raw.Run) { - return ErrEncoding - } - p.Run = append(p.Run, ymodel.Op{ - Range: []int{raw.Run[i+1], raw.Run[i+2], raw.Run[i+3]}, - }) - i += 4 - } else { - return ErrEncoding - } - } - - // decode Set - var i int - for i < len(raw.Set) { - var before int - if err := json.Unmarshal(raw.Set[i], &before); err != nil { - return err - } - i++ - - var tmp []json.RawMessage - - // decode any number of array/string parts - for i < len(raw.Set) { - next := raw.Set[i] - if len(next) == 0 { - return ErrEncoding - } - switch next[0] { - case '"', '[': - tmp = append(tmp, next) - i++ - continue - } - break - } - - out, err := decodeJSONParts(tmp) - if err != nil { - return err - } - p.Set = append(p.Set, ymodel.Set[GumnutData]{ - Before: before, - Body: []GumnutData(out), - }) - } - - return nil -} diff --git a/server/pkg/ylayer/patch_test.go b/server/pkg/ylayer/patch_test.go deleted file mode 100644 index 9c030cd..0000000 --- a/server/pkg/ylayer/patch_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package ylayer - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/gumnutdev/foiled/server/pkg/ymodel" -) - -func TestPatch_MarshalUnmarshal(t *testing.T) { - tests := []struct { - name string - patch Patch - }{ - { - name: "empty", - patch: Patch{}, - }, - { - name: "run operations", - patch: Patch{ - Run: []ymodel.Op{ - {Range: []int{10}, Length: 5}, // Insert 5 at 10 - {Range: []int{20, 30}}, // Remove between 20 and 30 - {Range: []int{40, 50, 60}}, // Flip 40, 50, 60 - }, - }, - }, - { - name: "set operations", - patch: Patch{ - Set: ymodel.ManySet[GumnutData]{ - { - Before: 10, - Body: []GumnutData{ - {Type: DataTypeString, String: "hello"}, - }, - }, - { - Before: 20, - Body: DataForString("world"), - }, - }, - }, - }, - { - name: "combined operations", - patch: Patch{ - Run: []ymodel.Op{ - {Range: []int{0}, Length: 10}, - }, - Set: ymodel.ManySet[GumnutData]{ - { - Before: -1, - Body: DataForString("test"), - }, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b, err := json.Marshal(tt.patch) - if err != nil { - t.Fatalf("MarshalJSON failed: %v", err) - } - - var got Patch - err = json.Unmarshal(b, &got) - if err != nil { - t.Fatalf("UnmarshalJSON failed: %v", err) - } - - // Clean up nil slices which may be unmarshaled as empty slices - if len(got.Run) == 0 && got.Run != nil { - got.Run = nil - } - if len(got.Set) == 0 && got.Set != nil { - got.Set = nil - } - for i := range got.Set { - if len(got.Set[i].Body) == 0 && got.Set[i].Body != nil { - got.Set[i].Body = nil - } - } - - if !reflect.DeepEqual(tt.patch, got) { - t.Errorf("roundtrip failed\nexp: %+v\ngot: %+v", tt.patch, got) - } - }) - } -} - -func TestPatch_MarshalJSONFormat(t *testing.T) { - patch := Patch{ - Run: []ymodel.Op{ - {Range: []int{10}, Length: 5}, - {Range: []int{20, 30}}, - {Range: []int{40, 50, 60}}, - }, - Set: ymodel.ManySet[GumnutData]{ - { - Before: 15, - Body: []GumnutData{ - {Type: DataTypeNull}, - }, - }, - }, - } - - b, err := json.Marshal(patch) - if err != nil { - t.Fatalf("MarshalJSON failed: %v", err) - } - - expected := `{"r":[-5,10,0,20,30,1,40,50,60],"s":[15,[null]]}` - if string(b) != expected { - t.Errorf("unexpected JSON format\nexp: %s\ngot: %s", expected, string(b)) - } -} diff --git a/server/pkg/ylayer/types.go b/server/pkg/ylayer/types.go deleted file mode 100644 index a8496ac..0000000 --- a/server/pkg/ylayer/types.go +++ /dev/null @@ -1,33 +0,0 @@ -package ylayer - -type Doc interface { - // Apply transforms and applies the given DocWork here. - // This checks that all work can be applied before making changes. - // If an error is returned here, the Doc will be unchanged (but the DocWork may be). - // - // On a non-error return, the DocWork will contain ops ready-for-broadcast. - // The map entries will match the input, but may not have length. - // - // This does not care about barrier ops, the caller must fail before this based on its own version checks. - Apply(work *DocWork) (err error) - - // Version returns the high version here. - Version() (version int) - - // Read reads the current status of this Doc. - // Internally, this may apply ops forward: we don't always keep a "live" copy around. - Read() (dw *DocWork) - - // Barrier returns true if the given handle can pass the barrier with the given inos and their known target version. - // This is effectively checking all the target inos have not changed since ... - Barrier(handle, targetVersion int, inos []string) (ok bool) - - // ClearHandle removes the given handle from known history. - ClearHandle(handleID int) (change bool) -} - -type DocWork struct { - TargetVersion int // target version of doc - transform > this (zero => no transform) - Handle int // globally unique author of work - Update map[string]Patch // mods to be made -} diff --git a/server/pkg/ymodel/op.go b/server/pkg/ymodel/op.go deleted file mode 100644 index 2892c98..0000000 --- a/server/pkg/ymodel/op.go +++ /dev/null @@ -1,182 +0,0 @@ -package ymodel - -import ( - "slices" -) - -// Op is an operation, applied in-order, which modifies an array-like. -// -// It is different based on the length of Range: -// 1. insert zero data of Length at the position -// 2. remove between lo,hi (in any order) -// 3. flip around this lo/mid/hi point (in any order) -// -// Any other length is invalid/zero. -type Op struct { - Range []int - Length int -} - -// PriorOpSelf must be passed to TransformOver, to denote whether the prior op was initated by this session. -// Required for transforming inserts. -type PriorOpSelf struct { - PriorOp - Self bool // is this Op created by the same session -} - -// TransformOver transforms the receiver Op over the prior ops. -// This simply nukes the length in an insert if the position is removed. -func (op *Op) TransformOver(prior []PriorOpSelf) { - for i, v := range op.Range { - var gone bool - op.Range[i], gone = TransformOver(v, prior) - if gone { - op.Length = 0 - } - } -} - -func (op *Op) IsEmpty() (ok bool) { - switch len(op.Range) { - case 1: - return op.Length <= 0 - case 2: - return op.Range[0] == op.Range[1] - case 3: - return op.Range[0] == op.Range[1] || op.Range[1] == op.Range[2] - } - return true -} - -// TransformOver transforms a single value over a run of PriorOpSelf. -func TransformOver(v int, over []PriorOpSelf) (out int, gone bool) { - // match -ve focus - if v < 0 { - for i := len(over) - 1; i >= 0; i-- { - if !over[i].Self { - continue - } - - rop, _ := over[i].PriorOp.(ResolveOp) - if rop == nil { - continue - } - - v = rop.ResolvePos(v) - if v >= 0 { - over = over[i+1:] - break - } - } - if v < 0 { - return v, false - } - } - - // now do actual transform - for _, each := range over { - if v == 0 { - break - } - var innerGone bool - v, innerGone = each.TransformPos(v) - gone = gone || innerGone - } - return v, gone -} - -// VirtApplySlice virtually applies this Op to the given slice. -// Returns the updated length, or -1 if invalid. -func VirtApplySlice(op Op, sourceLen int) (outLen int, prior PriorOp) { - outLen = -1 - - for _, v := range op.Range { - if v < 0 || v > sourceLen { - return // out of range - } - } - - r := make([]int, len(op.Range)) - copy(r, op.Range) - slices.Sort(r) - - switch len(r) { - default: - return - - case 1: - if op.Length <= 0 { - break // ok but no-op - } - sourceLen += op.Length - prior = &priorOpInsert{after: r[0], length: op.Length} - - case 2: - if r[0] == r[1] { - break // ok but no-op - } - - sourceLen -= (r[1] - r[0]) - prior = &priorOpRemove{lo: r[0], hi: r[1]} - - case 3: - if r[0] == r[1] || r[1] == r[2] { - break // ok but no-op - } - prior = &priorOpFlip{lo: r[0], mid: r[1], hi: r[2]} - } - - return sourceLen, prior -} - -// ApplySlice applies this Op to the given source. -// Adds zero X in the insert case. -// Returns an updated source. -func ApplySlice[X any](op Op, source []X) (out []X, prior PriorOp, ok bool) { - out = source - - for _, v := range op.Range { - if v < 0 || v > len(out) { - return // out of range - } - } - - r := make([]int, len(op.Range)) - copy(r, op.Range) - slices.Sort(r) - - switch len(r) { - default: - return // invalid - - case 1: - if op.Length <= 0 { - break // ok but no-op - } - - ins := make([]X, op.Length) - out = slices.Insert(out, r[0], ins...) - prior = &priorOpInsert{after: r[0], length: op.Length} - - case 2: - if r[0] == r[1] { - break // ok but no-op - } - - out = slices.Delete(out, r[0], r[1]) - prior = &priorOpRemove{lo: r[0], hi: r[1]} - - case 3: - if r[0] == r[1] || r[1] == r[2] { - break // ok but no-op - } - - extract := out[r[0]:r[1]] - out = slices.Insert(out, r[2], extract...) - out = slices.Delete(out, r[0], r[1]) - prior = &priorOpFlip{lo: r[0], mid: r[1], hi: r[2]} - } - - ok = true - return -} diff --git a/server/pkg/ymodel/prior.go b/server/pkg/ymodel/prior.go deleted file mode 100644 index 89bf1b1..0000000 --- a/server/pkg/ymodel/prior.go +++ /dev/null @@ -1,91 +0,0 @@ -package ymodel - -// PriorOp describes how to transform a position over time. -// Note that pos is 1-indexed. -type PriorOp interface { - TransformPos(pos int) (out int, gone bool) -} - -// ResolveOp is run backwards, and is used to match -ve IDs for user focus. -// Note that pos is 1-indexed, or, likely, -ve indexed when passed here (-1 is in front of last user data inserted). -type ResolveOp interface { - ResolvePos(pos int) (out int) -} - -type priorOpInsert struct { - after, length int -} - -func (p *priorOpInsert) TransformPos(pos int) (out int, gone bool) { - if pos <= p.after { - // before or equal to insert (later ops win) - return pos, false - } else { - // after insert - return pos + p.length, false - } -} - -func (p *priorOpInsert) ResolvePos(pos int) (out int) { - // pos will be -1 and higher for us - if pos >= 0 { - return pos // no resolve here - } - - into := -(pos + 1) - - if into < p.length { - out = p.after + p.length - into - if out < 0 { - panic("bad out") - } - return - } - - pos += p.length - if pos > -1 { - panic("bad pos") - } - return pos -} - -type priorOpRemove struct { - lo, hi int -} - -func (p *priorOpRemove) TransformPos(pos int) (out int, gone bool) { - if pos <= p.lo { - // before range - return pos, false - } else if pos <= p.hi { - // within range - return p.lo, true - } else { - // after range - return pos - (p.hi - p.lo), false - } -} - -type priorOpFlip struct { - lo, mid, hi int -} - -func (p *priorOpFlip) TransformPos(pos int) (out int, gone bool) { - switch { - case pos <= p.lo: - // before - - case pos <= p.mid: - // lo-mid - return pos + (p.hi - p.mid), false - - case pos <= p.hi: - // mid-hi - return pos - (p.mid - p.lo), false - - default: - // after - } - - return pos, false -} diff --git a/server/pkg/ymodel/prior_test.go b/server/pkg/ymodel/prior_test.go deleted file mode 100644 index d124373..0000000 --- a/server/pkg/ymodel/prior_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package ymodel - -import ( - "testing" -) - -func TestPriorFlip(t *testing.T) { - p := &priorOpFlip{lo: 2, mid: 5, hi: 10} - - tests := []struct { - pos int - want int - }{ - {0, 0}, - {1, 1}, - {2, 2}, - {3, 8}, - {4, 9}, - {5, 10}, - {6, 3}, - {7, 4}, - {8, 5}, - {9, 6}, - {10, 7}, - {11, 11}, - } - - for _, tt := range tests { - got, _ := p.TransformPos(tt.pos) - if got != tt.want { - t.Errorf("TransformPos(%d) = %d, want %d", tt.pos, got, tt.want) - } - } -} - -func TestPriorRemove(t *testing.T) { - p := &priorOpRemove{lo: 2, hi: 5} - tests := []struct { - pos int - want int - wantGone bool - }{ - {0, 0, false}, - {1, 1, false}, - {2, 2, false}, - {3, 2, true}, - {4, 2, true}, - {5, 2, true}, - {6, 3, false}, - } - for _, tt := range tests { - got, gotGone := p.TransformPos(tt.pos) - if got != tt.want || gotGone != tt.wantGone { - t.Errorf("TransformPos(%d) = (%d, %v), want (%d, %v)", tt.pos, got, gotGone, tt.want, tt.wantGone) - } - } -} diff --git a/server/pkg/ymodel/sets.go b/server/pkg/ymodel/sets.go deleted file mode 100644 index 438a20d..0000000 --- a/server/pkg/ymodel/sets.go +++ /dev/null @@ -1,128 +0,0 @@ -package ymodel - -import ( - "slices" -) - -// Set describes a single contiguous update to an array-like. -// When transformed, we always use an array of Set, because it's trivial to break apart due to transforms. -type Set[X any] struct { - Before int // set data before this position - Body []X // set this much data -} - -// ManySet is a slice of Set. -type ManySet[X any] []Set[X] - -// IterAll calls the provided function with a reference to every X here. -func (s ManySet[X]) IterAll(fn func(x *X)) { - for i := range s { - set := &s[i] - for i := range set.Body { - fn(&set.Body[i]) - } - } -} - -// ApplySlice applies an already-transformed ManySet to the target. -func (s ManySet[X]) ApplySlice(target []X) (ok bool) { - // check all parts first - for _, part := range s { - low := part.Before - len(part.Body) - if low < 0 { - return false // we extend -ve? - } - } - - // run parts - for _, part := range s { - low := part.Before - len(part.Body) - copy(target[low:], part.Body) - } - return true -} - -// MergeSetOp returns a number of prior transformed SetOp operations. -// This accepts an optional merge function which is called when sets match. -func MergeSetOp[X any](merge func(old, new X) (out X), all ManySet[X]) (out ManySet[X]) { - return internalMerge(all, nil, merge) -} - -// TransformOver updates this SetOp over these operations. -// -// If the SetOp internally has duplicate sets, this will replace older ones (in parts) with newer ones. -// This should be invalid to send over the wire anyway. -func (s *ManySet[X]) TransformOver(prior []PriorOpSelf) (ok bool) { - update := internalMerge(*s, prior, nil) - if update == nil { - return - } - *s = update - return true -} - -func internalMerge[X any](srcParts ManySet[X], prior []PriorOpSelf, merge func(old, new X) (out X)) (out ManySet[X]) { - // for now, we just splay out the data to all their positions and reassemble later - // TODO: we might care one day if we're setting O(millions) - we allocate *X for the maximum position. - var tmp []*X - - for _, part := range srcParts { - if part.Before >= 0 { - low := part.Before - len(part.Body) - if low < 0 { - return nil // we got a +ve set that extended -ve - } - } - - for i, x := range part.Body { - pos := part.Before - len(part.Body) + i + 1 - - var gone bool - pos, gone = TransformOver(pos, prior) - - if pos < 0 { - return nil - } else if gone || pos == 0 { - continue - } - - tmp = extendSlice(tmp, pos+1) - - if merge != nil && tmp[pos] != nil { - x = merge(*tmp[pos], x) - } - tmp[pos] = &x // later sets win, that's fine (user shouldn't send this anyway) - } - } - - out = ManySet[X]{} - - // now find runs and convert to Set - for i := 0; i < len(tmp); i++ { - if tmp[i] == nil { - continue - } - - // consume as many as we can - part := Set[X]{} - - for { - part.Body = append(part.Body, *tmp[i]) - i++ - if i == len(tmp) || tmp[i] == nil { - break - } - } - part.Before = i - 1 - out = append(out, part) - - } - return out -} - -func extendSlice[X any](slice []X, length int) (out []X) { - if len(slice) >= length { - return slice - } - return append(slice, slices.Repeat(make([]X, 1), length-len(slice))...) -} diff --git a/server/pkg/ymodel/sets_test.go b/server/pkg/ymodel/sets_test.go deleted file mode 100644 index 4cce873..0000000 --- a/server/pkg/ymodel/sets_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package ymodel - -import ( - "reflect" - "testing" -) - -func TestManySet(t *testing.T) { - tests := []struct { - name string - start ManySet[int] - prior []PriorOpSelf - expected ManySet[int] - wantErr bool - }{ - { - name: "zero transform just merges inputs", - start: ManySet[int]{ - {Before: 100, Body: []int{1, 2, 3}}, // pos: 98, 99, 100 - {Before: 103, Body: []int{4, 5, 6}}, // pos: 101, 102, 103 - }, - expected: ManySet[int]{{Before: 103, Body: []int{1, 2, 3, 4, 5, 6}}}, - }, - { - name: "transform over remove (self)", - start: ManySet[int]{ - {Before: 100, Body: []int{1, 2, 3}}, // pos: 98, 99, 100 - {Before: 105, Body: []int{4, 5, 6}}, // pos: 103, 104, 105 - }, - prior: []PriorOpSelf{ - { - Self: true, - PriorOp: &priorOpRemove{lo: 100, hi: 102}, - }, - }, - expected: ManySet[int]{{Before: 103, Body: []int{1, 2, 3, 4, 5, 6}}}, - }, - { - name: "transform over remove (other)", - start: ManySet[int]{ - {Before: 100, Body: []int{1, 2, 3}}, - {Before: 105, Body: []int{4, 5, 6}}, - }, - prior: []PriorOpSelf{ - { - Self: false, - PriorOp: &priorOpRemove{lo: 100, hi: 102}, - }, - }, - expected: ManySet[int]{{Before: 103, Body: []int{1, 2, 3, 4, 5, 6}}}, - }, - { - name: "set inside removed range", - start: ManySet[int]{ - {Before: 102, Body: []int{1, 2, 3}}, // pos: 100, 101, 102 - }, - prior: []PriorOpSelf{ - { - Self: true, - PriorOp: &priorOpRemove{lo: 100, hi: 102}, - }, - }, - expected: ManySet[int]{{Before: 100, Body: []int{1}}}, // pos: 100. 101, 102 gone. - }, - { - name: "transform over insert (self)", - start: ManySet[int]{ - {Before: 100, Body: []int{1, 2, 3}}, // pos: 98, 99, 100 - {Before: 105, Body: []int{4, 5, 6}}, // pos: 103, 104, 105 - }, - prior: []PriorOpSelf{ - { - Self: true, - PriorOp: &priorOpInsert{after: 101, length: 2}, - }, - }, - expected: ManySet[int]{ - {Before: 100, Body: []int{1, 2, 3}}, // pos 98, 99, 100 (not shifted) - {Before: 107, Body: []int{4, 5, 6}}, // pos 105, 106, 107 - }, - }, - { - name: "-ve addressing own insert", - start: ManySet[int]{ - {Before: -1, Body: []int{9}}, // pos = -1 - }, - prior: []PriorOpSelf{ - { - Self: true, - PriorOp: &priorOpInsert{after: 10, length: 5}, - }, - }, - expected: ManySet[int]{ - {Before: 15, Body: []int{9}}, // ResolvePos(-1) -> 15. - }, - }, - { - name: "-ve addressing own insert multiple", - start: ManySet[int]{ - {Before: -2, Body: []int{80, 90}}, // pos: -3, -2 - }, - prior: []PriorOpSelf{ - { - Self: true, - PriorOp: &priorOpInsert{after: 10, length: 5}, - }, - }, - expected: ManySet[int]{ - {Before: 14, Body: []int{80, 90}}, // -3->13 (80), -2->14 (90) - }, - }, - { - name: "-ve addressing with remove", - start: ManySet[int]{ - {Before: -2, Body: []int{80, 90}}, // pos: -3, -2 - }, - prior: []PriorOpSelf{ - { - Self: true, - PriorOp: &priorOpInsert{after: 10, length: 5}, // pos becomes 13, 14 - }, - { - Self: false, - PriorOp: &priorOpRemove{lo: 9, hi: 13}, - }, - }, - expected: ManySet[int]{ - {Before: 10, Body: []int{90}}, // -2->14->10 (90) - }, - }, - { - name: "-ve addressing other insert fails", - start: ManySet[int]{ - {Before: -1, Body: []int{9}}, // pos = -1 - }, - prior: []PriorOpSelf{ - { - Self: false, - PriorOp: &priorOpInsert{after: 10, length: 5}, - }, - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := make(ManySet[int], len(tt.start)) - copy(s, tt.start) - - ok := s.TransformOver(tt.prior) - if tt.wantErr { - if ok { - t.Errorf("expected error/false, got true") - } - return - } - - if !ok { - t.Fatalf("TransformOver failed unexpectedly") - } - - // nil expected means we expect it to be empty but let's compare length - if len(tt.expected) == 0 && len(s) == 0 { - return // both empty - } - - if !reflect.DeepEqual(s, tt.expected) { - t.Errorf("expected %+v, got %+v", tt.expected, s) - } - }) - } -} From 69972cc30d19683941f490074b1255c33c4208c0 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sat, 4 Apr 2026 18:12:59 +1100 Subject: [PATCH 11/22] barrier fix --- server/pkg/model/doc/doc.go | 31 +++++++----------- server/pkg/model/doc/types.go | 6 ++-- server/pkg/model/node/node.go | 51 ++++++++++++++++++++---------- server/pkg/model/node/node_test.go | 2 +- 4 files changed, 49 insertions(+), 41 deletions(-) diff --git a/server/pkg/model/doc/doc.go b/server/pkg/model/doc/doc.go index ca1a854..ad6df14 100644 --- a/server/pkg/model/doc/doc.go +++ b/server/pkg/model/doc/doc.go @@ -142,38 +142,29 @@ func (d *docImpl) Read() (dw *Work) { return } -// inoToVersion returns the high version for the specific inode. -// This is for the caller to check whether work can be applied (check barrier). -func (d *docImpl) inoToVersion(ino string) (version int) { - nh := d.byIno[ino] - if nh == nil { - return 0 - } - _, version = nh.VersionRange() - return -} - -func (d *docImpl) Barrier(targetVersion int, inos []string) (ok bool) { - if targetVersion <= 0 { - return true - } +func (d *docImpl) Barrier(handle, targetVersion int, inos []string) (ok bool) { for _, ino := range inos { - if d.inoToVersion(ino) > targetVersion { + nh := d.byIno[ino] + if nh == nil { + continue + } + + if !nh.AllowBarrier(handle, targetVersion) { return false } } return true } -func (d *docImpl) ClearHandle(handleID int) (change bool) { - _, ok := d.active[handleID] +func (d *docImpl) ClearHandle(handle int) (change bool) { + _, ok := d.active[handle] if !ok { return false } - delete(d.active, handleID) + delete(d.active, handle) for _, nh := range d.byIno { - local := nh.ClearHandle(handleID) + local := nh.ClearHandle(handle) change = change || local } return change diff --git a/server/pkg/model/doc/types.go b/server/pkg/model/doc/types.go index 0eb3947..4aad668 100644 --- a/server/pkg/model/doc/types.go +++ b/server/pkg/model/doc/types.go @@ -26,12 +26,12 @@ type Doc interface { // Internally, this may apply ops forward: we don't always keep a "live" copy around. Read() (dw *Work) - // Barrier returns true if the given client can pass the barrier with the given inos and their known target version. + // Barrier returns true if the given handle can pass the barrier with the given inos and their known target version. // This is effectively checking all the target inos have not changed since the targetVersion. - Barrier(targetVersion int, inos []string) (ok bool) + Barrier(handle, targetVersion int, inos []string) (ok bool) // ClearHandle removes the given handle from known history. - ClearHandle(handleID int) (change bool) + ClearHandle(handle int) (change bool) } type Work struct { diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index 51dbe01..7f8741a 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -2,7 +2,6 @@ package node import ( "fmt" - "log" "sort" "github.com/gumnutdev/foiled/server/pkg/model/raw" @@ -19,8 +18,10 @@ import ( // // It knows about the handle who performed history operations, so it is exposed to end-clients. type Node struct { - set raw.GumnutSetArray // data here - readAt int // when we last read/flattened data + set raw.GumnutSetArray // data here + readAt int // when we last read/flattened data + + prune bool // if history is not exhaustive history []historyEntry Transform TransformHandleFunc // must be set to xform data @@ -51,18 +52,18 @@ func (n *Node) ClearHandle(handle int) (change bool) { return } -func (n *Node) VersionRange() (lo, hi int) { +func (n *Node) versionRange() (prune bool, lo, hi int) { if len(n.history) != 0 { - lo = n.history[0].entry.Version - hi = n.history[len(n.history)-1].entry.Version + lo, hi = n.history[0].entry.Version, n.history[len(n.history)-1].entry.Version } + prune = n.prune return } // Read reads the content of this node. // This will transform data forward as needed. func (n *Node) Read(docVersion int) (sm raw.GumnutSetArray) { - _, hi := n.VersionRange() + _, _, hi := n.versionRange() docVersion = max(hi, docVersion) // log.Printf("reading %v -> %v", n.readAt, docVersion) @@ -108,20 +109,36 @@ type PrepareArg struct { NewVersion int } +// allowBarrier checks that the handle with the given target is "as they see it". +// i.e., there are only ops past the target which the handle created. +func (n *Node) AllowBarrier(handle, targetVersion int) (allow bool) { + prune, lo, _ := n.versionRange() + if prune && targetVersion < lo { + return false + } + + // look for the first entry that is > the known version + idx := sort.Search(len(n.history), func(i int) (match bool) { + return n.history[i].entry.Version > targetVersion + }) + + for _, entry := range n.history[idx:] { + if handle != entry.handle { + return false + } + } + return true +} + // Prepare accepts the user's work on this node and applies it to the history. // It returns a PrepareToken which must later be finalized or rolled back. // The PrepareToken can be nil if this is a no-op. func (n *Node) Prepare(arg PrepareArg, p Patch) (tok *PrepareToken, err error) { - lo, hi := n.VersionRange() - - if len(n.history) != 0 { - if arg.TargetVersion > 0 && arg.TargetVersion < lo { - log.Printf("failed to apply: arg.tv=%v lo=%v", arg.TargetVersion, lo) - return nil, ErrNodeVersion - } - } - if arg.NewVersion <= max(hi, n.readAt) { - return nil, ErrNodeVersion + prune, lo, hi := n.versionRange() + if prune && arg.TargetVersion > 0 && arg.TargetVersion < lo { + return nil, ErrNodeVersion // must be within known range - but we don't check hi, assume host has + } else if arg.NewVersion <= max(hi, n.readAt) { + return nil, ErrNodeVersion // can't apply at same version } hist := n.historyFrom(arg.TargetVersion, arg.Handle) diff --git a/server/pkg/model/node/node_test.go b/server/pkg/model/node/node_test.go index 521b82c..6866307 100644 --- a/server/pkg/model/node/node_test.go +++ b/server/pkg/model/node/node_test.go @@ -12,7 +12,7 @@ func TestNode(t *testing.T) { n := &Node{} mustEnact := func(handle, targetVersion int, patch Patch) (out Patch) { - _, hi := n.VersionRange() + _, _, hi := n.versionRange() tok, err := n.Prepare(PrepareArg{ Handle: handle, From 5607b32234afc77c5882ad9efde30a9638caf1a9 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sat, 4 Apr 2026 22:26:05 +1100 Subject: [PATCH 12/22] finish up --- client/lib/gumnut/floor.test.ts | 14 ++++++-------- client/lib/gumnut/floor.ts | 9 ++++++--- client/lib/ymodel/api.test.ts | 25 ++++++++++--------------- client/lib/ymodel/api.ts | 21 +++++++++------------ client/lib/ymodel/internal/state.ts | 18 +++++++++++++++--- client/lib/ymodel/types.test.ts | 18 +++++------------- server/pkg/model/doc/doc.go | 2 +- server/pkg/model/node/patch_test.go | 11 +++++++++++ server/pkg/model/raw/gumnut.go | 8 ++++---- 9 files changed, 67 insertions(+), 59 deletions(-) diff --git a/client/lib/gumnut/floor.test.ts b/client/lib/gumnut/floor.test.ts index dbb8534..02ae282 100644 --- a/client/lib/gumnut/floor.test.ts +++ b/client/lib/gumnut/floor.test.ts @@ -7,10 +7,7 @@ test('floor', () => { const changes1 = f.update({ p: { - def: { - r: [-5, 0], - s: [5, 'hello'], - }, + def: [[-5, 0], 5, 'hello'], }, }); assert.deepStrictEqual(f.byIno('def').string(), 'hello'); @@ -25,10 +22,11 @@ test('floor', () => { const changes3 = f.update({ p: { - def: { - r: [0, 2, 5], // delete "llo" - s: [2, 'i'], - }, + def: [ + [0, 2, 5], // delete "llo" + 2, + 'i', + ], }, }); assert.deepStrictEqual(f.byIno('def').string(), 'hi'); diff --git a/client/lib/gumnut/floor.ts b/client/lib/gumnut/floor.ts index 5e31b85..364daa0 100644 --- a/client/lib/gumnut/floor.ts +++ b/client/lib/gumnut/floor.ts @@ -15,6 +15,8 @@ class GumnutFloorApi extends GumnutLow { private encodeGumnutData(lookupId: (inode: string, id: Id) => number, data: GumnutData) { switch (typeof data) { case 'string': // intern string + return { s: data }; + case 'boolean': // true/false return data; @@ -64,7 +66,6 @@ class GumnutFloorApi extends GumnutLow { public decodeGumnutData(resolvePos: (inode: string, pos: number) => Id, raw: any): GumnutData { switch (typeof raw) { - case 'string': // intern string case 'boolean': // true/false case 'number': return raw; @@ -96,8 +97,10 @@ class GumnutFloorApi extends GumnutLow { throw new Error(`unhandled known: ${raw.x}`); } else if ('i' in raw && 'p' in raw) { // we got a real-space pos, convert to ID - const id = resolvePos(raw.s, raw.t); - return new InoPosRefImpl(raw.s, id); + const id = resolvePos(raw.i, raw.p); + return new InoPosRefImpl(raw.i, id); + } else if ('s' in raw) { + return raw.s; } // TODO: explicit inode ref diff --git a/client/lib/ymodel/api.test.ts b/client/lib/ymodel/api.test.ts index 0c7e9b4..d2ffa9c 100644 --- a/client/lib/ymodel/api.test.ts +++ b/client/lib/ymodel/api.test.ts @@ -2,8 +2,7 @@ import test from 'node:test'; import * as assert from 'node:assert'; import { GumnutLow } from './api.ts'; import type { Id } from './internal/shared.ts'; -import type { Op, Patch, SetPart } from './types.ts'; -import { mapRecord } from 'thorish'; +import type { Patch } from './types.ts'; type TestWireType = { barrier?: string[]; update: Record> }; @@ -12,24 +11,20 @@ class TestGumnutLow extends GumnutLow { super(0); } - public patchToServer( + protected wireToPatch( + w: TestWireType, + resolvePos: (inode: string, pos: number) => Id, + ): Record> { + return w.update; + } + + protected patchToWire( barrier: string[], update: Record>, lookupId: (inode: string, id: Id) => number, - ) { + ): TestWireType { return { barrier, update }; } - - protected serverToRun(w: TestWireType): Record { - return mapRecord(w.update, (ino, patch) => patch.run); - } - - protected serverToSet( - w: TestWireType, - resolvePos: (inode: string, pos: number) => Id, - ): Record[]> { - return mapRecord(w.update, (ino, patch) => patch.set); - } } test('api', () => { diff --git a/client/lib/ymodel/api.ts b/client/lib/ymodel/api.ts index e85d4b9..9962584 100644 --- a/client/lib/ymodel/api.ts +++ b/client/lib/ymodel/api.ts @@ -1,8 +1,8 @@ import { SimpleCache } from 'thorish'; import { type PatchApi, type ReadApi, WorkHolder } from './work-holder.ts'; import { RopeState, type InternalPatch } from './internal/state.ts'; -import type { Op, Patch, SetPart } from './types.ts'; -import type { Id } from './internal/shared.ts'; +import type { Patch } from './types.ts'; +import { zeroId, type Id } from './internal/shared.ts'; import { accumulateRangeUpdate, invertRangeUpdate, type RangeUpdate } from './internal/range.ts'; type InoState = { @@ -179,9 +179,6 @@ export abstract class GumnutLow { } }; - // FIXME: the reason wireToPatch was in two parts is because we could only resolve sets _once ready_... - const p = this.wireToPatch(update, this.partialLookupId.bind(this, -1)); - // #1: reset all inodes to server state for (const [ino, is] of this._byIno.entries()) { is.state = is.serverState; @@ -193,27 +190,27 @@ export abstract class GumnutLow { } // #2a: enact structural changes - const ops = this.serverToRun(update); - for (const ino in ops) { + const p = this.wireToPatch(update, () => zeroId); // nb. we do this twice but first time don't resolve - sets are "at end" + for (const ino in p) { const is = this._byIno.get(ino); const target = is.serverState; - ops[ino].forEach((op) => { + p[ino].run.forEach((op) => { const update = target.applyOp(target.coerceOp(op)); accumulate(ino, update); }); } // #3a: enact sets - const sets = this.serverToSet(update, (ino, pos) => { + const ps = this.wireToPatch(update, (ino, pos) => { const is = this._byIno.get(ino); return is.serverState.posToId(pos); }); - for (const ino in sets) { + for (const ino in ps) { const is = this._byIno.get(ino); const target = is.serverState; - sets[ino].forEach((set) => { + ps[ino].set.forEach((set) => { const update = target.applySet(target.coerceSet(set)); accumulate(ino, update); }); @@ -221,7 +218,7 @@ export abstract class GumnutLow { // #3: reapply all local this.pendingWork.forEach(({ pending, seq }) => { - const match = pending.barrier().some((ino) => ino in ops || ino in sets); + const match = pending.barrier().some((ino) => ino in ps); // #3a: resume code to give it a chance to change its mind // (otherwise we just apply already known prior ops) diff --git a/client/lib/ymodel/internal/state.ts b/client/lib/ymodel/internal/state.ts index 3ffd845..f953e98 100644 --- a/client/lib/ymodel/internal/state.ts +++ b/client/lib/ymodel/internal/state.ts @@ -152,12 +152,24 @@ export class RopeState { /** * Converts a public position, in the range `[0,length]`, to an internal {@link Id}. * - * Throws if out of range. + * Throws if negative, as we can expand this state forever. */ public posToId(pos: number): Id { - if (pos < 0 || pos > this.rope.length()) { - throw new Error(`can't pos out of range: ${pos} vs ${this.rope.length()}`); + if (pos < 0) { + throw new Error(`can't lookup -ve pos`); } + if (pos === 0) { + return zeroId; + } + + if (pos > this.rope.length()) { + const extra = pos - this.rope.length(); + const newId = allocLocalId(extra); + this.rope.insertAfter(this.rope.last(), newId, extra, extra); + this.tree.insert(newId); + return newId; + } + const lookup = this.rope.byPosition(pos); // O(logn-ish) return (lookup.id - lookup.offset) as Id; } diff --git a/client/lib/ymodel/types.test.ts b/client/lib/ymodel/types.test.ts index 7c7c226..63063c7 100644 --- a/client/lib/ymodel/types.test.ts +++ b/client/lib/ymodel/types.test.ts @@ -9,17 +9,11 @@ test('encodePatch', () => { }; const wp = encodePatch(p, (x) => x); - assert.deepStrictEqual(wp, { - r: [-2, 5], - s: [1234, 'hello 🗺️'], - }); + assert.deepStrictEqual(wp, [[-2, 5], 1234, 'hello 🗺️']); }); test('decodePatch', () => { - const wp: WirePatch = { - r: [-2, 5], - s: [1234, 'hello 🗺️'], - }; + const wp: WirePatch = [[-2, 5], 1234, 'hello 🗺️']; const p = decodePatch(wp, (x) => x as string); assert.deepStrictEqual(p, { @@ -35,11 +29,11 @@ test('empty arrays are omitted', () => { }; const wp = encodePatch(p, (x) => x); - assert.deepStrictEqual(wp, {}); + assert.deepStrictEqual(wp, []); }); test('missing arrays are handled', () => { - const wp: WirePatch = {}; + const wp: WirePatch = []; const p = decodePatch(wp, (x) => x as string); assert.deepStrictEqual(p, { @@ -58,9 +52,7 @@ test('removes and flips', () => { }; const wp = encodePatch(p, (x) => x); - assert.deepStrictEqual(wp, { - r: [0, 5, 10, 1, 1, 2, 3], - }); + assert.deepStrictEqual(wp, [[0, 5, 10, 1, 1, 2, 3]]); const p2 = decodePatch(wp, (x) => x as string); assert.deepStrictEqual(p2, p); diff --git a/server/pkg/model/doc/doc.go b/server/pkg/model/doc/doc.go index ad6df14..27b4029 100644 --- a/server/pkg/model/doc/doc.go +++ b/server/pkg/model/doc/doc.go @@ -180,7 +180,7 @@ func (d *docImpl) moveForward(handle, fromVersion int, gd *raw.GumnutData) (err return // can't transform, already at zero } - pos := int(gd.Data) + pos := int(gd.Data) // stores as uint64 but we change to int64 - user might have given -ve node := d.byIno[gd.String] if node == nil { return // bad node diff --git a/server/pkg/model/node/patch_test.go b/server/pkg/model/node/patch_test.go index f8813f6..c97275d 100644 --- a/server/pkg/model/node/patch_test.go +++ b/server/pkg/model/node/patch_test.go @@ -58,6 +58,17 @@ func TestPatch_MarshalUnmarshal(t *testing.T) { }, `[10,"a",20,"b"]`, }, + { + "sample data", + Patch{ + Run: raw.OpRun{{Range: []int{0}, Length: 2}}, + Set: raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + {Type: raw.DataTypeString, String: "abc"}, + {Type: raw.DataTypeString, String: "hello"}, + }}}, + }, + `[[-2,0],-1,[{"s":"abc"},{"s":"hello"}]]`, + }, } for _, tc := range cases { diff --git a/server/pkg/model/raw/gumnut.go b/server/pkg/model/raw/gumnut.go index 5921e4e..29bcaa9 100644 --- a/server/pkg/model/raw/gumnut.go +++ b/server/pkg/model/raw/gumnut.go @@ -106,10 +106,10 @@ func (gd GumnutData) MarshalJSON() (b []byte, err error) { case DataTypePosRef: var out struct { Inode string `json:"i"` - Pos uint64 `json:"p"` + Pos int64 `json:"p"` } out.Inode = gd.String - out.Pos = uint64(gd.Data) + out.Pos = int64(gd.Data) return json.Marshal(out) } @@ -149,7 +149,7 @@ func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { BigInt string `json:"b"` // bigint value Inf *int `json:"inf"` Inode *string `json:"i"` - Pos *uint64 `json:"p"` + Pos *int64 `json:"p"` } err = json.Unmarshal(b, &tmp) if err != nil { @@ -184,7 +184,7 @@ func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { case tmp.Pos != nil && tmp.Inode != nil: g.Type = DataTypePosRef g.String = *tmp.Inode - g.Data = *tmp.Pos + g.Data = uint64(*tmp.Pos) case tmp.Inode != nil: g.Type = DataTypeInode From c8ba2a575d82dd975c9e38174feea67d47fdd632 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sat, 4 Apr 2026 22:39:29 +1100 Subject: [PATCH 13/22] write back change --- server/pkg/model/node/node.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index 7f8741a..c66a126 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -222,7 +222,8 @@ func (pt *PrepareToken) Finalize() (err error) { if err != nil { return err } - pt.he.entry.Set = set + pt.patch.Set = set // write back for sending to other clients + pt.he.entry.Set = set // write for long-term history return nil } From bf4b9636f7feb4fdbe082973fe4e804810176257 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sat, 4 Apr 2026 22:39:42 +1100 Subject: [PATCH 14/22] fix import --- client/lib/ymodel/internal/state.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/lib/ymodel/internal/state.ts b/client/lib/ymodel/internal/state.ts index f953e98..a3298af 100644 --- a/client/lib/ymodel/internal/state.ts +++ b/client/lib/ymodel/internal/state.ts @@ -1,7 +1,7 @@ import { AATree, Rope } from 'thorish'; import type { Op, SetPart } from '../types.ts'; import { zeroId, allocLocalId, type Id } from './shared.ts'; -import { accumulateRangeUpdate, type RangeUpdate } from './range.ts'; +import type { RangeUpdate } from './range.ts'; type Entry = number | string | X[]; From 11939cb1178d3ec6aaeb32d97ab1f0eabeb0dd57 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sat, 4 Apr 2026 22:46:41 +1100 Subject: [PATCH 15/22] better note about length --- server/pkg/model/node/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index c66a126..e4065c4 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -14,7 +14,7 @@ import ( // - any number of named arrays of data in this inode // // Nodes do not actually have a specific length. -// They are effectively an infinite ticker tape. +// They are considered to have the length to the last non-empty item. // // It knows about the handle who performed history operations, so it is exposed to end-clients. type Node struct { From 266aadd26eeb3a0322ea750f86f8900518746bcb Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sat, 4 Apr 2026 23:54:33 +1100 Subject: [PATCH 16/22] work towards dropping explicit string --- client/lib/ymodel/api.test.ts | 4 - client/lib/ymodel/api.ts | 11 +- client/lib/ymodel/internal/shared.ts | 2 - client/lib/ymodel/internal/state.test.ts | 16 +- client/lib/ymodel/internal/state.ts | 271 ++++------------------- client/lib/ymodel/types.ts | 37 ++-- client/lib/ymodel/work-holder.test.ts | 36 ++- client/lib/ymodel/work-holder.ts | 42 +--- 8 files changed, 109 insertions(+), 310 deletions(-) diff --git a/client/lib/ymodel/api.test.ts b/client/lib/ymodel/api.test.ts index d2ffa9c..0b7a691 100644 --- a/client/lib/ymodel/api.test.ts +++ b/client/lib/ymodel/api.test.ts @@ -7,10 +7,6 @@ import type { Patch } from './types.ts'; type TestWireType = { barrier?: string[]; update: Record> }; class TestGumnutLow extends GumnutLow { - constructor() { - super(0); - } - protected wireToPatch( w: TestWireType, resolvePos: (inode: string, pos: number) => Id, diff --git a/client/lib/ymodel/api.ts b/client/lib/ymodel/api.ts index 9962584..961a462 100644 --- a/client/lib/ymodel/api.ts +++ b/client/lib/ymodel/api.ts @@ -30,7 +30,7 @@ type PatchToServerFn = ( export abstract class GumnutLow { private transactionSeq = 0; - constructor(private readonly zeroValue: X) {} + constructor() {} /** * Converts these changes to the wire format. @@ -52,7 +52,7 @@ export abstract class GumnutLow { ): Record>; private readonly _byIno = new SimpleCache>((ino) => { - const state = RopeState.new(this.zeroValue); + const state = RopeState.new(); const out: InoState = { state, @@ -61,11 +61,8 @@ export abstract class GumnutLow { length() { return out.state.length(); }, - readData(start, end) { - return out.state.readData().slice(start, end); - }, - readString(start, end) { - return out.state.readString().slice(start, end); + read(start, end) { + return out.state.read(start, end); }, posToId(pos: number): Id { return out.state.posToId(pos); diff --git a/client/lib/ymodel/internal/shared.ts b/client/lib/ymodel/internal/shared.ts index 65ccf88..42ed2dd 100644 --- a/client/lib/ymodel/internal/shared.ts +++ b/client/lib/ymodel/internal/shared.ts @@ -1,5 +1,3 @@ -import type { Op } from '../types.ts'; - /** * Doesn't really exist but helps us disambiguate {@link Id}. */ diff --git a/client/lib/ymodel/internal/state.test.ts b/client/lib/ymodel/internal/state.test.ts index 18d3e80..fd830cf 100644 --- a/client/lib/ymodel/internal/state.test.ts +++ b/client/lib/ymodel/internal/state.test.ts @@ -3,18 +3,18 @@ import * as assert from 'node:assert'; import { RopeState } from './state.ts'; test('state', () => { - const r = RopeState.new(0); + const r = RopeState.new(); r.applyOp(r.coerceOp({ r: [0], length: 5 })); - r.applySet(r.coerceSet({ before: 5, body: [[100, 1, 2]] })); - assert.deepStrictEqual(r.readData(), [0, 0, 100, 1, 2]); + r.applySet(r.coerceSet({ before: 5, body: [100, 1, 2] })); - r.applySet(r.coerceSet({ before: 3, body: ['abc'] })); + assert.deepStrictEqual(r.read(), new Array(2).concat(100, 1, 2)); // holy array + + r.applySet(r.coerceSet({ before: 3, body: [97, 98, 99] })); assert.strictEqual(r.length(), 5); - assert.deepStrictEqual(r.readString(), 'abc\ufffd\ufffd'); - assert.deepStrictEqual(r.readData(), [0, 0, 0, 1, 2]); + assert.deepStrictEqual(r.read(), [97, 98, 99, 1, 2]); - r.applySet(r.coerceSet({ before: 4, body: [[999]] })); - assert.deepStrictEqual(r.readData(), [0, 0, 0, 999, 2]); + r.applySet(r.coerceSet({ before: 4, body: [999] })); + assert.deepStrictEqual(r.read(), [97, 98, 99, 999, 2]); }); diff --git a/client/lib/ymodel/internal/state.ts b/client/lib/ymodel/internal/state.ts index a3298af..6b1bb8f 100644 --- a/client/lib/ymodel/internal/state.ts +++ b/client/lib/ymodel/internal/state.ts @@ -3,45 +3,22 @@ import type { Op, SetPart } from '../types.ts'; import { zeroId, allocLocalId, type Id } from './shared.ts'; import type { RangeUpdate } from './range.ts'; -type Entry = number | string | X[]; - -function splitEntryFromEnd(fromEnd: number, e: Entry): [Entry, Entry] { - const length = lengthOfEntry(e); - if (fromEnd <= 0 || fromEnd >= length) { - throw new Error(`split out of range`); - } - - const fromStart = length - fromEnd; - - if (typeof e === 'number') { - return [fromStart, fromEnd]; - } - return [e.slice(0, fromStart), e.slice(fromStart)]; -} - -function lengthOfEntry(e: Entry): number { - return typeof e === 'number' ? e : e.length; -} - /** * Internal state. */ export class RopeState { - private readonly rope: Rope>; + private readonly rope: Rope; private readonly tree: AATree; - public static new(zeroValue: X) { - return new RopeState(zeroValue); + public static new() { + return new RopeState(); } public clone(): RopeState { - return new RopeState(this.zeroValue, this); + return new RopeState(this); } - private constructor( - private readonly zeroValue: X, - other?: RopeState, - ) { + private constructor(other?: RopeState) { if (other) { this.rope = other.rope.clone(); this.tree = other.tree.clone(); @@ -55,50 +32,15 @@ export class RopeState { return this.rope.length(); } - public readString(): string { - const safeData = (d: Entry): string => { - switch (typeof d) { - case 'number': - return ''.padEnd(d, '\ufffd'); - case 'string': - return d; - default: - return ''.padEnd(d.length, '\ufffd'); - } - }; - - const parts: string[] = []; + public read(start?: number, end?: number): X[] { + let out: X[] = []; for (const part of this.rope.iter(zeroId)) { if (part.length) { - parts.push(safeData(part.data)); + out = out.concat(part.data); // concat to retain empty } } - return parts.join(''); - } - public readData(): X[] { - const safeData = (d: Entry): X[] => { - let length: number; - switch (typeof d) { - case 'number': - length = d; // unset data - break; - case 'string': - length = d.length; // TODO: disambiguate string data - e.g. "stringZero" symbol? - break; - default: - return d; - } - return Array(length).fill(this.zeroValue); - }; - - const out: X[] = []; - for (const part of this.rope.iter(zeroId)) { - if (part.length) { - out.push(...safeData(part.data)); - } - } - return out; + return out.slice(start, end); // TODO: do something better than slice at end } /** @@ -165,7 +107,7 @@ export class RopeState { if (pos > this.rope.length()) { const extra = pos - this.rope.length(); const newId = allocLocalId(extra); - this.rope.insertAfter(this.rope.last(), newId, extra, extra); + this.rope.insertAfter(this.rope.last(), newId, extra, new Array(extra)); this.tree.insert(newId); return newId; } @@ -196,11 +138,13 @@ export class RopeState { const exists = host.length !== 0; if (highId !== id) { - const [left, right] = splitEntryFromEnd(host.id - id, host.data); + const fromLeft = host.data.length - (highId - id); + const left = host.data.slice(0, fromLeft); + const right = host.data.slice(fromLeft); // adjust right node, insert new left node before it - this.rope.adjust(host.id, right, exists ? lengthOfEntry(right) : 0); - this.rope.insertAfter(host.prevId, id, exists ? lengthOfEntry(left) : 0, left); + this.rope.adjust(host.id, right, exists ? right.length : 0); + this.rope.insertAfter(host.prevId, id, exists ? left.length : 0, left); this.tree.insert(id); } @@ -229,7 +173,7 @@ export class RopeState { const exists = this.ensureIdEdge(at); - this.rope.insertAfter(at, newId, exists ? length : 0, length); + this.rope.insertAfter(at, newId, exists ? length : 0, new Array(length)); this.tree.insert(newId); if (exists) { @@ -239,7 +183,7 @@ export class RopeState { } /** - * Deletes data from this {@link AbstractRopeState}. + * Deletes data here. * * Returns if a visible change occured. */ @@ -259,7 +203,7 @@ export class RopeState { let cleared = 0; for (const prev of this.rope.iter(lo, hi)) { if (prev.length) { - this.rope.adjust(prev.id, lengthOfEntry(prev.data), 0); + this.rope.adjust(prev.id, prev.data, 0); // TODO: we retain old data, just deleted cleared += prev.length; } } @@ -271,6 +215,9 @@ export class RopeState { return { lo: pos, hi: pos + cleared, length: 0 }; } + /** + * Coerces to an internal op for later consistent apply. + */ public coerceOp(op: Op): InternalOp { const int: InternalOp = { op: structuredClone(op), @@ -314,21 +261,21 @@ export class RopeState { } } + /** + * Coerces to an internal set for later consistent set. + * Returns the 'real-space' {@link RangeUpdate} for this work. + */ public coerceSet(set: SetPart): InternalSet { - let length = 0; - for (const part of set.body) { - length += part.length; - } - const start = set.before - length; + const start = set.before - set.body.length; if (start < 0) { throw new Error(`can't set index<0`); } - // TODO: right now, InteralSet splays out every entry or char with a target ID! const out: InternalSet = { ids: [], data: set.body }; + // TODO: this is now "N sets at every ID" - is there a more efficient way to track it? contiguous IDs? let at = start; - for (let i = 0; i < length; ++i) { + for (let i = 0; i < set.body.length; ++i) { ++at; const id = this.posToId(at); out.ids.push(id); @@ -338,118 +285,37 @@ export class RopeState { } /** - * Applies a single chunk of a {@link InteralSet}, i.e., a string or array chunk. + * Applies the set operation here. + * */ - private internalApplySet(chunk: InternalSetChunk): boolean { - if (chunk.data.length === 0) { - return false; - } - let any = false; - let i = 0; - - // fast-path array inserts - if (typeof chunk.data !== 'string') { - while (i < chunk.ids.length) { - let id = chunk.ids[i]; - - const hostId = this.tree.equalAfter(id); - if (hostId === undefined) { - throw new Error(`panic; bad ID`); - } - - const e = this.rope.lookup(hostId); - if (e.length === 0) { - ++i; // couldn't set, already deleted - continue; - } - - if (typeof e.data === 'number') { - // create new and adjust (e.data isn't live) - e.data = Array(e.data).fill(this.zeroValue); - this.rope.adjust(hostId, e.data, e.data.length); - any = true; - } else if (!Array.isArray(e.data)) { - break; // give up, is a string so we need to split it - } - - const index = e.length - 1 - (hostId - id); - e.data[index] = chunk.data[i]; - any = true; - ++i; - } + public applySet(int: InternalSet): RangeUpdate | undefined { + const pos: number[] = []; - if (i === chunk.ids.length) { - return any; - } + if (int.ids.length !== int.data.length) { + throw new Error(`panic; bad IDs/data mismatch`); } - // consume as much contiguous data and set, rinse and repeat - - while (i < chunk.ids.length) { - let id = chunk.ids[i]; - const lo = i; - - while (++i < chunk.ids.length) { - const nextId = chunk.ids[i]; - if (this.leftOf(nextId) !== id) { - break; - } - id = nextId; - } - let localData = chunk.data.slice(lo, i); - - // work backwards to fill out the space - this.ensureIdEdge(id); // TODO: we always slice at end for now - - while (localData.length) { - const tail = this.rope.lookup(id); - id = tail.prevId; - - if (tail.length === 0) { - // deleted, skip past item length - localData = localData.slice(0, localData.length - lengthOfEntry(tail.data)); - continue; - } - - if (tail.length <= localData.length) { - const update = localData.slice(localData.length - tail.length); - localData = localData.slice(0, localData.length - tail.length); - this.rope.adjust(tail.id, update, tail.length); - any = true; - continue; - } - - this.ensureIdEdge((tail.id - localData.length) as Id); - this.rope.adjust(tail.id, localData, localData.length); - any = true; - break; + int.ids.forEach((id, index) => { + const hostId = this.tree.equalAfter(id)!; + const e = this.rope.lookup(hostId); + if (!e.length) { + return; // deleted } - } - return any; - } - - public applySet(int: InternalSet): RangeUpdate | undefined { - let any = false; + const off = e.data.length - (hostId - id) - 1; + e.data[off] = int.data[index]; - // consume chunks (typically just one!) - const chunks = chunkInternalSet(int); - for (const chunk of chunks) { - const change = this.internalApplySet(chunk); - any ||= change; - } + const at = this.rope.find(hostId) - (hostId - id); + pos.push(at); + }); - if (!any) { + if (!pos.length) { return undefined; } // otherwise, because IDs can be anywhere, ... find the lo/hi // this is stupidly slow because a set might apply in literally any placement after xform - const pos = int.ids.map((id) => this.posOfValid(id)).filter((pos) => pos !== -1); - if (!pos.length) { - throw new Error(`panic; got change set without valid IDs`); - } pos.sort((a, b) => a - b); const lo = pos[0] - 1; @@ -457,38 +323,11 @@ export class RopeState { return { lo, hi, length: hi - lo }; } - - /** - * Finds the {@link Id} directly to the left of the one passed. - * - * This includes now-deleted IDs. - */ - private leftOf(id: Id): Id { - if (id === zeroId) { - throw new Error(`can't get leftOf zero`); - } - - const highId = this.tree.equalAfter(id); - if (!highId) { - throw new Error(`can't find ID: ${id}`); - } - - const info = this.rope.lookup(highId); - const at = highId - id; - const length = lengthOfEntry(info.data); - - if (at === length - 1) { - // use previous entry - return info.prevId; - } - // within run - return (id - 1) as Id; - } } export type InternalOp = { op: Op; id: Id }; -export type InternalSet = { ids: Id[]; data: (X[] | string)[] }; +export type InternalSet = { ids: Id[]; data: X[] }; export type InternalPatch = { run: InternalOp[]; @@ -506,21 +345,3 @@ export function rollbackForOp(int: InternalOp): InternalOp | undefined { }; } } - -export type InternalSetChunk = { ids: Id[]; data: string | X[] }; - -export function chunkInternalSet(int: InternalSet): InternalSetChunk[] { - // consume chunks (typically just one!) - let remainingIds = int.ids; - const chunks: InternalSetChunk[] = []; - - for (let i = 0; i < int.data.length; ++i) { - const data = int.data[i]; - const ids = remainingIds.slice(0, data.length); - remainingIds = remainingIds.slice(data.length); - - chunks.push({ ids, data }); - } - - return chunks; -} diff --git a/client/lib/ymodel/types.ts b/client/lib/ymodel/types.ts index ad2befb..4d185c8 100644 --- a/client/lib/ymodel/types.ts +++ b/client/lib/ymodel/types.ts @@ -46,7 +46,7 @@ export function decodeOpRun(run: number[]): Op[] { export type SetPart = { before: number; - body: (X[] | string)[]; + body: X[]; }; /** @@ -55,7 +55,7 @@ export type SetPart = { export function encodeSetOp(parts: SetPart[], encode: (x: X) => any): any[] { const out: any[] = []; for (const part of parts) { - out.push(part.before, ...part.body.map((p) => (typeof p === 'string' ? p : p.map(encode)))); + out.push(part.before, ...part.body.map(encode)); } return out; } @@ -66,31 +66,22 @@ export function encodeSetOp(parts: SetPart[], encode: (x: X) => any): any[ export function decodeSetOp(parts: any[], decode: (raw: any) => X = (x) => x): SetPart[] { const out: SetPart[] = []; - let i = 0; - while (i < parts.length) { - const before = parts[i] as number; - if (typeof before !== 'number' || Math.floor(before) !== before) { - throw new Error(`can't decode, bad set: ${JSON.stringify(parts)}`); - } - ++i; + if (parts.length % 2) { + throw new Error(`must have even parts in setOp`); + } - const part: SetPart = { before, body: [] }; - out.push(part); + for (let i = 0; i < parts.length; i += 2) { + const before = parts[i]; - // decode any number of array/string parts - while (i < parts.length && typeof parts[i] !== 'number') { - const next = parts[i]; - ++i; - - if (typeof next === 'string') { - part.body.push(next); - } else if (Array.isArray(next)) { - part.body.push(next.map(decode)); - } else { - throw new Error(`bad set part`); - } + const raw = parts[i + 1]; + if (!Array.isArray(raw)) { + throw new Error(`bad setOp`); } + + const part: SetPart = { before, body: raw.map(decode) }; + out.push(part); } + return out; } diff --git a/client/lib/ymodel/work-holder.test.ts b/client/lib/ymodel/work-holder.test.ts index 1ccdbe8..bd1085f 100644 --- a/client/lib/ymodel/work-holder.test.ts +++ b/client/lib/ymodel/work-holder.test.ts @@ -3,8 +3,23 @@ import * as assert from 'node:assert'; import { WorkHolder } from './work-holder.ts'; import { RopeState } from './internal/state.ts'; +/** + * Converts {@link undefined} to a "hole" in an array. + */ +function convertToEmpty(arr: any[]): any[] { + const out = new Array(arr.length); + + arr.forEach((x, i) => { + if (x !== undefined) { + out[i] = x; + } + }); + + return out; +} + test('generic', () => { - let actual = RopeState.new(0); + let actual = RopeState.new(); let calls = 0; const w = new WorkHolder(() => { ++calls; @@ -18,13 +33,12 @@ test('generic', () => { assert.strictEqual(w.length(), 5); assert.strictEqual(calls, 2); - w.applySet({ before: 3, body: [[1, 2, 3]] }); - assert.deepStrictEqual(w.readData(), [1, 2, 3, 0, 0]); + w.applySet({ before: 3, body: [1, 2, 3] }); + assert.deepStrictEqual(w.read(), convertToEmpty([1, 2, 3, undefined, undefined])); // now include some string data - w.applySet({ before: 4, body: ['HI'] }); - assert.deepStrictEqual(w.readData(), [1, 2, 0, 0, 0]); - assert.deepStrictEqual(w.readString(), '\ufffd\ufffdHI\ufffd'); + w.applySet({ before: 4, body: [72, 73] }); + assert.deepStrictEqual(w.read(), convertToEmpty([1, 2, 72, 73, undefined])); let out = w.forServer((id) => { assert.fail('should not call'); @@ -38,13 +52,13 @@ test('generic', () => { assert.deepStrictEqual(out?.set, [ { before: -2, - body: [[1, 2], 'HI'], + body: [1, 2, 72, 73], }, ]); // remove something const remove = w.applyOp({ r: [3, 4] }); - assert.deepStrictEqual(w.readString(), '\ufffd\ufffdH\ufffd'); + assert.deepStrictEqual(w.read(), convertToEmpty([1, 2, 72, undefined])); out = w.forServer((id) => { assert.fail('should not call'); @@ -61,13 +75,13 @@ test('generic', () => { assert.deepStrictEqual(out?.set, [ { before: -3, - body: [[1, 2], 'H'], + body: [1, 2, 72], }, ]); // rollback remove remove.rollback(); - assert.deepStrictEqual(w.readString(), '\ufffd\ufffdHI\ufffd'); + assert.deepStrictEqual(w.read(), convertToEmpty([1, 2, 72, 73, undefined])); out = w.forServer((id) => { assert.fail('should not call'); }); @@ -80,7 +94,7 @@ test('generic', () => { assert.deepStrictEqual(out?.set, [ { before: -2, - body: [[1, 2], 'HI'], + body: [1, 2, 72, 73], }, ]); }); diff --git a/client/lib/ymodel/work-holder.ts b/client/lib/ymodel/work-holder.ts index 1625549..081169c 100644 --- a/client/lib/ymodel/work-holder.ts +++ b/client/lib/ymodel/work-holder.ts @@ -1,7 +1,6 @@ import type { Op, Patch, SetPart } from './types.ts'; import type { Id } from './internal/shared.ts'; import { - chunkInternalSet, type InternalPatch, type InternalSet, rollbackForOp, @@ -11,8 +10,7 @@ import type { RangeUpdate } from './internal/range.ts'; export interface ReadApi { length(): number; - readString(start?: number, end?: number): string; - readData(start?: number, end?: number): X[]; + read(start?: number, end?: number): X[]; posToId(pos: number): Id; posOf(id: Id): number; @@ -148,12 +146,8 @@ export class WorkHolder implements PatchApi { return this.effectiveState().length(); } - readString(start?: number, end?: number): string { - return this.effectiveState().readString().slice(start, end); - } - - readData(start?: number, end?: number): X[] { - return this.effectiveState().readData().slice(start, end); + read(start?: number, end?: number): X[] { + return this.effectiveState().read(start, end); } posToId(pos: number): Id { @@ -265,19 +259,16 @@ function mergePatchSet( combinedLookupId: (consumedLength: number, Id: Id) => number, src: InternalSet[], ) { - const unique = new Map(); + const unique = new Map(); // aggregate first for (const set of src) { - const chunks = chunkInternalSet(set); - for (const chunk of chunks) { - chunk.ids.forEach((id, index) => { - if (m.posOfValid(id) === -1) { - return; // don't send now-deleted sets - } - unique.set(combinedLookupId(0, id), chunk.data.slice(index, index + 1)); - }); - } + set.ids.forEach((id, index) => { + if (m.posOfValid(id) === -1) { + return; // don't send now-deleted sets + } + unique.set(combinedLookupId(0, id), set.data[index]); + }); } // now merge @@ -304,19 +295,10 @@ function mergePatchSet( // we have data of length=i, before - const body: (X[] | string)[] = []; - + const body: X[] = []; for (let j = 0; j < i; ++j) { const data = unique.get(start + j)!; - const last = body.at(-1); - - if (typeof last === 'string' && typeof data === 'string') { - body[body.length - 1] = last + data; - } else if (Array.isArray(last) && Array.isArray(data)) { - body[body.length - 1] = last.concat(data); - } else { - body.push(data); - } + body.push(data); } out.push({ before, body }); From 49e3608379bfbcd30f68a9ee7dff994c28fce352 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sun, 5 Apr 2026 00:11:26 +1100 Subject: [PATCH 17/22] use numbers internally --- client/lib/gumnut/floor.test.ts | 5 +++-- client/lib/gumnut/floor.ts | 29 +++++++++++++----------- client/lib/ymodel/api.test.ts | 32 ++++++++++++++++----------- client/lib/ymodel/helper.ts | 29 ++++++++++++++++++++++++ client/lib/ymodel/types.test.ts | 4 ++-- client/lib/ymodel/types.ts | 5 +++-- client/lib/ymodel/work-holder.test.ts | 16 +------------- 7 files changed, 73 insertions(+), 47 deletions(-) create mode 100644 client/lib/ymodel/helper.ts diff --git a/client/lib/gumnut/floor.test.ts b/client/lib/gumnut/floor.test.ts index 02ae282..90e1c4c 100644 --- a/client/lib/gumnut/floor.test.ts +++ b/client/lib/gumnut/floor.test.ts @@ -1,13 +1,14 @@ import test from 'node:test'; import * as assert from 'node:assert'; import { GumnutFloor } from './floor.ts'; +import { stringToArray } from '../ymodel/helper.ts'; test('floor', () => { const f = new GumnutFloor(); const changes1 = f.update({ p: { - def: [[-5, 0], 5, 'hello'], + def: [[-5, 0], 5, stringToArray('hello')], }, }); assert.deepStrictEqual(f.byIno('def').string(), 'hello'); @@ -25,7 +26,7 @@ test('floor', () => { def: [ [0, 2, 5], // delete "llo" 2, - 'i', + stringToArray('i'), ], }, }); diff --git a/client/lib/gumnut/floor.ts b/client/lib/gumnut/floor.ts index 364daa0..1b852ec 100644 --- a/client/lib/gumnut/floor.ts +++ b/client/lib/gumnut/floor.ts @@ -5,6 +5,7 @@ import { encodePatch, type Patch, type WirePatch, decodePatch } from '../ymodel/ import { emptySymbol, InoPosRef, InoPosRefImpl, type GumnutData } from './types.ts'; import type { PatchApi } from '../ymodel/work-holder.ts'; import type { RangeUpdate } from '../ymodel/internal/range.ts'; +import { arrayToString } from '../ymodel/helper.ts'; type WireType = { b?: string[]; @@ -129,10 +130,6 @@ class GumnutFloorApi extends GumnutLow { const decodeGumnutData = this.decodeGumnutData.bind(this, resolvePos); return mapRecord(w.p, (id, patch) => decodePatch(patch, decodeGumnutData)); } - - constructor() { - super(emptySymbol); - } } export type InoChanges = ReadonlyMap>; @@ -232,13 +229,14 @@ export class GumnutFloor { if (index < 0) { index = internal.length() + index; } - return internal.readData(index, index + 1)[0]; + return internal.read(index, index + 1)[0]; }, data(start, end) { - return internal.readData(start, end); + return internal.read(start, end); }, string(start, end) { - return internal.readString(start, end); + const raw = internal.read(start, end); + return arrayToString(raw); }, refAt(at: number): InoPosRef { const id = internal.posToId(at); @@ -259,13 +257,14 @@ export class GumnutFloor { if (index < 0) { index = internal.length() + index; } - return internal.readData(index, index + 1)[0]; + return internal.read(index, index + 1)[0]; }, data(start, end) { - return internal.readData(start, end); + return internal.read(start, end); }, string(start, end) { - return internal.readString(start, end); + const raw = internal.read(start, end); + return arrayToString(raw); }, refAt(at: number): InoPosRef { const id = internal.posToId(at); @@ -275,7 +274,7 @@ export class GumnutFloor { // TODO: above this is the same as byIno set(before: number, ...data: GumnutData[]): void { - internal.applySet({ before, body: [data] }); + internal.applySet({ before, body: data }); }, dataUpdate(at, deleteCount, ...data) { @@ -284,7 +283,7 @@ export class GumnutFloor { } if (data.length) { internal.applyOp({ r: [at], length: data.length }); - internal.applySet({ before: at + data.length, body: [data] }); + internal.applySet({ before: at + data.length, body: data }); } }, @@ -293,8 +292,12 @@ export class GumnutFloor { internal.applyOp({ r: [at, at + deleteCount] }); } if (str) { + const uint = new Uint16Array(str.length); + for (let i = 0; i < str.length; ++i) { + uint[i] = str.charCodeAt(i); + } internal.applyOp({ r: [at], length: str.length }); - internal.applySet({ before: at + str.length, body: [str] }); + internal.applySet({ before: at + str.length, body: Array.from(uint) }); } }, }; diff --git a/client/lib/ymodel/api.test.ts b/client/lib/ymodel/api.test.ts index 0b7a691..b041590 100644 --- a/client/lib/ymodel/api.test.ts +++ b/client/lib/ymodel/api.test.ts @@ -3,6 +3,7 @@ import * as assert from 'node:assert'; import { GumnutLow } from './api.ts'; import type { Id } from './internal/shared.ts'; import type { Patch } from './types.ts'; +import { convertToEmpty } from './helper.ts'; type TestWireType = { barrier?: string[]; update: Record> }; @@ -34,10 +35,13 @@ test('api', () => { api.applyOp({ r: [0], length: 7 }); const id = api.posToId(4); - const xxSet = api.applySet({ before: api.posOf(id), body: ['XX'] }); + const xxSet = api.applySet({ before: api.posOf(id), body: [88, 88] }); // 'XX' assert.strictEqual(api.length(), 7); - assert.deepStrictEqual(api.readString(), '\ufffd\ufffdXX\ufffd\ufffd\ufffd'); + assert.deepStrictEqual( + api.read(), + convertToEmpty([undefined, undefined, 88, 88, undefined, undefined, undefined]), + ); // shouldn't effect host assert.strictEqual(l.byIno('abc').length(), 0); @@ -45,7 +49,7 @@ test('api', () => { yield ['abc']; xxSet.rollback(); - api.applySet({ before: api.posOf(id), body: ['YY'] }); + api.applySet({ before: api.posOf(id), body: [89, 89] }); // 'YY' ++calls; }); @@ -59,14 +63,16 @@ test('api', () => { update: { abc: { run: [{ r: [0], length: 1 }], - set: [{ before: 1, body: [[999]] }], + set: [{ before: 1, body: [999] }], }, }, }); assert.strictEqual(calls, 2); - assert.deepStrictEqual(l.byIno('abc').readString(), '\ufffd\ufffdYY\ufffd\ufffd\ufffd\ufffd'); - assert.deepStrictEqual(l.byIno('abc').readData(), [0, 0, 0, 0, 0, 0, 0, 999]); + assert.deepStrictEqual( + l.byIno('abc').read(), + convertToEmpty([undefined, undefined, 89, 89, undefined, undefined, undefined, 999]), + ); }); test('prepareForServer', () => { @@ -75,17 +81,17 @@ test('prepareForServer', () => { l.perform(function* (byIno) { const api = byIno('abc'); api.applyOp({ r: [0], length: 2 }); - api.applySet({ before: 2, body: ['XX'] }); + api.applySet({ before: 2, body: [88, 88] }); }); l.perform(function* (byIno) { const api = byIno('abc'); api.applyOp({ r: [1], length: 1 }); - api.applySet({ before: 2, body: ['x'] }); + api.applySet({ before: 2, body: [120] }); }); const outer = l.byIno('abc'); - assert.deepStrictEqual(outer.readString(), 'XxX'); + assert.deepStrictEqual(outer.read(), [88, 120, 88]); assert.deepStrictEqual( l.prepareForServer(), @@ -97,7 +103,7 @@ test('prepareForServer', () => { update: { abc: { run: [{ r: [0], length: 2 }], - set: [{ before: -1, body: ['XX'] }], + set: [{ before: -1, body: [88, 88] }], }, }, }, @@ -109,7 +115,7 @@ test('prepareForServer', () => { update: { abc: { run: [{ r: [-2], length: 1 }], - set: [{ before: -1, body: ['x'] }], + set: [{ before: -1, body: [120] }], }, }, }, @@ -118,11 +124,11 @@ test('prepareForServer', () => { ); l.ackToSeq(1); - assert.deepStrictEqual(outer.readString(), 'XxX'); + assert.deepStrictEqual(outer.read(), [88, 120, 88]); assert.deepStrictEqual([...l.pendingSeq()], [2]); l.ackToSeq(1); l.ackToSeq(2); - assert.deepStrictEqual(outer.readString(), 'XxX'); + assert.deepStrictEqual(outer.read(), [88, 120, 88]); assert.deepStrictEqual([...l.pendingSeq()], []); }); diff --git a/client/lib/ymodel/helper.ts b/client/lib/ymodel/helper.ts new file mode 100644 index 0000000..620dd27 --- /dev/null +++ b/client/lib/ymodel/helper.ts @@ -0,0 +1,29 @@ +/** + * Converts {@link undefined} to a "hole" in an array. + * + * This is mostly for tests. + */ +export function convertToEmpty(arr: any[]): any[] { + const out = new Array(arr.length); + + arr.forEach((x, i) => { + if (x !== undefined) { + out[i] = x; + } + }); + + return out; +} + +export function stringToArray(str: string): number[] { + const out = new Uint16Array(str.length); + for (let i = 0; i < str.length; ++i) { + out[i] = str.charCodeAt(i); + } + return Array.from(out); +} + +export function arrayToString(arr: any[]): string { + const data = new Uint16Array(arr); + return new TextDecoder('utf-16le').decode(data); +} diff --git a/client/lib/ymodel/types.test.ts b/client/lib/ymodel/types.test.ts index 63063c7..00a1d60 100644 --- a/client/lib/ymodel/types.test.ts +++ b/client/lib/ymodel/types.test.ts @@ -9,11 +9,11 @@ test('encodePatch', () => { }; const wp = encodePatch(p, (x) => x); - assert.deepStrictEqual(wp, [[-2, 5], 1234, 'hello 🗺️']); + assert.deepStrictEqual(wp, [[-2, 5], 1234, ['hello 🗺️']]); }); test('decodePatch', () => { - const wp: WirePatch = [[-2, 5], 1234, 'hello 🗺️']; + const wp: WirePatch = [[-2, 5], 1234, ['hello 🗺️']]; const p = decodePatch(wp, (x) => x as string); assert.deepStrictEqual(p, { diff --git a/client/lib/ymodel/types.ts b/client/lib/ymodel/types.ts index 4d185c8..9b4f07e 100644 --- a/client/lib/ymodel/types.ts +++ b/client/lib/ymodel/types.ts @@ -55,7 +55,7 @@ export type SetPart = { export function encodeSetOp(parts: SetPart[], encode: (x: X) => any): any[] { const out: any[] = []; for (const part of parts) { - out.push(part.before, ...part.body.map(encode)); + out.push(part.before, part.body.map(encode)); } return out; } @@ -74,8 +74,9 @@ export function decodeSetOp(parts: any[], decode: (raw: any) => X = (x) => x) const before = parts[i]; const raw = parts[i + 1]; + if (!Array.isArray(raw)) { - throw new Error(`bad setOp`); + throw new Error(`bad setOp: wanted array, was ${JSON.stringify(raw)}`); } const part: SetPart = { before, body: raw.map(decode) }; diff --git a/client/lib/ymodel/work-holder.test.ts b/client/lib/ymodel/work-holder.test.ts index bd1085f..ebadad1 100644 --- a/client/lib/ymodel/work-holder.test.ts +++ b/client/lib/ymodel/work-holder.test.ts @@ -2,21 +2,7 @@ import { test } from 'node:test'; import * as assert from 'node:assert'; import { WorkHolder } from './work-holder.ts'; import { RopeState } from './internal/state.ts'; - -/** - * Converts {@link undefined} to a "hole" in an array. - */ -function convertToEmpty(arr: any[]): any[] { - const out = new Array(arr.length); - - arr.forEach((x, i) => { - if (x !== undefined) { - out[i] = x; - } - }); - - return out; -} +import { convertToEmpty } from './helper.ts'; test('generic', () => { let actual = RopeState.new(); From 6880b8f24fa27964537931f3d1b90d2c699b6679 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sun, 5 Apr 2026 00:13:58 +1100 Subject: [PATCH 18/22] faster coerceSet --- client/lib/ymodel/internal/state.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/client/lib/ymodel/internal/state.ts b/client/lib/ymodel/internal/state.ts index 6b1bb8f..968c7eb 100644 --- a/client/lib/ymodel/internal/state.ts +++ b/client/lib/ymodel/internal/state.ts @@ -271,14 +271,15 @@ export class RopeState { throw new Error(`can't set index<0`); } - const out: InternalSet = { ids: [], data: set.body }; + const out: InternalSet = { ids: new Array(set.body.length), data: set.body }; // TODO: this is now "N sets at every ID" - is there a more efficient way to track it? contiguous IDs? - let at = start; + let at = set.before; for (let i = 0; i < set.body.length; ++i) { - ++at; + // we go backwards to avoid growing N times const id = this.posToId(at); - out.ids.push(id); + out.ids[out.ids.length - i - 1] = id; + --at; } return out; @@ -286,7 +287,7 @@ export class RopeState { /** * Applies the set operation here. - * + * Returns the 'real-space' {@link RangeUpdate} for this work. */ public applySet(int: InternalSet): RangeUpdate | undefined { const pos: number[] = []; From 468bbdef15d734e670bd3e735fded5f65fd8d317 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sun, 5 Apr 2026 00:15:20 +1100 Subject: [PATCH 19/22] fix comment --- client/lib/ymodel/internal/state.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/lib/ymodel/internal/state.ts b/client/lib/ymodel/internal/state.ts index 968c7eb..2528314 100644 --- a/client/lib/ymodel/internal/state.ts +++ b/client/lib/ymodel/internal/state.ts @@ -216,7 +216,7 @@ export class RopeState { } /** - * Coerces to an internal op for later consistent apply. + * Coerces this real-space op to an internal op for later consistent apply. */ public coerceOp(op: Op): InternalOp { const int: InternalOp = { @@ -262,8 +262,7 @@ export class RopeState { } /** - * Coerces to an internal set for later consistent set. - * Returns the 'real-space' {@link RangeUpdate} for this work. + * Coerces this real-space set to an internal set for later consistent set. */ public coerceSet(set: SetPart): InternalSet { const start = set.before - set.body.length; From c75f7b7f6dde74d3c86248f86ee418019d7e8414 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sun, 5 Apr 2026 09:55:17 +1000 Subject: [PATCH 20/22] client-side skip changes, support string DOWN TO client --- client/bin/app.ts | 4 +- client/lib/gumnut/data.ts | 2 +- client/lib/gumnut/floor.test.ts | 8 +- client/lib/gumnut/floor.ts | 114 ++--------------- client/lib/gumnut/wire.ts | 149 +++++++++++++++++++++++ client/lib/ymodel/api.test.ts | 16 +-- client/lib/ymodel/internal/state.test.ts | 6 +- client/lib/ymodel/internal/state.ts | 5 +- client/lib/ymodel/types.test.ts | 14 +-- client/lib/ymodel/types.ts | 57 ++++++--- client/lib/ymodel/work-holder.test.ts | 29 ++++- client/lib/ymodel/work-holder.ts | 21 ++-- tests/integration/basic.test.ts | 2 +- 13 files changed, 265 insertions(+), 162 deletions(-) create mode 100644 client/lib/gumnut/wire.ts diff --git a/client/bin/app.ts b/client/bin/app.ts index 73417db..8f5c383 100755 --- a/client/bin/app.ts +++ b/client/bin/app.ts @@ -78,7 +78,7 @@ e.addListener( pos = 1; } - c.set(pos, text.refAt(e.pos)); + c.set(pos - 1, text.refAt(e.pos)); }); }, c.signal, @@ -106,7 +106,7 @@ e.addListener( } const last = api.at(0); // we just inserted ourself const update = typeof last === 'number' ? last + 1 : 1; - api.set(1, update); + api.set(0, update); console.warn('! setting num', update); // TODO: we're not announcing this in the client anywhere diff --git a/client/lib/gumnut/data.ts b/client/lib/gumnut/data.ts index 766394c..8d53786 100644 --- a/client/lib/gumnut/data.ts +++ b/client/lib/gumnut/data.ts @@ -147,7 +147,7 @@ class DataMapImpl extends DataMap { for (let i = 0; i < d.length; i += 2) { if (k === d[i]) { // found it - ino.set(i + 2, v); + ino.set(i + 1, v); outer.refreshCache(); // TODO: we can be more surgical? return; } diff --git a/client/lib/gumnut/floor.test.ts b/client/lib/gumnut/floor.test.ts index 90e1c4c..435fd9c 100644 --- a/client/lib/gumnut/floor.test.ts +++ b/client/lib/gumnut/floor.test.ts @@ -3,12 +3,12 @@ import * as assert from 'node:assert'; import { GumnutFloor } from './floor.ts'; import { stringToArray } from '../ymodel/helper.ts'; -test('floor', () => { +test('floor server changes', () => { const f = new GumnutFloor(); const changes1 = f.update({ p: { - def: [[-5, 0], 5, stringToArray('hello')], + def: [[-5, 0], 0, 'h', 'ello'], // confirms we can decode strings + multiple parts }, }); assert.deepStrictEqual(f.byIno('def').string(), 'hello'); @@ -25,7 +25,7 @@ test('floor', () => { p: { def: [ [0, 2, 5], // delete "llo" - 2, + 1, stringToArray('i'), ], }, @@ -43,7 +43,7 @@ test('floor local changes', () => { assert.deepStrictEqual(changes1, new Map([['abc', { lo: 0, hi: 0, length: 3 }]])); const changes2 = f.perform(function* (t) { - t.byIno('abc').set(2, 200); + t.byIno('abc').set(1, 200); }); assert.deepStrictEqual(changes2, new Map([['abc', { lo: 1, hi: 2, length: 1 }]])); }); diff --git a/client/lib/gumnut/floor.ts b/client/lib/gumnut/floor.ts index 1b852ec..297373d 100644 --- a/client/lib/gumnut/floor.ts +++ b/client/lib/gumnut/floor.ts @@ -2,10 +2,11 @@ import { mapRecord, newChannel } from 'thorish'; import { GumnutLow } from '../ymodel/api.ts'; import type { Id } from '../ymodel/internal/shared.ts'; import { encodePatch, type Patch, type WirePatch, decodePatch } from '../ymodel/types.ts'; -import { emptySymbol, InoPosRef, InoPosRefImpl, type GumnutData } from './types.ts'; +import { InoPosRef, InoPosRefImpl, type GumnutData } from './types.ts'; import type { PatchApi } from '../ymodel/work-holder.ts'; import type { RangeUpdate } from '../ymodel/internal/range.ts'; import { arrayToString } from '../ymodel/helper.ts'; +import { decodeGumnutDataArray, encodeGumnutDataArray } from './wire.ts'; type WireType = { b?: string[]; @@ -13,108 +14,13 @@ type WireType = { }; class GumnutFloorApi extends GumnutLow { - private encodeGumnutData(lookupId: (inode: string, id: Id) => number, data: GumnutData) { - switch (typeof data) { - case 'string': // intern string - return { s: data }; - - case 'boolean': // true/false - return data; - - case 'bigint': - return { b: data.toString() }; - - case 'number': - if (!isFinite(data)) { - if (data === +Infinity) { - return { inf: 1 }; - } else if (data === -Infinity) { - return { inf: -1 }; - } else { - return { inf: 0 }; - } - } - return data; - - case 'undefined': - return { x: 'undefined' }; - - case 'object': - if (data === null) { - return null; - } - break; - - case 'symbol': - break; - - default: - throw new Error(`unexpected data: ${JSON.stringify(data)}`); - } - - // TODO: explicit inode ref - - if (data instanceof InoPosRefImpl) { - // we need to deref the ID here - on the way up to the server - const pos = lookupId(data.target, data.id); - return { i: data.target, p: pos }; - } else if (data === emptySymbol) { - throw new Error(`can't encode emptySymbol`); - } - - throw new Error(`unexpected data: ${JSON.stringify(data)}`); - } - - public decodeGumnutData(resolvePos: (inode: string, pos: number) => Id, raw: any): GumnutData { - switch (typeof raw) { - case 'boolean': // true/false - case 'number': - return raw; - - case 'object': - if (raw === null) { - return raw; - } - break; - - default: - throw new Error(`unexpected raw: ${JSON.stringify(raw)}`); - } - - if ('inf' in raw) { - if (raw.inf > 0) { - return Infinity; - } else if (raw.inf < 0) { - return -Infinity; - } else { - return NaN; - } - } else if ('b' in raw) { - return BigInt(raw.b); - } else if ('x' in raw) { - if (raw.x === 'undefined') { - return undefined; - } - throw new Error(`unhandled known: ${raw.x}`); - } else if ('i' in raw && 'p' in raw) { - // we got a real-space pos, convert to ID - const id = resolvePos(raw.i, raw.p); - return new InoPosRefImpl(raw.i, id); - } else if ('s' in raw) { - return raw.s; - } - // TODO: explicit inode ref - - throw new Error(`unexpected raw: ${JSON.stringify(raw)}`); - } - protected patchToWire( barrier: string[], update: Record>, lookupId: (inode: string, id: Id) => number, ): WireType { - const encodeGumnutData = this.encodeGumnutData.bind(this, lookupId); - const wireUpdate = mapRecord(update, (ino, patch) => encodePatch(patch, encodeGumnutData)); + const boundEncode = encodeGumnutDataArray.bind(this, lookupId); + const wireUpdate = mapRecord(update, (ino, patch) => encodePatch(patch, boundEncode)); const out: WireType = { p: wireUpdate }; if (barrier.length) { @@ -127,8 +33,8 @@ class GumnutFloorApi extends GumnutLow { w: WireType, resolvePos: (inode: string, pos: number) => Id, ): Record> { - const decodeGumnutData = this.decodeGumnutData.bind(this, resolvePos); - return mapRecord(w.p, (id, patch) => decodePatch(patch, decodeGumnutData)); + const boundDecode = decodeGumnutDataArray.bind(this, resolvePos); + return mapRecord(w.p, (id, patch) => decodePatch(patch, boundDecode)); } } @@ -273,8 +179,8 @@ export class GumnutFloor { // TODO: above this is the same as byIno - set(before: number, ...data: GumnutData[]): void { - internal.applySet({ before, body: data }); + set(at: number, ...data: GumnutData[]): void { + internal.applySet({ skip: at, body: data }); }, dataUpdate(at, deleteCount, ...data) { @@ -283,7 +189,7 @@ export class GumnutFloor { } if (data.length) { internal.applyOp({ r: [at], length: data.length }); - internal.applySet({ before: at + data.length, body: data }); + internal.applySet({ skip: at, body: data }); } }, @@ -297,7 +203,7 @@ export class GumnutFloor { uint[i] = str.charCodeAt(i); } internal.applyOp({ r: [at], length: str.length }); - internal.applySet({ before: at + str.length, body: Array.from(uint) }); + internal.applySet({ skip: at, body: Array.from(uint) }); } }, }; diff --git a/client/lib/gumnut/wire.ts b/client/lib/gumnut/wire.ts new file mode 100644 index 0000000..7c946f5 --- /dev/null +++ b/client/lib/gumnut/wire.ts @@ -0,0 +1,149 @@ +import { stringToArray } from '../ymodel/helper.ts'; +import type { Id } from '../ymodel/internal/shared.ts'; +import { emptySymbol, InoPosRefImpl, type GumnutData } from './types.ts'; + +/** + * Encodes a single {@link GumnutData}. + */ +export function encodeGumnutData(lookupId: (inode: string, id: Id) => number, data: GumnutData) { + switch (typeof data) { + case 'string': // intern string + return { s: data }; + + case 'boolean': // true/false + return data; + + case 'bigint': + return { b: data.toString() }; + + case 'number': + if (!isFinite(data)) { + if (data === +Infinity) { + return { inf: 1 }; + } else if (data === -Infinity) { + return { inf: -1 }; + } else { + return { inf: 0 }; + } + } + return data; + + case 'undefined': + return { x: 'undefined' }; + + case 'object': + if (data === null) { + return null; + } + break; + + case 'symbol': + break; + + default: + throw new Error(`unexpected data: ${JSON.stringify(data)}`); + } + + // TODO: explicit inode ref + + if (data instanceof InoPosRefImpl) { + // we need to deref the ID here - on the way up to the server + const pos = lookupId(data.target, data.id); + return { i: data.target, p: pos }; + } else if (data === emptySymbol) { + throw new Error(`can't encode emptySymbol`); + } + + throw new Error(`unexpected data: ${JSON.stringify(data)}`); +} + +/** + * Encodes a run of {@link GumnutData}. + * + * TODO: for now just coerces to array. + * later: string! + */ +export function encodeGumnutDataArray( + lookupId: (inode: string, id: Id) => number, + arr: GumnutData[], +): any[] { + const encs = arr.map((x) => encodeGumnutData(lookupId, x)); + + // remember: the array itself is the run + // later, we might (?) tease out string-like numbers here and convert to `string` for efficient wire transfer + return [encs]; +} + +/** + * Decodes a single {@link GumnutData}. + */ +export function decodeGumnutData( + resolvePos: (inode: string, pos: number) => Id, + raw: any, +): GumnutData { + switch (typeof raw) { + case 'boolean': // true/false + case 'number': + return raw; + + case 'object': + if (raw === null) { + return raw; + } + break; + + default: + throw new Error(`unexpected raw: ${JSON.stringify(raw)}`); + } + + if ('inf' in raw) { + if (raw.inf > 0) { + return Infinity; + } else if (raw.inf < 0) { + return -Infinity; + } else { + return NaN; + } + } else if ('b' in raw) { + return BigInt(raw.b); + } else if ('x' in raw) { + if (raw.x === 'undefined') { + return undefined; + } + throw new Error(`unhandled known: ${raw.x}`); + } else if ('i' in raw && 'p' in raw) { + // we got a real-space pos, convert to ID + const id = resolvePos(raw.i, raw.p); + return new InoPosRefImpl(raw.i, id); + } else if ('s' in raw) { + return raw.s; + } + // TODO: explicit inode ref + + throw new Error(`unexpected raw: ${JSON.stringify(raw)}`); +} + +/** + * Decodes to a run of {@link GumnutData}. + * + * This supports string wire encoding. + */ +export function decodeGumnutDataArray( + resolvePos: (inode: string, pos: number) => Id, + raw: any[], +): GumnutData[] { + let out: GumnutData[] = []; + + for (const each of raw) { + if (typeof each === 'string') { + out = out.concat(stringToArray(each)); + } else if (Array.isArray(each)) { + const part = each.map((x) => decodeGumnutData(resolvePos, x)); + out = out.concat(part); + } else { + throw new Error(`unknown run part`); + } + } + + return out; +} diff --git a/client/lib/ymodel/api.test.ts b/client/lib/ymodel/api.test.ts index b041590..83a2d09 100644 --- a/client/lib/ymodel/api.test.ts +++ b/client/lib/ymodel/api.test.ts @@ -34,8 +34,8 @@ test('api', () => { const api = byIno('abc'); api.applyOp({ r: [0], length: 7 }); - const id = api.posToId(4); - const xxSet = api.applySet({ before: api.posOf(id), body: [88, 88] }); // 'XX' + const id = api.posToId(2); + const xxSet = api.applySet({ skip: api.posOf(id), body: [88, 88] }); // 'XX' assert.strictEqual(api.length(), 7); assert.deepStrictEqual( @@ -49,7 +49,7 @@ test('api', () => { yield ['abc']; xxSet.rollback(); - api.applySet({ before: api.posOf(id), body: [89, 89] }); // 'YY' + api.applySet({ skip: api.posOf(id), body: [89, 89] }); // 'YY' ++calls; }); @@ -63,7 +63,7 @@ test('api', () => { update: { abc: { run: [{ r: [0], length: 1 }], - set: [{ before: 1, body: [999] }], + set: [{ skip: 0, body: [999] }], }, }, }); @@ -81,13 +81,13 @@ test('prepareForServer', () => { l.perform(function* (byIno) { const api = byIno('abc'); api.applyOp({ r: [0], length: 2 }); - api.applySet({ before: 2, body: [88, 88] }); + api.applySet({ skip: 0, body: [88, 88] }); }); l.perform(function* (byIno) { const api = byIno('abc'); api.applyOp({ r: [1], length: 1 }); - api.applySet({ before: 2, body: [120] }); + api.applySet({ skip: 1, body: [120] }); }); const outer = l.byIno('abc'); @@ -103,7 +103,7 @@ test('prepareForServer', () => { update: { abc: { run: [{ r: [0], length: 2 }], - set: [{ before: -1, body: [88, 88] }], + set: [{ skip: -2, body: [88, 88] }], }, }, }, @@ -115,7 +115,7 @@ test('prepareForServer', () => { update: { abc: { run: [{ r: [-2], length: 1 }], - set: [{ before: -1, body: [120] }], + set: [{ skip: -1, body: [120] }], }, }, }, diff --git a/client/lib/ymodel/internal/state.test.ts b/client/lib/ymodel/internal/state.test.ts index fd830cf..99642a9 100644 --- a/client/lib/ymodel/internal/state.test.ts +++ b/client/lib/ymodel/internal/state.test.ts @@ -6,15 +6,15 @@ test('state', () => { const r = RopeState.new(); r.applyOp(r.coerceOp({ r: [0], length: 5 })); - r.applySet(r.coerceSet({ before: 5, body: [100, 1, 2] })); + r.applySet(r.coerceSet({ skip: 2, body: [100, 1, 2] })); assert.deepStrictEqual(r.read(), new Array(2).concat(100, 1, 2)); // holy array - r.applySet(r.coerceSet({ before: 3, body: [97, 98, 99] })); + r.applySet(r.coerceSet({ skip: 0, body: [97, 98, 99] })); assert.strictEqual(r.length(), 5); assert.deepStrictEqual(r.read(), [97, 98, 99, 1, 2]); - r.applySet(r.coerceSet({ before: 4, body: [999] })); + r.applySet(r.coerceSet({ skip: 3, body: [999] })); assert.deepStrictEqual(r.read(), [97, 98, 99, 999, 2]); }); diff --git a/client/lib/ymodel/internal/state.ts b/client/lib/ymodel/internal/state.ts index 2528314..4242104 100644 --- a/client/lib/ymodel/internal/state.ts +++ b/client/lib/ymodel/internal/state.ts @@ -265,15 +265,14 @@ export class RopeState { * Coerces this real-space set to an internal set for later consistent set. */ public coerceSet(set: SetPart): InternalSet { - const start = set.before - set.body.length; - if (start < 0) { + if (set.skip < 0) { throw new Error(`can't set index<0`); } const out: InternalSet = { ids: new Array(set.body.length), data: set.body }; // TODO: this is now "N sets at every ID" - is there a more efficient way to track it? contiguous IDs? - let at = set.before; + let at = set.skip + set.body.length; for (let i = 0; i < set.body.length; ++i) { // we go backwards to avoid growing N times const id = this.posToId(at); diff --git a/client/lib/ymodel/types.test.ts b/client/lib/ymodel/types.test.ts index 00a1d60..0f951aa 100644 --- a/client/lib/ymodel/types.test.ts +++ b/client/lib/ymodel/types.test.ts @@ -5,20 +5,20 @@ import { type Patch, type WirePatch, encodePatch, decodePatch } from './types.ts test('encodePatch', () => { const p: Patch = { run: [{ r: [5], length: 2 }], - set: [{ before: 1234, body: ['hello 🗺️'] }], + set: [{ skip: 1234, body: ['hello 🗺️', 'there'] }], }; const wp = encodePatch(p, (x) => x); - assert.deepStrictEqual(wp, [[-2, 5], 1234, ['hello 🗺️']]); + assert.deepStrictEqual(wp, [[-2, 5], 1234, 'hello 🗺️', 'there']); }); test('decodePatch', () => { - const wp: WirePatch = [[-2, 5], 1234, ['hello 🗺️']]; + const wp: WirePatch = [[-2, 5], 1234, 'hello 🗺️', 'there']; - const p = decodePatch(wp, (x) => x as string); + const p = decodePatch(wp, (x) => x); assert.deepStrictEqual(p, { run: [{ r: [5], length: 2 }], - set: [{ before: 1234, body: ['hello 🗺️'] }], + set: [{ skip: 1234, body: ['hello 🗺️', 'there'] }], }); }); @@ -35,7 +35,7 @@ test('empty arrays are omitted', () => { test('missing arrays are handled', () => { const wp: WirePatch = []; - const p = decodePatch(wp, (x) => x as string); + const p = decodePatch(wp, (x) => x); assert.deepStrictEqual(p, { run: [], set: [], @@ -54,6 +54,6 @@ test('removes and flips', () => { const wp = encodePatch(p, (x) => x); assert.deepStrictEqual(wp, [[0, 5, 10, 1, 1, 2, 3]]); - const p2 = decodePatch(wp, (x) => x as string); + const p2 = decodePatch(wp, (x) => x); assert.deepStrictEqual(p2, p); }); diff --git a/client/lib/ymodel/types.ts b/client/lib/ymodel/types.ts index 9b4f07e..c0dfc35 100644 --- a/client/lib/ymodel/types.ts +++ b/client/lib/ymodel/types.ts @@ -45,17 +45,37 @@ export function decodeOpRun(run: number[]): Op[] { } export type SetPart = { - before: number; + skip: number; body: X[]; }; /** * Encodes this set op. */ -export function encodeSetOp(parts: SetPart[], encode: (x: X) => any): any[] { +function encodeSetOp(parts: SetPart[], encode: (x: X[]) => any[]): any[] { + if (!parts.length) { + return []; + } const out: any[] = []; + + let first = true; for (const part of parts) { - out.push(part.before, part.body.map(encode)); + const enc = encode(part.body); // TODO: this method could glum together adjacent X[], or do skip resolution + if (!enc.length) { + continue; + } + + // we use numbers to denote skip + for (const check of enc) { + if (typeof check === 'number') { + throw new Error(`can't encode X[] as number, ambiguous`); + } + } + + if (first || part.skip !== 0) { + out.push(part.skip); + } + out.push(...enc); } return out; } @@ -63,24 +83,27 @@ export function encodeSetOp(parts: SetPart[], encode: (x: X) => any): any[ /** * Decodes this set op. */ -export function decodeSetOp(parts: any[], decode: (raw: any) => X = (x) => x): SetPart[] { +function decodeSetOp(parts: any[], decode: (raw: any[]) => X[] = (x) => x): SetPart[] { const out: SetPart[] = []; - if (parts.length % 2) { - throw new Error(`must have even parts in setOp`); - } - - for (let i = 0; i < parts.length; i += 2) { - const before = parts[i]; + for (let i = 0; i < parts.length; ) { + if (typeof parts[i] !== 'number') { + throw new Error(`setOp missing skip number`); + } - const raw = parts[i + 1]; + const skip = parts[i]; + const cand = parts.slice(i + 1); - if (!Array.isArray(raw)) { - throw new Error(`bad setOp: wanted array, was ${JSON.stringify(raw)}`); + // consume as much before a number (contiguous run) + let until = cand.findIndex((x) => typeof x === 'number'); + if (until === -1) { + until = cand.length; } - const part: SetPart = { before, body: raw.map(decode) }; - out.push(part); + const body = decode(cand.slice(0, until)); + out.push({ skip, body }); + + i = i + 1 + until; } return out; @@ -98,7 +121,7 @@ export type Patch = { export type WirePatch = any[]; -export function encodePatch(patch: Patch, encode: (x: X) => any): WirePatch { +export function encodePatch(patch: Patch, encode: (x: X[]) => any[]): WirePatch { const out = encodeSetOp(patch.set, encode); const run = encodeOpRun(patch.run); @@ -109,7 +132,7 @@ export function encodePatch(patch: Patch, encode: (x: X) => any): WireP return out; } -export function decodePatch(wire: WirePatch, decode: (raw: any) => X = (x) => x): Patch { +export function decodePatch(wire: WirePatch, decode: (raw: any[]) => X[] = (x) => x): Patch { const out: Patch = { run: [], set: [] }; if (!wire.length) { diff --git a/client/lib/ymodel/work-holder.test.ts b/client/lib/ymodel/work-holder.test.ts index ebadad1..d75d7ac 100644 --- a/client/lib/ymodel/work-holder.test.ts +++ b/client/lib/ymodel/work-holder.test.ts @@ -4,6 +4,25 @@ import { WorkHolder } from './work-holder.ts'; import { RopeState } from './internal/state.ts'; import { convertToEmpty } from './helper.ts'; +test('number ranges', () => { + let actual = RopeState.new(); + actual.applyOp(actual.coerceOp({ r: [0], length: 5 })); // we have 5 length from server + + const w = new WorkHolder(() => actual); + + // insert our own 2 at start + w.applyOp({ r: [0], length: 2 }); + w.applySet({ skip: 0, body: ['local-a', 'local-b'] }); + w.applySet({ skip: 2, body: ['server'] }); + + const out = w.forServer((id) => { + return actual.posOf(id); + }); + + // allows contiguous - we set a/b in new data, and then something at server pos 0 + assert.deepStrictEqual(out?.set, [{ skip: -2, body: ['local-a', 'local-b', 'server'] }]); +}); + test('generic', () => { let actual = RopeState.new(); let calls = 0; @@ -19,11 +38,11 @@ test('generic', () => { assert.strictEqual(w.length(), 5); assert.strictEqual(calls, 2); - w.applySet({ before: 3, body: [1, 2, 3] }); + w.applySet({ skip: 0, body: [1, 2, 3] }); assert.deepStrictEqual(w.read(), convertToEmpty([1, 2, 3, undefined, undefined])); // now include some string data - w.applySet({ before: 4, body: [72, 73] }); + w.applySet({ skip: 2, body: [72, 73] }); assert.deepStrictEqual(w.read(), convertToEmpty([1, 2, 72, 73, undefined])); let out = w.forServer((id) => { @@ -37,7 +56,7 @@ test('generic', () => { ]); assert.deepStrictEqual(out?.set, [ { - before: -2, + skip: -5, body: [1, 2, 72, 73], }, ]); @@ -60,7 +79,7 @@ test('generic', () => { ]); assert.deepStrictEqual(out?.set, [ { - before: -3, + skip: -5, body: [1, 2, 72], }, ]); @@ -79,7 +98,7 @@ test('generic', () => { ]); assert.deepStrictEqual(out?.set, [ { - before: -2, + skip: -5, body: [1, 2, 72, 73], }, ]); diff --git a/client/lib/ymodel/work-holder.ts b/client/lib/ymodel/work-holder.ts index 081169c..eb9bd18 100644 --- a/client/lib/ymodel/work-holder.ts +++ b/client/lib/ymodel/work-holder.ts @@ -259,7 +259,7 @@ function mergePatchSet( combinedLookupId: (consumedLength: number, Id: Id) => number, src: InternalSet[], ) { - const unique = new Map(); + const unique = new Map(); // pos to data // aggregate first for (const set of src) { @@ -267,7 +267,16 @@ function mergePatchSet( if (m.posOfValid(id) === -1) { return; // don't send now-deleted sets } - unique.set(combinedLookupId(0, id), set.data[index]); + let target = combinedLookupId(0, id); + if (target === 0) { + throw new Error(`panic; can't set at zero`); + } + if (target < 0) { + // important: -ve indexes refer to after, not before + // we internally accumulate them targeting zero, because we go back by body length (so is -ve) + ++target; + } + unique.set(target, set.data[index]); }); } @@ -279,29 +288,27 @@ function mergePatchSet( while (sortedBefore.length) { // consume contiguous parts - const start = sortedBefore[0]; let before = start; let i = 0; while (++i < sortedBefore.length) { const next = sortedBefore[i]; - if (next === 0 || next !== before + 1) { + if (next !== before + 1) { break; } before = next; } sortedBefore = sortedBefore.slice(i); - // we have data of length=i, before - + // we have data of length=i, and a valid 'before' pos const body: X[] = []; for (let j = 0; j < i; ++j) { const data = unique.get(start + j)!; body.push(data); } - out.push({ before, body }); + out.push({ skip: before - body.length, body }); } return out; diff --git a/tests/integration/basic.test.ts b/tests/integration/basic.test.ts index 55360f3..390a1a3 100644 --- a/tests/integration/basic.test.ts +++ b/tests/integration/basic.test.ts @@ -19,7 +19,7 @@ test('basic', async (t) => { doc2.floor.perform(function* (t) { const api = t.byIno('abc'); - api.set(5, 'interned string', -100n, -Infinity, undefined); + api.set(1, 'interned string', -100n, -Infinity, undefined); }); await checkCond(() => { From 9be3cdc7c7e0a4e4e9d0ede8c50f86f0ad1e993a Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sun, 5 Apr 2026 12:18:40 +1000 Subject: [PATCH 21/22] fix raw/node --- server/pkg/jsonhelp/utf16.go | 6 +- server/pkg/model/node/node.go | 6 +- server/pkg/model/node/node_test.go | 50 ++++- server/pkg/model/node/patch.go | 4 +- server/pkg/model/node/patch_test.go | 28 +-- server/pkg/model/raw/check.go | 20 -- server/pkg/model/raw/const.go | 6 +- server/pkg/model/raw/enc.go | 129 ------------- server/pkg/model/raw/enc_test.go | 118 ------------ server/pkg/model/raw/helper.go | 37 ++++ server/pkg/model/raw/merge.go | 237 ----------------------- server/pkg/model/raw/merge_test.go | 150 --------------- server/pkg/model/raw/set.go | 286 ++++++++++++++++++++++++++++ server/pkg/model/raw/set_test.go | 199 +++++++++++++++++++ 14 files changed, 588 insertions(+), 688 deletions(-) delete mode 100644 server/pkg/model/raw/check.go delete mode 100644 server/pkg/model/raw/enc.go delete mode 100644 server/pkg/model/raw/enc_test.go create mode 100644 server/pkg/model/raw/helper.go delete mode 100644 server/pkg/model/raw/merge.go delete mode 100644 server/pkg/model/raw/merge_test.go create mode 100644 server/pkg/model/raw/set.go create mode 100644 server/pkg/model/raw/set_test.go diff --git a/server/pkg/jsonhelp/utf16.go b/server/pkg/jsonhelp/utf16.go index b4da8f7..c959392 100644 --- a/server/pkg/jsonhelp/utf16.go +++ b/server/pkg/jsonhelp/utf16.go @@ -23,7 +23,7 @@ func CountUTF16(s string) (count int) { } // EncodeToUTF16 returns the UTF-16 encoding of the string s. -func EncodeToUTF16(s string) []uint16 { +func EncodeToUTF16(s string) (runes []uint16) { a := make([]uint16, len(s)) // creates more than we need n := 0 for _, v := range s { @@ -46,7 +46,7 @@ func EncodeToUTF16(s string) []uint16 { // UnmarshalToUTF16 unmarshals the given JSON-encoded string into a []uint16. // If the raw message is not a string, returns the nil array. -func UnmarshalToUTF16(raw json.RawMessage) []uint16 { +func UnmarshalToUTF16(raw json.RawMessage) (runes []uint16) { if len(raw) == 0 || raw[0] != '"' { return nil // Unmarshal doesn't always error here } @@ -61,7 +61,7 @@ func UnmarshalToUTF16(raw json.RawMessage) []uint16 { } // DecodeFromUTF16 returns the UTF-8 encoding of the UTF-16 string s. -func DecodeFromUTF16(s []uint16) string { +func DecodeFromUTF16(s []uint16) (str string) { runes := utf16.Decode(s) return string(runes) } diff --git a/server/pkg/model/node/node.go b/server/pkg/model/node/node.go index e4065c4..078f247 100644 --- a/server/pkg/model/node/node.go +++ b/server/pkg/model/node/node.go @@ -18,8 +18,8 @@ import ( // // It knows about the handle who performed history operations, so it is exposed to end-clients. type Node struct { - set raw.GumnutSetArray // data here - readAt int // when we last read/flattened data + set raw.SetOp // data here + readAt int // when we last read/flattened data prune bool // if history is not exhaustive history []historyEntry @@ -62,7 +62,7 @@ func (n *Node) versionRange() (prune bool, lo, hi int) { // Read reads the content of this node. // This will transform data forward as needed. -func (n *Node) Read(docVersion int) (sm raw.GumnutSetArray) { +func (n *Node) Read(docVersion int) (sm raw.SetOp) { _, _, hi := n.versionRange() docVersion = max(hi, docVersion) diff --git a/server/pkg/model/node/node_test.go b/server/pkg/model/node/node_test.go index 6866307..6f170dc 100644 --- a/server/pkg/model/node/node_test.go +++ b/server/pkg/model/node/node_test.go @@ -1,6 +1,8 @@ package node import ( + "log" + "math" "reflect" "testing" @@ -32,8 +34,8 @@ func TestNode(t *testing.T) { } mustEnact(1, 0, Patch{ - Set: raw.GumnutSetArray{ - {Before: 10, String: jsonhelp.EncodeToUTF16("hello")}, + Set: raw.SetOp{ + {Skip: 5, Data: raw.MapString("hello")}, }, }) @@ -41,8 +43,8 @@ func TestNode(t *testing.T) { Run: []raw.Op{ {Range: []int{10}, Length: 6}, }, - Set: raw.GumnutSetArray{ - {Before: -1, String: jsonhelp.EncodeToUTF16(" there")}, + Set: raw.SetOp{ + {Skip: -6, Data: raw.MapString(" there")}, }, }) @@ -50,15 +52,43 @@ func TestNode(t *testing.T) { Run: []raw.Op{ {Range: []int{10}, Length: 4}, }, - Set: raw.GumnutSetArray{ - {Before: -1, String: jsonhelp.EncodeToUTF16(" bob")}, + Set: raw.SetOp{ + {Skip: -4, Data: raw.MapString(" bob")}, }, }) out := n.Read(3) - if !reflect.DeepEqual(out, raw.GumnutSetArray{ - {Before: 20, String: jsonhelp.EncodeToUTF16("hello bob there")}, - }) { - t.Errorf("got bad state: %+v", out) + expected := raw.SetOp{ + {Skip: 5, Data: raw.MapString("hello bob there")}, } + if !reflect.DeepEqual(out, expected) { + t.Errorf("got bad state=%+v expected=%+v", out, expected) + + log.Printf("actual->") + logSetOp(out) + log.Printf("expected->") + logSetOp(expected) + } +} + +func logSetOp(s raw.SetOp) { + + for _, part := range s { + runes := make([]uint16, len(part.Data)) + + for i, gd := range part.Data { + if gd.Type != raw.DataTypeNumber { + continue + } + f := math.Float64frombits(gd.Data) + u := uint16(f) + if float64(u) != f { + continue + } + runes[i] = u + } + + log.Printf("..part skip=%d str=\"%s\"", part.Skip, jsonhelp.DecodeFromUTF16(runes)) + } + } diff --git a/server/pkg/model/node/patch.go b/server/pkg/model/node/patch.go index c21b960..d4d26e2 100644 --- a/server/pkg/model/node/patch.go +++ b/server/pkg/model/node/patch.go @@ -8,8 +8,8 @@ import ( ) type Patch struct { - Run raw.OpRun // structural changes - Set raw.GumnutSetArray // sets for data + Run raw.OpRun // structural changes + Set raw.SetOp // sets for data } func (p *Patch) IsZero() (is bool) { diff --git a/server/pkg/model/node/patch_test.go b/server/pkg/model/node/patch_test.go index c97275d..d9d9367 100644 --- a/server/pkg/model/node/patch_test.go +++ b/server/pkg/model/node/patch_test.go @@ -29,45 +29,45 @@ func TestPatch_MarshalUnmarshal(t *testing.T) { { "only data string", Patch{ - Set: raw.GumnutSetArray{{Before: 10, String: []uint16{'a', 'b'}}}, + Set: raw.SetOp{{Skip: 8, Data: raw.MapString("ab")}}, }, - `[10,"ab"]`, + `[8,[97,98]]`, }, { "only data raw", Patch{ - Set: raw.GumnutSetArray{{Before: 10, Data: []raw.GumnutData{{Type: raw.DataTypeNumber, Data: 0x4000000000000000}}}}, // 2.0 + Set: raw.SetOp{{Skip: 9, Data: []raw.GumnutData{{Type: raw.DataTypeNumber, Data: 0x4000000000000000}}}}, // 2.0 }, - `[10,[2]]`, + `[9,[2]]`, }, { "both structural and data", Patch{ Run: raw.OpRun{{Range: []int{5}, Length: 3}}, - Set: raw.GumnutSetArray{{Before: 10, String: []uint16{'a', 'b'}}}, + Set: raw.SetOp{{Skip: 8, Data: raw.MapString("ab")}}, }, - `[[-3,5],10,"ab"]`, + `[[-3,5],8,[97,98]]`, }, { "multiple data", Patch{ - Set: raw.GumnutSetArray{ - {Before: 10, String: []uint16{'a'}}, - {Before: 20, String: []uint16{'b'}}, + Set: raw.SetOp{ + {Skip: 10, Data: raw.MapString("a")}, + {Skip: 20, Data: raw.MapString("b")}, }, }, - `[10,"a",20,"b"]`, + `[10,[97],20,[98]]`, }, { "sample data", Patch{ Run: raw.OpRun{{Range: []int{0}, Length: 2}}, - Set: raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + Set: raw.SetOp{{Skip: -2, Data: []raw.GumnutData{ {Type: raw.DataTypeString, String: "abc"}, {Type: raw.DataTypeString, String: "hello"}, }}}, }, - `[[-2,0],-1,[{"s":"abc"},{"s":"hello"}]]`, + `[[-2,0],-2,[{"s":"abc"},{"s":"hello"}]]`, }, } @@ -106,7 +106,7 @@ func normalizeRun(r raw.OpRun) raw.OpRun { return r } -func normalizeSet(s raw.GumnutSetArray) raw.GumnutSetArray { +func normalizeSet(s raw.SetOp) raw.SetOp { if len(s) == 0 { return nil } @@ -120,7 +120,7 @@ func TestPatch_IsZero(t *testing.T) { if (&Patch{Run: raw.OpRun{{Range: []int{1}}}}).IsZero() { t.Error("expected false for Patch with Run") } - if (&Patch{Set: raw.GumnutSetArray{{Before: 1}}}).IsZero() { + if (&Patch{Set: raw.SetOp{{Skip: 0}}}).IsZero() { t.Error("expected false for Patch with Set") } } diff --git a/server/pkg/model/raw/check.go b/server/pkg/model/raw/check.go deleted file mode 100644 index f611a4f..0000000 --- a/server/pkg/model/raw/check.go +++ /dev/null @@ -1,20 +0,0 @@ -package raw - -import ( - "slices" - "unsafe" -) - -func init() { - var i int - if unsafe.Sizeof(i) != 8 { - panic("must run where int is 8 bytes") - } -} - -func extendSlice[X any](slice []X, length int) (out []X) { - if len(slice) >= length { - return slice - } - return append(slice, slices.Repeat(make([]X, 1), length-len(slice))...) -} diff --git a/server/pkg/model/raw/const.go b/server/pkg/model/raw/const.go index a2f3224..0cb76f5 100644 --- a/server/pkg/model/raw/const.go +++ b/server/pkg/model/raw/const.go @@ -5,6 +5,8 @@ import ( ) var ( - ErrEncoding = errors.New("data encoding error") - ErrVersion = errors.New("version must always increase") + ErrEncoding = errors.New("data encoding error") + ErrVersion = errors.New("version must always increase") + ErrForwardSet = errors.New("got -ve set in normal forward") + ErrNormalizeSet = errors.New("couldn't normalize user set") ) diff --git a/server/pkg/model/raw/enc.go b/server/pkg/model/raw/enc.go deleted file mode 100644 index 67c32ff..0000000 --- a/server/pkg/model/raw/enc.go +++ /dev/null @@ -1,129 +0,0 @@ -package raw - -import ( - "encoding/json" - "math" - - "github.com/gumnutdev/foiled/server/pkg/jsonhelp" -) - -type GumnutSet struct { - Before int // apply before this location (maybe -ve for user) - - String []uint16 // if set, prefer - only has uint16's - Data []GumnutData // otherwise use Data as fallback -} - -func (gs *GumnutSet) asData() (gd []GumnutData) { - if gs.String != nil { - gd = make([]GumnutData, len(gs.String)) - for i, r := range gs.String { - gd[i].Type = DataTypeNumber - gd[i].Data = math.Float64bits(float64(r)) - } - return gd - } - - return gs.Data -} - -// GumnutSetArray is a sequence of set operations which will be applied one after another. -type GumnutSetArray []GumnutSet - -func (arr GumnutSetArray) MarshalJSON() (b []byte, err error) { - out, err := arr.MarshalJSONArray() - if err != nil { - return nil, err - } - return json.Marshal(out) -} - -func (arr GumnutSetArray) MarshalJSONArray() (out []json.RawMessage, err error) { - for _, each := range arr { - if each.length() == 0 { - continue - } - - num, err := json.Marshal(each.Before) - if err != nil { - return nil, err - } - - b, err := each.internalEncode() - if err != nil { - return nil, err - } - - out = append(out, num, json.RawMessage(b)) - } - - return out, nil -} - -func (arr *GumnutSetArray) UnmarshalJSON(b []byte) (err error) { - var tmp []json.RawMessage - err = json.Unmarshal(b, &tmp) - if err != nil { - return err - } - return arr.UnmarshalJSONArray(tmp) -} - -func (arr *GumnutSetArray) UnmarshalJSONArray(raw []json.RawMessage) (err error) { - if len(raw)%2 != 0 { - return ErrEncoding - } - - update := make([]GumnutSet, 0, len(raw)/2) - - for i := 0; i < len(raw); i += 2 { - var s GumnutSet - - err = json.Unmarshal(raw[i], &s.Before) - if err != nil { - return err - } - - err = s.internalDecode(raw[i+1]) - if err != nil { - return err - } - - update = append(update, s) - } - - *arr = update - return nil -} - -func (s GumnutSet) internalEncode() (b []byte, err error) { - if s.String != nil { - s := jsonhelp.DecodeFromUTF16(s.String) - return json.Marshal(s) - } - return json.Marshal(s.Data) -} - -func (s *GumnutSet) internalDecode(b []byte) (err error) { - if len(b) == 0 { - return ErrEncoding - } - - if b[0] == '"' { - x := jsonhelp.UnmarshalToUTF16(b) - if x == nil { - return ErrEncoding - } - s.String = x - return nil - } - - return json.Unmarshal(b, &s.Data) -} - -func (s *GumnutSet) length() (length int) { - if s.String != nil { - return len(s.String) - } - return len(s.Data) -} diff --git a/server/pkg/model/raw/enc_test.go b/server/pkg/model/raw/enc_test.go deleted file mode 100644 index a488156..0000000 --- a/server/pkg/model/raw/enc_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package raw - -import ( - "encoding/json" - "math" - "reflect" - "testing" -) - -func TestGumnutSetArray_EncodeDecode(t *testing.T) { - cases := []struct { - name string - in GumnutSetArray - }{ - { - name: "empty", - in: GumnutSetArray{}, - }, - { - name: "fast-path uint16 string", - in: GumnutSetArray{ - { - Before: 10, - String: []uint16{'h', 'e', 'l', 'l', 'o'}, - }, - { - Before: 20, - String: []uint16{'w', 'o', 'r', 'l', 'd'}, - }, - }, - }, - { - name: "fallback data array", - in: GumnutSetArray{ - { - Before: 5, - Data: []GumnutData{ - {Type: DataTypeNumber, Data: math.Float64bits(42)}, - {Type: DataTypeNumber, Data: math.Float64bits(-1.5)}, - }, - }, - }, - }, - { - name: "mixed fast-path and data", - in: GumnutSetArray{ - { - Before: 1, - String: []uint16{'A'}, - }, - { - Before: 2, - Data: []GumnutData{ - {Type: DataTypeNumber, Data: math.Float64bits(1.23)}, - }, - }, - }, - }, - { - name: "skips empty sets", - in: GumnutSetArray{ - {Before: 100, String: []uint16{}}, - {Before: 101, Data: []GumnutData{}}, - {Before: 102, String: []uint16{'x'}}, // only this should be serialized - }, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - b, err := json.Marshal(tc.in) - if err != nil { - t.Fatalf("MarshalJSON failed: %v", err) - } - - var out GumnutSetArray - err = json.Unmarshal(b, &out) - if err != nil { - t.Fatalf("UnmarshalJSON failed: %v", err) - } - - // Generate expected values (since empty sets are dropped on encode) - var expected GumnutSetArray - for _, s := range tc.in { - if s.length() > 0 { - expected = append(expected, s) - } - } - - if len(expected) == 0 && len(out) == 0 { - return // both are empty, matches - } - - if !reflect.DeepEqual(expected, out) { - t.Errorf("got %#v, want %#v", out, expected) - } - }) - } -} - -func TestGumnutSet_AsData(t *testing.T) { - s := GumnutSet{ - Before: 1, - String: []uint16{'H', 'i'}, - } - - gd := s.asData() - if len(gd) != 2 { - t.Fatalf("expected 2 elements, got %d", len(gd)) - } - - if gd[0].Type != DataTypeNumber || math.Float64frombits(gd[0].Data) != 'H' { - t.Errorf("unexpected gd[0]: %+v", gd[0]) - } - if gd[1].Type != DataTypeNumber || math.Float64frombits(gd[1].Data) != 'i' { - t.Errorf("unexpected gd[1]: %+v", gd[1]) - } -} diff --git a/server/pkg/model/raw/helper.go b/server/pkg/model/raw/helper.go new file mode 100644 index 0000000..f1cb1a8 --- /dev/null +++ b/server/pkg/model/raw/helper.go @@ -0,0 +1,37 @@ +package raw + +import ( + "math" + "slices" + "unsafe" + + "github.com/gumnutdev/foiled/server/pkg/jsonhelp" +) + +func init() { + var i int + if unsafe.Sizeof(i) != 8 { + panic("must run where int is 8 bytes") + } +} + +func extendSlice[X any](slice []X, length int) (out []X) { + if len(slice) >= length { + return slice + } + return append(slice, slices.Repeat(make([]X, 1), length-len(slice))...) +} + +// MapString converts the given normal, UTF-8 string to []GumnutData. +// For testing. +func MapString(s string) (gd []GumnutData) { + runes := jsonhelp.EncodeToUTF16(s) + + gd = make([]GumnutData, len(runes)) + for i, r := range runes { + gd[i].Type = DataTypeNumber + gd[i].Data = math.Float64bits(float64(r)) + } + + return +} diff --git a/server/pkg/model/raw/merge.go b/server/pkg/model/raw/merge.go deleted file mode 100644 index 6b3fd3a..0000000 --- a/server/pkg/model/raw/merge.go +++ /dev/null @@ -1,237 +0,0 @@ -package raw - -import ( - "iter" - "slices" -) - -// IterData returns an iterator which yields all GumnutData here. -// This will not yield fast-path string data. -func (s GumnutSetArray) IterData() (it iter.Seq[*GumnutData]) { - return func(yield func(*GumnutData) (more bool)) { - for i := range s { - d := s[i].Data - - for j := range d { - if !yield(&d[j]) { - return - } - } - } - } -} - -// SetEntry describes how to modify data over time. -// The first SetEntry in any array is effectively the base state. -type SetEntry struct { - Prior []PriorOp - Version int - Set GumnutSetArray -} - -// TransformFunc describes how we modify a GumnutData forward. -type TransformFunc func(fromVersion int, gd *GumnutData) (err error) - -type sliceSet struct { - version int - str bool // was this encoded as String - gd GumnutData -} - -func (se *sliceSet) isStr() (is bool) { - is = se.str - if is { - _, is = se.gd.JSRune() - } - return -} - -// MoveForward helps to move already "normalized" sets forward over structural ops. -// This transforms it structurally and wrt its data. -// The SetEntry array must have an incrementing version number. -func MoveForward(hist []SetEntry, tr TransformFunc) (out GumnutSetArray, err error) { - // this effectively flattens history to a single entry - - // confirm we always go > version - version := -1 - for _, e := range hist { - // log.Printf("got eVersion=%v vs version=%v", e.Version, version) - if e.Version <= version { - return nil, ErrVersion - } - version = e.Version - } - - var slice []sliceSet - priors := []PriorOp{} - - // work backwards - for _, each := range slices.Backward(hist) { - - // for each keyed set - for _, s := range each.Set { - target := s.Before - str := s.String != nil - - for _, gd := range slices.Backward(s.asData()) { - pos := target - if len(priors) > 0 { - pos = transformOverLocal(target, priors) - } - target-- - - if pos == 0 { - continue // xformed away - } - index := pos - 1 - - if len(slice) < pos { - // off end, always exists - slice = extendSlice(slice, pos) - slice[index] = sliceSet{each.Version, str, gd} - - } else if slice[index].version == 0 { - // check nothing already here - slice[index] = sliceSet{each.Version, str, gd} - - } else { - continue - } - } - } - - // if we have more priors, put it at the start of the array - if len(each.Prior) != 0 { - tmp := make([]PriorOp, len(each.Prior)+len(priors)) - copy(tmp, each.Prior) - copy(tmp[len(each.Prior):], priors) - priors = tmp - } - } - - // finally, xform all data _forward_ - if tr != nil { - for i := range slice { - if slice[i].gd.IsZero() { - continue - } - err = tr(slice[i].version, &slice[i].gd) - if err != nil { - return nil, err - } - } - } - - // create minimal sets - out = GumnutSetArray{} - - length := len(slice) - for i := 0; i < length; { - if slice[i].gd.IsZero() { - i++ - continue - } - - str := slice[i].isStr() - - // consume matching (non-zero version, str == base) - j := i + 1 - for j < length { - jstr := slice[j].isStr() - if jstr != str || slice[j].gd.IsZero() { - break - } - j++ - } - - // got slice [i,j], convert to String or Data - set := GumnutSet{Before: j} - if str { - set.String = make([]uint16, j-i) - for k := i; k < j; k++ { - set.String[k-i], _ = slice[k].gd.JSRune() - } - } else { - set.Data = make([]GumnutData, j-i) - for k := i; k < j; k++ { - set.Data[k-i] = slice[k].gd - } - } - out = append(out, set) - - i = j - } - - return out, nil -} - -func transformOverLocal(v int, over []PriorOp) (out int) { - for _, each := range over { - if v == 0 { - return 0 - } - var innerGone bool - v, innerGone = each.TransformPos(v) - - if innerGone || v == 0 { - return 0 - } - } - - return v -} - -// TransformSetOver is used to transform Set locations for a given node and "normalize" them. -// Basically, this maps a user setting at "-1" to the real location, or to fix a server-space location which has changed. -// This happens after a patch's structural changes have been applied and the PriorOpSelf will reflect that. -func TransformSetOver(arr GumnutSetArray, prior []PriorOpSelf) (out GumnutSetArray, err error) { - // accumulate into single entries - var tmp []GumnutSet - - for _, x := range arr { - - targets := make([]int, x.length()) - for i := range targets { - pos := x.Before - len(targets) + i + 1 - - var gone bool - pos, gone = TransformOver(pos, prior) - if gone { - pos = 0 - } - if pos < 0 { - return nil, ErrVersion - // log.Printf("couldn't transform (%v -> %v) over=%+v", was, pos, prior) - // for _, p := range prior { - // log.Printf("prior=%+v", p.PriorOp) - // } - // panic("BAD POS") - } - - targets[i] = pos - } - - if x.String != nil { - for i, r := range x.String { - pos := targets[i] - if pos == 0 { - continue - } - tmp = append(tmp, GumnutSet{Before: pos, String: []uint16{r}}) - } - continue - } - - for i, r := range x.Data { - pos := targets[i] - if pos == 0 { - continue - } - tmp = append(tmp, GumnutSet{Before: pos, Data: []GumnutData{r}}) - } - continue - } - - // run regular MoveForward to flatten data - return MoveForward([]SetEntry{{Version: 1, Set: tmp}}, nil) -} diff --git a/server/pkg/model/raw/merge_test.go b/server/pkg/model/raw/merge_test.go deleted file mode 100644 index 8fd0b7d..0000000 --- a/server/pkg/model/raw/merge_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package raw - -import ( - "reflect" - "testing" - - "github.com/gumnutdev/foiled/server/pkg/jsonhelp" -) - -func TestMoveForward(t *testing.T) { - tests := []struct { - name string - src []SetEntry - out GumnutSetArray - tr TransformFunc - }{ - { - "zero", - []SetEntry{}, - GumnutSetArray{}, - nil, - }, - { - "basic", - []SetEntry{ - { - Version: 1, - Set: GumnutSetArray{ - {Before: 16, String: jsonhelp.EncodeToUTF16("hello there")}, - }, - }, - { - Version: 2, - Prior: []PriorOp{ - &priorOpRemove{lo: 9, hi: 10}, - &priorOpInsert{after: 9, length: 1}, - }, - Set: GumnutSetArray{ - {Before: 10, String: jsonhelp.EncodeToUTF16("!")}, - }, - }, - }, - GumnutSetArray{ - {Before: 16, String: jsonhelp.EncodeToUTF16("hell! there")}, - }, - nil, - }, - { - "merged", - []SetEntry{ - { - Version: 100, - Set: GumnutSetArray{ - {Before: 80, String: jsonhelp.EncodeToUTF16("hello")}, - {Before: 86, String: jsonhelp.EncodeToUTF16("? there")}, - {Before: 87, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, - }, - }, - }, - GumnutSetArray{ - {Before: 86, String: jsonhelp.EncodeToUTF16("hello there")}, - {Before: 87, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, - }, - nil, - }, - { - "multiple_versions", - []SetEntry{ - { - Version: 1, - Set: GumnutSetArray{ - {Before: 5, String: jsonhelp.EncodeToUTF16("ABCD")}, - }, - }, - { - Version: 2, - Set: GumnutSetArray{ - {Before: 4, String: jsonhelp.EncodeToUTF16("x")}, - }, - }, - }, - GumnutSetArray{ - {Before: 5, String: jsonhelp.EncodeToUTF16("ABxD")}, - }, - nil, - }, - { - "transform_func", - []SetEntry{ - { - Version: 1, - Set: GumnutSetArray{ - {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 100}}}, - }, - }, - }, - GumnutSetArray{ - {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 200}}}, - }, - func(ver int, gd *GumnutData) (err error) { - if gd.Type == DataTypeNumber { - gd.Data += 100 - } - return nil - }, - }, - } - - for _, each := range tests { - t.Run(each.name, func(t *testing.T) { - got, err := MoveForward(each.src, each.tr) - if err != nil { - t.Errorf("expected non-nil err: %+v", err) - } - if !reflect.DeepEqual(got, each.out) { - t.Errorf("got bad out=%+v expected=%+v", got, each.out) - } - }) - } -} - -func TestIterData(t *testing.T) { - arr := GumnutSetArray{ - {Before: 10, Data: []GumnutData{{Type: DataTypeNumber, Data: 42}, {Type: DataTypeNumber, Data: 43}}}, - {Before: 20, String: []uint16{'a', 'b'}}, - {Before: 30, Data: []GumnutData{{Type: DataTypeString, String: "foo"}}}, - } - - var got []GumnutData - for gd := range arr.IterData() { - got = append(got, *gd) - } - - if len(got) != 3 { - t.Fatalf("expected 3 data items, got %d", len(got)) - } - if got[0].Data != 42 || got[1].Data != 43 || got[2].String != "foo" { - t.Errorf("unexpected elements in iter: %+v", got) - } - - // test early exit - count := 0 - for range arr.IterData() { - count++ - break - } - if count != 1 { - t.Errorf("expected early exit to yield 1 item, got %d", count) - } -} diff --git a/server/pkg/model/raw/set.go b/server/pkg/model/raw/set.go new file mode 100644 index 0000000..caf0be1 --- /dev/null +++ b/server/pkg/model/raw/set.go @@ -0,0 +1,286 @@ +package raw + +import ( + "encoding/json" + "math" + "slices" + + "github.com/gumnutdev/foiled/server/pkg/jsonhelp" +) + +type SetPart struct { + Skip int + Data []GumnutData +} + +// SetOp is a sequence of set operations which will be applied one after another. +type SetOp []SetPart + +func (arr SetOp) MarshalJSONArray() (out []json.RawMessage, err error) { + for i, each := range arr { + if len(each.Data) == 0 { + continue + } + + if i == 0 || each.Skip != 0 { + num, err := json.Marshal(each.Skip) + if err != nil { + return nil, err + } + out = append(out, json.RawMessage(num)) + } + + // TODO: we want to string-ify low numbers + b, err := json.Marshal(each.Data) + if err != nil { + return nil, err + } + out = append(out, json.RawMessage(b)) + } + + return out, nil +} + +func (arr *SetOp) UnmarshalJSONArray(raw []json.RawMessage) (err error) { + update := make([]SetPart, 0, len(raw)/2) + + for i := 0; i < len(raw); i++ { + var s SetPart + + // must always start with a number + err = json.Unmarshal(raw[i], &s.Skip) + if err != nil { + return err + } + + // now, decode as many arrays and strings as we can! + + inner: + for i+1 < len(raw) { + first := raw[i+1][0] + switch first { + case '[': + var data []GumnutData + err = json.Unmarshal(raw[i+1], &data) + if err != nil { + return err + } + s.Data = append(s.Data, data...) + + case '"': + runes := jsonhelp.UnmarshalToUTF16(raw[i+1]) + if len(runes) == 0 { + return ErrEncoding + } + + segment := make([]GumnutData, len(runes)) + for i, r := range runes { + segment[i].Type = DataTypeNumber + segment[i].Data = math.Float64bits(float64(r)) + } + + default: + break inner + + } + + i++ + } + + update = append(update, s) + } + + *arr = update + return nil +} + +// SetEntry describes how to modify data over time. +// The first SetEntry in any array is effectively the base state. +type SetEntry struct { + Prior []PriorOp + Version int + Set SetOp +} + +// TransformFunc describes how we modify a GumnutData forward. +type TransformFunc func(fromVersion int, gd *GumnutData) (err error) + +type sliceSet struct { + version int + gd GumnutData +} + +// MoveForward helps to move already "normalized" sets forward over structural ops. +// This transforms it structurally and wrt its data. +// The SetEntry array must have an incrementing version number. +func MoveForward(hist []SetEntry, tr TransformFunc) (out SetOp, err error) { + // this effectively flattens history to a single entry + + // confirm we always go > version + version := -1 + for _, e := range hist { + // log.Printf("got eVersion=%v vs version=%v", e.Version, version) + if e.Version <= version { + return nil, ErrVersion + } + version = e.Version + } + + var slice []sliceSet + priors := []PriorOp{} + + // work backwards + for _, each := range slices.Backward(hist) { + var at int + + // for each keyed set + for _, s := range each.Set { + at += s.Skip + if at < 0 { + return nil, ErrForwardSet + } + + for _, gd := range s.Data { + at++ // move past to transform + pos := transformOverLocal(at, priors) + index := pos - 1 + if index < 0 { + continue // xformed away + } + + if len(slice) < pos { + // off end, always exists + slice = extendSlice(slice, pos) + slice[index] = sliceSet{each.Version, gd} + + } else if slice[index].version == 0 || slice[index].version == each.Version { + // check nothing already here (or replacing self for some reason) + slice[index] = sliceSet{each.Version, gd} + + } else { + continue + } + } + } + + // if we have more priors, put it at the start of the array + if len(each.Prior) != 0 { + tmp := make([]PriorOp, len(each.Prior)+len(priors)) + copy(tmp, each.Prior) + copy(tmp[len(each.Prior):], priors) + priors = tmp + } + } + + // finally, xform all data _forward_ + if tr != nil { + for i := range slice { + if slice[i].gd.IsZero() { + continue + } + err = tr(slice[i].version, &slice[i].gd) + if err != nil { + return nil, err + } + } + } + + // create minimal sets + out = SetOp{} + + length := len(slice) + var last int + for i := 0; i < length; { + if slice[i].gd.IsZero() { + i++ + continue + } + + // consume non-zero parts + j := i + 1 + for j < length { + if slice[j].gd.IsZero() { + break + } + j++ + } + + // got slice [i,j], convert to String or Data + set := SetPart{Skip: i - last, Data: make([]GumnutData, j-i)} + for k := i; k < j; k++ { + set.Data[k-i] = slice[k].gd + } + out = append(out, set) + + last = j + i = j + } + + return out, nil +} + +func transformOverLocal(v int, over []PriorOp) (out int) { + for _, each := range over { + if v == 0 { + return 0 + } + var innerGone bool + v, innerGone = each.TransformPos(v) + + if innerGone || v == 0 { + return 0 + } + } + + return v +} + +// TransformSetOver is used to transform Set locations for a given node and "normalize" them. +// Basically, this maps a user setting at "-1" to the real location, or to fix a server-space location which has changed. +// This happens after a patch's structural changes have been applied and the PriorOpSelf will reflect that. +func TransformSetOver(arr SetOp, prior []PriorOpSelf) (out SetOp, err error) { + // accumulate into single entries + var tmp []SetPart + var last int // where are we in tmp (total of all skip + data) + var at int // where are we wrt. source data + + // This helper deals with the fact that -ve numbers are already "after" the target, and +ve are indexes, so we add one. + internalTransform := func(at int) (pos int) { + if at >= 0 { + at++ + } + pos, gone := TransformOver(at, prior) + if gone { + return 0 + } + return pos + } + + for _, x := range arr { + targets := make([]int, len(x.Data)) + at += x.Skip + + for i := range targets { + pos := internalTransform(at) + if pos < 0 { + return nil, ErrNormalizeSet + } + targets[i] = pos - 1 // -ve is "gone" + at++ + } + + for i, r := range x.Data { + index := targets[i] + if index < 0 { + continue + } + + skip := index - last + tmp = append(tmp, SetPart{Skip: skip, Data: []GumnutData{r}}) + last = index + 1 + } + } + + // run regular MoveForward to flatten data + return MoveForward([]SetEntry{{Version: 1, Set: tmp}}, nil) +} diff --git a/server/pkg/model/raw/set_test.go b/server/pkg/model/raw/set_test.go new file mode 100644 index 0000000..af1925b --- /dev/null +++ b/server/pkg/model/raw/set_test.go @@ -0,0 +1,199 @@ +package raw + +import ( + "math" + "reflect" + "testing" +) + +func TestSetOp_EncodeDecode(t *testing.T) { + cases := []struct { + name string + in SetOp + }{ + { + name: "empty", + in: SetOp{}, + }, + { + name: "fallback data array", + in: SetOp{ + { + Skip: 3, + Data: []GumnutData{ + {Type: DataTypeNumber, Data: math.Float64bits(42)}, + {Type: DataTypeNumber, Data: math.Float64bits(-1.5)}, + }, + }, + }, + }, + { + name: "mixed fast-path and data", + in: SetOp{ + { + Skip: 0, + Data: []GumnutData{ + {Type: DataTypeNumber, Data: math.Float64bits(float64('A'))}, + }, + }, + { + Skip: 1, + Data: []GumnutData{ + {Type: DataTypeNumber, Data: math.Float64bits(1.23)}, + }, + }, + }, + }, + { + name: "skips empty sets", + in: SetOp{ + {Skip: 100, Data: []GumnutData{}}, + {Skip: 1, Data: []GumnutData{}}, + {Skip: 1, Data: []GumnutData{{Type: DataTypeString, String: "hello"}}}, // only this should be serialized + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b, err := tc.in.MarshalJSONArray() + if err != nil { + t.Fatalf("MarshalJSONArray failed: %v", err) + } + + var out SetOp + err = out.UnmarshalJSONArray(b) + if err != nil { + t.Fatalf("UnmarshalJSONArray failed: %v", err) + } + + // Generate expected values (since empty sets are dropped on encode) + var expected SetOp + for _, s := range tc.in { + if len(s.Data) > 0 { + expected = append(expected, s) + } + } + + if len(expected) == 0 && len(out) == 0 { + return // both are empty, matches + } + + if !reflect.DeepEqual(expected, out) { + t.Errorf("got %#v, want %#v", out, expected) + } + }) + } +} + +func TestMoveForward(t *testing.T) { + tests := []struct { + name string + src []SetEntry + out SetOp + tr TransformFunc + }{ + { + "zero", + []SetEntry{}, + SetOp{}, + nil, + }, + { + "basic", + []SetEntry{ + { + Version: 1, + Set: SetOp{ + {Skip: 5, Data: MapString("hello there")}, + }, + }, + { + Version: 2, + Prior: []PriorOp{ + &priorOpRemove{lo: 9, hi: 10}, + &priorOpInsert{after: 9, length: 1}, + }, + Set: SetOp{ + {Skip: 9, Data: MapString("!")}, + }, + }, + }, + SetOp{ + {Skip: 5, Data: MapString("hell! there")}, + }, + nil, + }, + { + "merged", + []SetEntry{ + { + Version: 100, + Set: SetOp{ + {Skip: 75, Data: MapString("hello")}, + {Skip: -1, Data: MapString("? there")}, + {Skip: 7, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, + }, + }, + }, + SetOp{ + {Skip: 75, Data: MapString("hell? there")}, + {Skip: 7, Data: []GumnutData{{Type: DataTypeString, String: "whatever"}}}, + }, + nil, + }, + { + "multiple_versions", + []SetEntry{ + { + Version: 1, + Set: SetOp{ + {Skip: 1, Data: MapString("ABCD")}, + }, + }, + { + Version: 2, + Set: SetOp{ + {Skip: 3, Data: MapString("x")}, + }, + }, + }, + SetOp{ + {Skip: 1, Data: MapString("ABxD")}, + }, + nil, + }, + { + "transform_func", + []SetEntry{ + { + Version: 1, + Set: SetOp{ + {Skip: 9, Data: []GumnutData{{Type: DataTypeNumber, Data: 100}}}, + }, + }, + }, + SetOp{ + {Skip: 9, Data: []GumnutData{{Type: DataTypeNumber, Data: 200}}}, + }, + func(ver int, gd *GumnutData) (err error) { + if gd.Type == DataTypeNumber { + gd.Data += 100 + } + return nil + }, + }, + } + + for _, each := range tests { + t.Run(each.name, func(t *testing.T) { + got, err := MoveForward(each.src, each.tr) + if err != nil { + t.Errorf("expected non-nil err: %+v", err) + } + if !reflect.DeepEqual(got, each.out) { + t.Errorf("got bad out=%+v expected=%+v", got, each.out) + } + }) + } +} From 5156c141e472f0873fbf7a81c500e492efa4d0d7 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Sun, 5 Apr 2026 12:24:42 +1000 Subject: [PATCH 22/22] fix doc test --- server/pkg/model/doc/doc_test.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/server/pkg/model/doc/doc_test.go b/server/pkg/model/doc/doc_test.go index 69736d5..ab167c2 100644 --- a/server/pkg/model/doc/doc_test.go +++ b/server/pkg/model/doc/doc_test.go @@ -21,7 +21,7 @@ func TestPosRef(t *testing.T) { Update: map[string]node.Patch{ "inode_abc": { Run: []raw.Op{{Range: []int{0}, Length: 4}}, - Set: raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + Set: raw.SetOp{{Skip: -1, Data: []raw.GumnutData{ {Type: raw.DataTypePosRef, String: "inode_abc", Data: intToUint(-2)}, }}}, }, @@ -34,8 +34,8 @@ func TestPosRef(t *testing.T) { // read and confirm pos out := d.Read() p := out.Update["inode_abc"] - if !reflect.DeepEqual(p.Set, raw.GumnutSetArray{ - {Before: 4, Data: []raw.GumnutData{{Type: raw.DataTypePosRef, String: "inode_abc", Data: 3}}}, + if !reflect.DeepEqual(p.Set, raw.SetOp{ + {Skip: 3, Data: []raw.GumnutData{{Type: raw.DataTypePosRef, String: "inode_abc", Data: 3}}}, }) { t.Errorf("bad set: %+v", p.Set) } @@ -47,7 +47,7 @@ func TestPosRef(t *testing.T) { Update: map[string]node.Patch{ "inode_abc": { Run: []raw.Op{{Range: []int{1}, Length: 1}}, - Set: raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + Set: raw.SetOp{{Skip: -1, Data: []raw.GumnutData{ {Type: raw.DataTypeString, String: "whatever"}, }}}, }, @@ -59,9 +59,9 @@ func TestPosRef(t *testing.T) { out = d.Read() p = out.Update["inode_abc"] - if !reflect.DeepEqual(p.Set, raw.GumnutSetArray{ - {Before: 2, Data: []raw.GumnutData{{Type: raw.DataTypeString, String: "whatever"}}}, - {Before: 5, Data: []raw.GumnutData{{Type: raw.DataTypePosRef, String: "inode_abc", Data: 4}}}, + if !reflect.DeepEqual(p.Set, raw.SetOp{ + {Skip: 1, Data: []raw.GumnutData{{Type: raw.DataTypeString, String: "whatever"}}}, + {Skip: 2, Data: []raw.GumnutData{{Type: raw.DataTypePosRef, String: "inode_abc", Data: 4}}}, }) { t.Errorf("bad set: %+v", p.Set) } @@ -76,7 +76,7 @@ func TestInvalidPosRef(t *testing.T) { Update: map[string]node.Patch{ "inode_abc": { Run: []raw.Op{{Range: []int{0}, Length: 4}}, - Set: raw.GumnutSetArray{{Before: -1, Data: []raw.GumnutData{ + Set: raw.SetOp{{Skip: -1, Data: []raw.GumnutData{ {Type: raw.DataTypePosRef, String: "inode_abc", Data: intToUint(-5)}, }}}, }, @@ -106,17 +106,20 @@ func TestTransform(t *testing.T) { } // set within that deleted state (but xformed away) - err = d.Apply(&Work{ + w := &Work{ TargetVersion: 1, Handle: 2, Update: map[string]node.Patch{ "inode_abc": { - Set: raw.GumnutSetArray{{Before: 4, String: []uint16{'h', 'i'}}}, + Set: raw.SetOp{{Skip: 2, Data: raw.MapString("hi")}}, }, }, - }) + } + err = d.Apply(w) if err != nil { t.Fatalf("couldn't set xformed away: %+v", err) } - + if len(w.Update) != 0 { + t.Errorf("expected op transformed away") + } }