From 36c152abaad1e4053eef83e55315273a204d3b06 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Wed, 18 Mar 2026 18:42:30 +1100 Subject: [PATCH 1/4] WIP --- client/lib/gumnut/floor.ts | 15 ++-- client/lib/ymodel/types.ts | 49 +++++----- server/pkg/ylayer/gumnut.go | 149 +++++++++++++++++++------------ server/pkg/ylayer/gumnut_test.go | 34 ++++--- server/pkg/ylayer/patch.go | 51 +++++++++-- tests/integration/basic.test.ts | 14 +++ 6 files changed, 208 insertions(+), 104 deletions(-) diff --git a/client/lib/gumnut/floor.ts b/client/lib/gumnut/floor.ts index 9177731..9017851 100644 --- a/client/lib/gumnut/floor.ts +++ b/client/lib/gumnut/floor.ts @@ -21,15 +21,13 @@ type WireType = { class GumnutFloorApi extends GumnutLow { private encodeGumnutData(lookupId: (inode: string, id: Id) => number, data: GumnutData) { switch (typeof data) { - case 'string': - return { s: data }; + case 'string': // intern string + case 'boolean': // true/false + return data; case 'bigint': return { p: `b:${data.toString()}` }; - case 'boolean': - return data; - case 'number': if (!isFinite(data)) { if (data === +Infinity) { @@ -71,7 +69,8 @@ class GumnutFloorApi extends GumnutLow { public decodeGumnutData(resolvePos: (inode: string, pos: number) => Id, raw: any): GumnutData { switch (typeof raw) { - case 'boolean': + case 'string': // intern string + case 'boolean': // true/false case 'number': return raw; @@ -247,7 +246,7 @@ export class GumnutFloor { // TODO: above this is the same as byIno set(before: number, ...data: GumnutData[]): void { - internal.applySet({ before, body: data }); + internal.applySet({ before, body: [data] }); }, dataUpdate(at, deleteCount, ...data) { @@ -256,7 +255,7 @@ export class GumnutFloor { } if (data.length) { internal.applyOp({ r: [at], length: data.length }); - internal.applySet({ before: at + data.length, body: data }); + internal.applySet({ before: at + data.length, body: [data] }); } }, diff --git a/client/lib/ymodel/types.ts b/client/lib/ymodel/types.ts index 0a56484..962db58 100644 --- a/client/lib/ymodel/types.ts +++ b/client/lib/ymodel/types.ts @@ -46,25 +46,16 @@ export function decodeOpRun(run: number[]): Op[] { export type SetPart = { before: number; - body: X[] | string; + body: (X[] | string)[]; }; -export type NotString = T extends string ? never : T; - /** * Encodes this set op. */ -export function encodeSetOp(parts: SetPart[], encode: (x: X) => NotString): any[] { +export function encodeSetOp(parts: SetPart[], encode: (x: X) => any): any[] { const out: any[] = []; for (const part of parts) { - if (typeof part.body === 'string') { - out.push(part.before, part.body); - } else { - out.push( - part.before, - part.body.map((p) => encode(p)), - ); - } + out.push(part.before, ...part.body.map((p) => (typeof p === 'string' ? p : p.map(encode)))); } return out; } @@ -74,18 +65,30 @@ export function encodeSetOp(parts: SetPart[], encode: (x: X) => NotStri */ export function decodeSetOp(parts: any[], decode: (raw: any) => X = (x) => x): SetPart[] { const out: SetPart[] = []; - if (parts.length % 2 !== 0) { - throw new Error('model encoding error'); - } - for (let i = 0; i < parts.length; i += 2) { + + let i = 0; + while (i < parts.length) { const before = parts[i] as number; - const raw = parts[i + 1]; + if (typeof before !== 'number' || Math.floor(before) !== before) { + throw new Error(`can't decode, bad set: ${JSON.stringify(parts)}`); + } + ++i; - if (typeof raw === 'string') { - out.push({ before, body: raw }); - } else { - const body = (raw as any[]).map((p) => decode(p)); - out.push({ before, body }); + const part: SetPart = { before, body: [] }; + out.push(part); + + // decode any number of array/string parts + while (i < parts.length && typeof parts[i] !== 'number') { + const next = parts[i]; + ++i; + + if (typeof next === 'string') { + part.body.push(next); + } else if (Array.isArray(next)) { + part.body.push(next.map(decode)); + } else { + throw new Error(`bad set part`); + } } } return out; @@ -106,7 +109,7 @@ export type WirePatch = { s?: any[]; }; -export function encodePatch(patch: Patch, encode: (x: X) => NotString): WirePatch { +export function encodePatch(patch: Patch, encode: (x: X) => any): WirePatch { const run = encodeOpRun(patch.run); const set = encodeSetOp(patch.set, encode); const out: WirePatch = {}; diff --git a/server/pkg/ylayer/gumnut.go b/server/pkg/ylayer/gumnut.go index 6d15fe8..15f5847 100644 --- a/server/pkg/ylayer/gumnut.go +++ b/server/pkg/ylayer/gumnut.go @@ -56,66 +56,98 @@ func DataForString(s string) (d []GumnutData) { type GumnutDataArray []GumnutData func (arr GumnutDataArray) MarshalJSON() (b []byte, err error) { - out := make([]any, 0, len(arr)) + panic("should not run MarshalJSON") +} + +func (arr GumnutDataArray) buildJSONParts() (out []json.RawMessage, err error) { + // consume runs of DataTypeJSRune _or_ anything else + rest := arr + for len(rest) > 0 { + first := rest[0] + + // generate rune run + if first.Type == DataTypeJSRune { + runes := []uint16{} + for i := 0; i < len(rest); i++ { + if rest[i].Type != DataTypeJSRune { + break + } + runes = append(runes, uint16(rest[i].Data)) + } + rest = rest[len(runes):] - length := len(arr) - for i := 0; i < length; i++ { - if arr[i].Type != DataTypeJSRune { - // encode "boring" value - out = append(out, arr[i]) + str := jsonhelp.DecodeFromUTF16(runes) + b, err := json.Marshal(str) + if err != nil { + return nil, err + } + out = append(out, b) continue } - // encode JS string - runes := []uint16{uint16(arr[i].Data)} - - for i+1 < length && arr[i+1].Type == DataTypeJSRune { - i++ - runes = append(runes, uint16(arr[i].Data)) + // generate normal data run + internal := []GumnutData{} + for i := 0; i < len(rest); i++ { + if rest[i].Type != first.Type { + break + } + internal = append(internal, rest[i]) } + rest = rest[len(internal):] - str := jsonhelp.DecodeFromUTF16(runes) - out = append(out, str) + b, err := json.Marshal(internal) + if err != nil { + return nil, err + } + out = append(out, b) } - return json.Marshal(out) + return out, nil } func (arr *GumnutDataArray) UnmarshalJSON(b []byte) (err error) { - var raw []json.RawMessage - err = json.Unmarshal(b, &raw) - if err != nil { - return + panic("should not run UnmarshalJSON") +} + +func decodeJSONParts(parts []json.RawMessage) (out GumnutDataArray, err error) { + for _, part := range parts { + var gd GumnutDataArray + gd, err = decodeJSONPart(part) + if err != nil { + return nil, err + } + out = append(out, gd...) } + return out, nil +} - out := make([]GumnutData, 0, len(raw)) +func decodeJSONPart(b json.RawMessage) (out GumnutDataArray, err error) { + if len(b) == 0 { + return nil, ErrEncoding + } - for _, each := range raw { - if len(each) == 0 { - return ErrEncoding + switch b[0] { + case '"': + runes := jsonhelp.UnmarshalToUTF16(b) + if runes == nil { + return nil, ErrEncoding } - - if each[0] == '"' { - runes := jsonhelp.UnmarshalToUTF16(each) - if runes == nil { - return ErrEncoding - } - for _, r := range runes { - out = append(out, GumnutData{Type: DataTypeJSRune, Data: uint64(r)}) - } - continue + out = make([]GumnutData, 0, len(runes)) + for _, r := range runes { + out = append(out, GumnutData{Type: DataTypeJSRune, Data: uint64(r)}) } + return out, nil - var gd GumnutData - err = json.Unmarshal(each, &gd) + case '[': + var tmp []GumnutData + err = json.Unmarshal(b, &tmp) if err != nil { - return err + return nil, err } - out = append(out, gd) + return tmp, nil } - *arr = out - return nil + return nil, ErrEncoding } func (gd GumnutData) MarshalJSON() (b []byte, err error) { @@ -144,11 +176,7 @@ func (gd GumnutData) MarshalJSON() (b []byte, err error) { return json.Marshal(out) case DataTypeString: - var out struct { - String string `json:"s"` - } - out.String = gd.String - return json.Marshal(out) + return json.Marshal(gd.String) case DataTypePosRef: var out struct { @@ -188,6 +216,18 @@ func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { return nil } + if s[0] == '"' { + // try string + var str string + err = json.Unmarshal(b, &str) + if err != nil { + return + } + g.Type = DataTypeString + g.String = str + return + } + if s[0] != '{' { // try number var num float64 @@ -203,10 +243,10 @@ func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { // otherwise, must be an object type var raw struct { - Empty bool `json:"e"` - Special string `json:"p"` - String *string `json:"s,omitzero"` - Target *int64 `json:"t,omitzero"` + Empty bool `json:"e"` + Special string `json:"p"` + String string `json:"s,omitzero"` // really, inode ref + Target *int64 `json:"t,omitzero"` } if err := json.Unmarshal(b, &raw); err != nil { return err @@ -223,15 +263,10 @@ func (g *GumnutData) UnmarshalJSON(b []byte) (err error) { return nil } - if raw.String != nil { - g.Type = DataTypeString - g.String = *raw.String - - if raw.Target != nil { - g.Type = DataTypePosRef - g.Data = uint64(*raw.Target) - } - + if raw.Target != nil { + g.Type = DataTypePosRef + g.String = raw.String + g.Data = uint64(*raw.Target) return nil } diff --git a/server/pkg/ylayer/gumnut_test.go b/server/pkg/ylayer/gumnut_test.go index b321085..1aad8b1 100644 --- a/server/pkg/ylayer/gumnut_test.go +++ b/server/pkg/ylayer/gumnut_test.go @@ -60,15 +60,14 @@ func TestGumnutDataArray_MarshalUnmarshal(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b, err := json.Marshal(tt.arr) + parts, err := tt.arr.buildJSONParts() if err != nil { - t.Fatalf("MarshalJSON failed: %v", err) + t.Fatalf("buildJSONParts failed: %v", err) } - var got GumnutDataArray - err = json.Unmarshal(b, &got) + got, err := decodeJSONParts(parts) if err != nil { - t.Fatalf("UnmarshalJSON failed: %v", err) + t.Fatalf("decodeJSONParts failed: %v", err) } if !reflect.DeepEqual(tt.arr, got) { @@ -91,11 +90,18 @@ func TestGumnutDataArray_MarshalRuns(t *testing.T) { {Type: DataTypeJSRune, Data: uint64('o')}, } - b, err := json.Marshal(arr) + parts, err := arr.buildJSONParts() if err != nil { t.Fatal(err) } + // render as single array + b, err := json.Marshal(parts) + if err != nil { + t.Fatal(err) + } + + // decode as single array we can check var raw []any if err := json.Unmarshal(b, &raw); err != nil { t.Fatal(err) @@ -108,8 +114,8 @@ func TestGumnutDataArray_MarshalRuns(t *testing.T) { if raw[0] != "hi" { t.Errorf("expected \"hi\", got %v", raw[0]) } - if raw[1] != 1.0 { - t.Errorf("expected 1.0, got %v", raw[1]) + if !reflect.DeepEqual(raw[1], []any{1.0}) { + t.Errorf("expected [1], got %+v", raw[1]) } if raw[2] != "o" { t.Errorf("expected \"o\", got %v", raw[2]) @@ -118,9 +124,15 @@ func TestGumnutDataArray_MarshalRuns(t *testing.T) { func TestGumnutDataArray_UnmarshalStrings(t *testing.T) { // Specifically test that strings in JSON are unmarshaled back to runs of DataTypeJSRune - input := `["abc", 123, "d"]` - var arr GumnutDataArray - if err := json.Unmarshal([]byte(input), &arr); err != nil { + input := `["abc", [123], "d"]` + + var parts []json.RawMessage + if err := json.Unmarshal([]byte(input), &parts); err != nil { + t.Fatal(err) + } + + arr, err := decodeJSONParts(parts) + if err != nil { t.Fatal(err) } diff --git a/server/pkg/ylayer/patch.go b/server/pkg/ylayer/patch.go index 6f485bf..3ea25e1 100644 --- a/server/pkg/ylayer/patch.go +++ b/server/pkg/ylayer/patch.go @@ -19,8 +19,8 @@ func (p *Patch) IsEmpty() (is bool) { func (p Patch) MarshalJSON() (b []byte, err error) { var out struct { - Run []int `json:"r,omitzero"` - Set []any `json:"s,omitzero"` + Run []int `json:"r,omitzero"` + Set []json.RawMessage `json:"s,omitzero"` } // encode Run @@ -40,7 +40,23 @@ func (p Patch) MarshalJSON() (b []byte, err error) { // encode Set for _, set := range p.Set { - out.Set = append(out.Set, set.Before, GumnutDataArray(set.Body)) + parts, err := GumnutDataArray(set.Body).buildJSONParts() + if err != nil { + return nil, err + } + if len(parts) == 0 { + continue + } + + // because "parts" is []json.RawMessage, encode Before to match + var beforeEnc json.RawMessage + beforeEnc, err = json.Marshal(set.Before) + if err != nil { + return nil, err + } + + out.Set = append(out.Set, beforeEnc) + out.Set = append(out.Set, parts...) } return json.Marshal(out) @@ -92,9 +108,34 @@ func (p *Patch) UnmarshalJSON(b []byte) error { } // decode Set - if len(raw.Set)%2 != 0 { - return ErrEncoding + var i int +next: + for i < len(raw.Set) { + var before int + if err := json.Unmarshal(raw.Set[i], &before); err != nil { + return err + } + i++ + + // decode any number of array/string parts + for i < len(raw.Set) { + next := raw.Set[i] + if len(next) == 0 { + return ErrEncoding + } + switch next[0] { + default: + continue next + case '"': + + case '[': + + } + + } + } + for i := 0; i < len(raw.Set); i += 2 { var before int if err := json.Unmarshal(raw.Set[i], &before); err != nil { diff --git a/tests/integration/basic.test.ts b/tests/integration/basic.test.ts index c059b16..55360f3 100644 --- a/tests/integration/basic.test.ts +++ b/tests/integration/basic.test.ts @@ -38,6 +38,20 @@ test('basic', async (t) => { }); }); +test('string', async (t) => { + const doc1 = await connectToGumnutLow(t.signal); + const doc2 = await connectToGumnutLow(t.signal, { docId: doc1.docId }); + + doc1.floor.perform(function* (t) { + const api = t.byIno('abc'); + api.stringUpdate(0, 0, 'hello'); + }); + + await checkCond(() => { + assert.strictEqual(doc2.floor.byIno('abc').string(), 'hello'); + }); +}); + test('naïve ref over time', async (t) => { const doc1 = await connectToGumnutLow(t.signal); From f89a9fe2ea4f4827e4a37d83c5e52b554674b297 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Thu, 19 Mar 2026 08:53:00 +1100 Subject: [PATCH 2/4] fix tests --- server/pkg/ylayer/patch.go | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/server/pkg/ylayer/patch.go b/server/pkg/ylayer/patch.go index 3ea25e1..fd6a9ea 100644 --- a/server/pkg/ylayer/patch.go +++ b/server/pkg/ylayer/patch.go @@ -109,7 +109,6 @@ func (p *Patch) UnmarshalJSON(b []byte) error { // decode Set var i int -next: for i < len(raw.Set) { var before int if err := json.Unmarshal(raw.Set[i], &before); err != nil { @@ -117,6 +116,8 @@ next: } i++ + var tmp []json.RawMessage + // decode any number of array/string parts for i < len(raw.Set) { next := raw.Set[i] @@ -124,32 +125,21 @@ next: return ErrEncoding } switch next[0] { - default: - continue next - case '"': - - case '[': - + case '"', '[': + tmp = append(tmp, next) + i++ + continue } - - } - - } - - for i := 0; i < len(raw.Set); i += 2 { - var before int - if err := json.Unmarshal(raw.Set[i], &before); err != nil { - return err + break } - var body GumnutDataArray - if err := json.Unmarshal(raw.Set[i+1], &body); err != nil { + out, err := decodeJSONParts(tmp) + if err != nil { return err } - p.Set = append(p.Set, ymodel.Set[GumnutData]{ Before: before, - Body: []GumnutData(body), + Body: []GumnutData(out), }) } From 0de2f9194769d8e148966051c4b749fc4f1c2989 Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Thu, 19 Mar 2026 09:02:00 +1100 Subject: [PATCH 3/4] fix --- client/lib/ymodel/internal/state.ts | 59 +++++++++++++++++++---------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/client/lib/ymodel/internal/state.ts b/client/lib/ymodel/internal/state.ts index 6b2dac2..3967ee0 100644 --- a/client/lib/ymodel/internal/state.ts +++ b/client/lib/ymodel/internal/state.ts @@ -287,7 +287,10 @@ export class RopeState { } public coerceSet(set: SetPart): InternalSet { - const length = set.body.length; + let length = 0; + for (const part of set.body) { + length += part.length; + } const start = set.before - length; // TODO: right now, InteralSet splays out every entry or char with a target ID! @@ -303,17 +306,16 @@ export class RopeState { return out; } - public applySet(int: InternalSet): boolean { - if (int.ids.length === 0) { - return false; - } - + /** + * Applies a single chunk of a {@link InteralSet}, i.e., a string or array chunk. + */ + private internalApplySet(ids: Id[], data: X[] | string): boolean { let i = 0; // fast-path array inserts - if (typeof int.data !== 'string') { - while (i < int.ids.length) { - let id = int.ids[i]; + if (typeof data !== 'string') { + while (i < ids.length) { + let id = ids[i]; const hostId = this.tree.equalAfter(id); if (hostId === undefined) { @@ -326,25 +328,25 @@ export class RopeState { } const index = e.length - 1 - (hostId - id); - e.data[index] = int.data[i]; + e.data[index] = data[i]; ++i; } - if (i === int.ids.length) { + if (i === ids.length) { return true; } } // consume as much contiguous data and set, rinse and repeat - let lastId = this.leftOf(int.ids[0]); + let lastId = this.leftOf(ids[0]); this.ensureIdEdge(lastId); - while (i < int.ids.length) { - let id = int.ids[i]; + while (i < ids.length) { + let id = ids[i]; const lo = i; - while (++i < int.ids.length) { - const nextId = int.ids[i]; + while (++i < ids.length) { + const nextId = ids[i]; if (this.leftOf(nextId) !== id) { break; } @@ -352,13 +354,13 @@ export class RopeState { } const length = i - lo; - const data = int.data.slice(lo, i); + const localData = data.slice(lo, i); this.ensureIdEdge(id); const info = this.rope.lookup(id); if (info.length === length) { - this.rope.adjust(id, data, length); + this.rope.adjust(id, localData, length); } else { // remove all tree entries >lastId { this.tree.remove(candId); } this.rope.deleteTo(lastId, id); - this.rope.insertAfter(lastId, id, length, data); + this.rope.insertAfter(lastId, id, length, localData); } lastId = id; @@ -379,6 +381,23 @@ export class RopeState { return true; } + public applySet(int: InternalSet): boolean { + // consume chunks (typically just one!) + let remainingIds = int.ids; + let any = false; + + for (let i = 0; i < int.data.length; ++i) { + const chunk = int.data[i]; + const ids = remainingIds.slice(0, chunk.length); + remainingIds = remainingIds.slice(chunk.length); + + const change = this.internalApplySet(ids, chunk); + any ||= change; + } + + return any; + } + /** * Finds the {@link Id} directly to the left of the one passed. * @@ -409,7 +428,7 @@ export class RopeState { export type InternalOp = { op: Op; id: Id }; -export type InternalSet = { ids: Id[]; data: X[] | string }; +export type InternalSet = { ids: Id[]; data: (X[] | string)[] }; export type InternalPatch = { run: InternalOp[]; From 072f887643c17bec4d2320425e28437c10e5062f Mon Sep 17 00:00:00 2001 From: Sam Thorogood Date: Thu, 19 Mar 2026 10:53:04 +1100 Subject: [PATCH 4/4] fix work-holder --- client/lib/ymodel/api.test.ts | 14 +-- client/lib/ymodel/internal/state.test.ts | 6 +- client/lib/ymodel/internal/state.ts | 55 +++++++----- client/lib/ymodel/types.test.ts | 4 +- client/lib/ymodel/work-holder.test.ts | 22 ++--- client/lib/ymodel/work-holder.ts | 104 +++++++++++++---------- 6 files changed, 109 insertions(+), 96 deletions(-) diff --git a/client/lib/ymodel/api.test.ts b/client/lib/ymodel/api.test.ts index 6b98da5..0c7e9b4 100644 --- a/client/lib/ymodel/api.test.ts +++ b/client/lib/ymodel/api.test.ts @@ -43,7 +43,7 @@ test('api', () => { api.applyOp({ r: [0], length: 7 }); const id = api.posToId(4); - const xxSet = api.applySet({ before: api.posOf(id), body: 'XX' }); + const xxSet = api.applySet({ before: api.posOf(id), body: ['XX'] }); assert.strictEqual(api.length(), 7); assert.deepStrictEqual(api.readString(), '\ufffd\ufffdXX\ufffd\ufffd\ufffd'); @@ -54,7 +54,7 @@ test('api', () => { yield ['abc']; xxSet.rollback(); - api.applySet({ before: api.posOf(id), body: 'YY' }); + api.applySet({ before: api.posOf(id), body: ['YY'] }); ++calls; }); @@ -68,7 +68,7 @@ test('api', () => { update: { abc: { run: [{ r: [0], length: 1 }], - set: [{ before: 1, body: [999] }], + set: [{ before: 1, body: [[999]] }], }, }, }); @@ -84,13 +84,13 @@ test('prepareForServer', () => { l.perform(function* (byIno) { const api = byIno('abc'); api.applyOp({ r: [0], length: 2 }); - api.applySet({ before: 2, body: 'XX' }); + api.applySet({ before: 2, body: ['XX'] }); }); l.perform(function* (byIno) { const api = byIno('abc'); api.applyOp({ r: [1], length: 1 }); - api.applySet({ before: 2, body: 'x' }); + api.applySet({ before: 2, body: ['x'] }); }); const outer = l.byIno('abc'); @@ -106,7 +106,7 @@ test('prepareForServer', () => { update: { abc: { run: [{ r: [0], length: 2 }], - set: [{ before: -1, body: 'XX' }], + set: [{ before: -1, body: ['XX'] }], }, }, }, @@ -118,7 +118,7 @@ test('prepareForServer', () => { update: { abc: { run: [{ r: [-2], length: 1 }], - set: [{ before: -1, body: 'x' }], + set: [{ before: -1, body: ['x'] }], }, }, }, diff --git a/client/lib/ymodel/internal/state.test.ts b/client/lib/ymodel/internal/state.test.ts index 48546b5..18d3e80 100644 --- a/client/lib/ymodel/internal/state.test.ts +++ b/client/lib/ymodel/internal/state.test.ts @@ -6,15 +6,15 @@ test('state', () => { const r = RopeState.new(0); r.applyOp(r.coerceOp({ r: [0], length: 5 })); - r.applySet(r.coerceSet({ before: 5, body: [100, 1, 2] })); + r.applySet(r.coerceSet({ before: 5, body: [[100, 1, 2]] })); assert.deepStrictEqual(r.readData(), [0, 0, 100, 1, 2]); - r.applySet(r.coerceSet({ before: 3, body: 'abc' })); + r.applySet(r.coerceSet({ before: 3, body: ['abc'] })); assert.strictEqual(r.length(), 5); assert.deepStrictEqual(r.readString(), 'abc\ufffd\ufffd'); assert.deepStrictEqual(r.readData(), [0, 0, 0, 1, 2]); - r.applySet(r.coerceSet({ before: 4, body: [999] })); + r.applySet(r.coerceSet({ before: 4, body: [[999]] })); assert.deepStrictEqual(r.readData(), [0, 0, 0, 999, 2]); }); diff --git a/client/lib/ymodel/internal/state.ts b/client/lib/ymodel/internal/state.ts index 3967ee0..0fdb932 100644 --- a/client/lib/ymodel/internal/state.ts +++ b/client/lib/ymodel/internal/state.ts @@ -309,13 +309,13 @@ export class RopeState { /** * Applies a single chunk of a {@link InteralSet}, i.e., a string or array chunk. */ - private internalApplySet(ids: Id[], data: X[] | string): boolean { + private internalApplySet(chunk: InternalSetChunk): boolean { let i = 0; // fast-path array inserts - if (typeof data !== 'string') { - while (i < ids.length) { - let id = ids[i]; + if (typeof chunk.data !== 'string') { + while (i < chunk.ids.length) { + let id = chunk.ids[i]; const hostId = this.tree.equalAfter(id); if (hostId === undefined) { @@ -328,25 +328,25 @@ export class RopeState { } const index = e.length - 1 - (hostId - id); - e.data[index] = data[i]; + e.data[index] = chunk.data[i]; ++i; } - if (i === ids.length) { + if (i === chunk.ids.length) { return true; } } // consume as much contiguous data and set, rinse and repeat - let lastId = this.leftOf(ids[0]); + let lastId = this.leftOf(chunk.ids[0]); this.ensureIdEdge(lastId); - while (i < ids.length) { - let id = ids[i]; + while (i < chunk.ids.length) { + let id = chunk.ids[i]; const lo = i; - while (++i < ids.length) { - const nextId = ids[i]; + while (++i < chunk.ids.length) { + const nextId = chunk.ids[i]; if (this.leftOf(nextId) !== id) { break; } @@ -354,7 +354,7 @@ export class RopeState { } const length = i - lo; - const localData = data.slice(lo, i); + const localData = chunk.data.slice(lo, i); this.ensureIdEdge(id); const info = this.rope.lookup(id); @@ -382,19 +382,14 @@ export class RopeState { } public applySet(int: InternalSet): boolean { - // consume chunks (typically just one!) - let remainingIds = int.ids; let any = false; - for (let i = 0; i < int.data.length; ++i) { - const chunk = int.data[i]; - const ids = remainingIds.slice(0, chunk.length); - remainingIds = remainingIds.slice(chunk.length); - - const change = this.internalApplySet(ids, chunk); + // consume chunks (typically just one!) + const chunks = chunkInternalSet(int); + for (const chunk of chunks) { + const change = this.internalApplySet(chunk); any ||= change; } - return any; } @@ -446,3 +441,21 @@ export function rollbackForOp(int: InternalOp): InternalOp | undefined { }; } } + +export type InternalSetChunk = { ids: Id[]; data: string | X[] }; + +export function chunkInternalSet(int: InternalSet): InternalSetChunk[] { + // consume chunks (typically just one!) + let remainingIds = int.ids; + const chunks: InternalSetChunk[] = []; + + for (let i = 0; i < int.data.length; ++i) { + const data = int.data[i]; + const ids = remainingIds.slice(0, data.length); + remainingIds = remainingIds.slice(data.length); + + chunks.push({ ids, data }); + } + + return chunks; +} diff --git a/client/lib/ymodel/types.test.ts b/client/lib/ymodel/types.test.ts index 57132d9..7c7c226 100644 --- a/client/lib/ymodel/types.test.ts +++ b/client/lib/ymodel/types.test.ts @@ -11,14 +11,14 @@ test('encodePatch', () => { const wp = encodePatch(p, (x) => x); assert.deepStrictEqual(wp, { r: [-2, 5], - s: [1234, ['hello 🗺️']], + s: [1234, 'hello 🗺️'], }); }); test('decodePatch', () => { const wp: WirePatch = { r: [-2, 5], - s: [1234, ['hello 🗺️']], + s: [1234, 'hello 🗺️'], }; const p = decodePatch(wp, (x) => x as string); diff --git a/client/lib/ymodel/work-holder.test.ts b/client/lib/ymodel/work-holder.test.ts index 3ef5ac1..1ccdbe8 100644 --- a/client/lib/ymodel/work-holder.test.ts +++ b/client/lib/ymodel/work-holder.test.ts @@ -18,11 +18,11 @@ test('generic', () => { assert.strictEqual(w.length(), 5); assert.strictEqual(calls, 2); - w.applySet({ before: 3, body: [1, 2, 3] }); + w.applySet({ before: 3, body: [[1, 2, 3]] }); assert.deepStrictEqual(w.readData(), [1, 2, 3, 0, 0]); // now include some string data - w.applySet({ before: 4, body: 'HI' }); + w.applySet({ before: 4, body: ['HI'] }); assert.deepStrictEqual(w.readData(), [1, 2, 0, 0, 0]); assert.deepStrictEqual(w.readString(), '\ufffd\ufffdHI\ufffd'); @@ -36,13 +36,9 @@ test('generic', () => { }, ]); assert.deepStrictEqual(out?.set, [ - { - before: -4, - body: [1, 2], - }, { before: -2, - body: 'HI', + body: [[1, 2], 'HI'], }, ]); @@ -63,13 +59,9 @@ test('generic', () => { }, ]); assert.deepStrictEqual(out?.set, [ - { - before: -4, - body: [1, 2], - }, { before: -3, - body: 'H', + body: [[1, 2], 'H'], }, ]); @@ -86,13 +78,9 @@ test('generic', () => { }, ]); assert.deepStrictEqual(out?.set, [ - { - before: -4, - body: [1, 2], - }, { before: -2, - body: 'HI', + body: [[1, 2], 'HI'], }, ]); }); diff --git a/client/lib/ymodel/work-holder.ts b/client/lib/ymodel/work-holder.ts index 0b6be76..37f951b 100644 --- a/client/lib/ymodel/work-holder.ts +++ b/client/lib/ymodel/work-holder.ts @@ -1,6 +1,12 @@ import type { Op, Patch, SetPart } from './types.ts'; import type { Id } from './internal/shared.ts'; -import { type InternalPatch, rollbackForOp, RopeState } from './internal/state.ts'; +import { + chunkInternalSet, + type InternalPatch, + type InternalSet, + rollbackForOp, + type RopeState, +} from './internal/state.ts'; export interface ReadApi { length(): number; @@ -132,20 +138,7 @@ export class WorkHolder implements PatchApi { } // step #2: convert step to real - // for now, this creates many 1-length (!) operations that are later merged - for (const set of this.patch.set) { - set.ids.forEach((id, index) => { - if (m.posOfValid(id) === -1) { - return; // don't send now-deleted sets - } - - out.set.push({ - before: combinedLookupId(length, id), - body: set.data.slice(index, index + 1), - }); - }); - } - out.set = mergeSetPart(out.set); + out.set = mergePatchSet(m, combinedLookupId, this.patch.set); return out.run.length || out.set.length ? out : undefined; } @@ -259,47 +252,66 @@ export class WorkHolder implements PatchApi { } } -/** - * Merges the single-entry {@link SetPart} into a contiguous block. - */ -function mergeSetPart(all: SetPart[]): SetPart[] { - const unique = new Map>(); - for (const part of all) { - if (part.body.length !== 1) { - throw new Error(`bad part, had length: ${part.body.length}`); +function mergePatchSet( + m: RopeState, + combinedLookupId: (consumedLength: number, Id: Id) => number, + src: InternalSet[], +) { + const unique = new Map(); + + // aggregate first + for (const set of src) { + const chunks = chunkInternalSet(set); + for (const chunk of chunks) { + chunk.ids.forEach((id, index) => { + if (m.posOfValid(id) === -1) { + return; // don't send now-deleted sets + } + unique.set(combinedLookupId(0, id), chunk.data.slice(index, index + 1)); + }); } - unique.set(part.before, part); } - const sorted = Array.from(unique.values()); - sorted.sort((a, b) => a.before - b.before); + // now merge + let sortedBefore = Array.from(unique.keys()); + sortedBefore.sort((a, b) => a - b); const out: SetPart[] = []; - for (const part of sorted) { - out.push(part); - if (out.length === 1) { - continue; - } - const prev = out[out.length - 2]; - const adjacent = part.before - part.body.length === prev.before; + while (sortedBefore.length) { + // consume contiguous parts + + const start = sortedBefore[0]; + let before = start; - if ( - !adjacent || - (prev.before < 0 && part.before >= 0) || - typeof prev.body !== typeof part.body - ) { - // not adjacent, can't run over zero, or not same type (string/array) - continue; + let i = 0; + while (++i < sortedBefore.length) { + const next = sortedBefore[i]; + if (next === 0 || next !== before + 1) { + break; + } + before = next; } + sortedBefore = sortedBefore.slice(i); - out.pop(); - if (typeof part.body === 'string') { - (prev.body as string) += part.body; - } else { - (prev.body as X[]).push(...part.body); + // we have data of length=i, before + + const body: (X[] | string)[] = []; + + for (let j = 0; j < i; ++j) { + const data = unique.get(start + j)!; + const last = body.at(-1); + + if (typeof last === 'string' && typeof data === 'string') { + body[body.length - 1] = last + data; + } else if (Array.isArray(last) && Array.isArray(data)) { + body[body.length - 1] = last.concat(data); + } else { + body.push(data); + } } - prev.before = part.before; + + out.push({ before, body }); } return out;