From 34e7dce751ade5bf522a87ef68b7e04d8baccf00 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Tue, 21 Apr 2026 23:03:24 -0700 Subject: [PATCH 1/7] Fix review issues in parser, runtime, and docs --- README.md | 13 +++++---- conversions.go | 13 +++++++++ interpreter.go | 70 ++++++++++++++++++++++----------------------- interpreter_test.go | 15 ++++++++-- lexer.go | 11 ++++--- parser.go | 10 +++++++ typecheck.go | 70 +++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 149 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index d2951ce..3fd6dd1 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ result, err := mexpr.Eval("a > b", map[string]interface{}{ // omitted for brevity. l := mexpr.NewLexer("a > b") p := mexpr.NewParser(l) -ast, err := mexpr.Parse() -typeExamples = map[string]interface{}{ +ast, err := p.Parse() +typeExamples := map[string]interface{}{ "a": 2, "b": 1, } @@ -52,7 +52,7 @@ result1, err := interpreter.Run(map[string]interface{}{ "a": 1, "b": 2, }) -result2, err := interpreter.Run(map[string]interfae{}{ +result2, err := interpreter.Run(map[string]interface{}{ "a": 150, "b": 30, }) @@ -79,10 +79,11 @@ When running the interpreter a set of options can be passed in to change behavio ```go // Using the top-level eval -mexpr.Eval(expression, inputObj, StrictMode) +result, err := mexpr.Eval(expression, inputObj, mexpr.StrictMode) // Using an interpreter instance -interpreter.Run(inputObj, StrictMode) +interpreter := mexpr.NewInterpreter(ast, mexpr.StrictMode) +result, err = interpreter.Run(inputObj) ``` ## Syntax @@ -165,7 +166,7 @@ Indexes are zero-based. Slice indexes are optional and are _inclusive_. `foo[1:2 Any value concatenated with a string will result in a string. For example `"id" + 1` will result in `"id1"`. -There is no distinction between strings, bytes, or runes. Everything is treated as a string. +String length, indexing, and slicing are Unicode-aware and operate on runes rather than raw bytes. #### Date Comparisons diff --git a/conversions.go b/conversions.go index 1195302..c92ea17 100644 --- a/conversions.go +++ b/conversions.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" "time" + "unicode/utf8" ) func isNumber(v interface{}) bool { @@ -68,6 +69,18 @@ func toString(v interface{}) string { return fmt.Sprintf("%v", v) } +func stringLength(v string) int { + return utf8.RuneCountInString(v) +} + +func stringIndex(v string, idx int) string { + return string([]rune(v)[idx]) +} + +func stringSlice(v string, start, end int) string { + return string([]rune(v)[start : end+1]) +} + // toTime converts a string value into a time.Time if possible, otherwise // returns a zero time. func toTime(v interface{}) time.Time { diff --git a/interpreter.go b/interpreter.go index a177dcb..4c5f641 100644 --- a/interpreter.go +++ b/interpreter.go @@ -37,8 +37,8 @@ func checkBounds(ast *Node, input any, idx int) Error { } } if v, ok := input.(string); ok { - if idx < 0 || idx >= len(v) { - return NewError(ast.Offset, ast.Length, "invalid index %d for string of length %d", int(idx), len(v)) + if idx < 0 || idx >= stringLength(v) { + return NewError(ast.Offset, ast.Length, "invalid index %d for string of length %d", int(idx), stringLength(v)) } } return nil @@ -97,7 +97,7 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { case "length": // Special pseudo-property to get the value's length. if s, ok := value.(string); ok { - return len(s), nil + return stringLength(s), nil } if a, ok := value.([]any); ok { return len(a), nil @@ -178,11 +178,12 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { return left[int(start) : int(end)+1], nil } left := toString(resultLeft) + leftLen := stringLength(left) if start < 0 { - start += float64(len(left)) + start += float64(leftLen) } if end < 0 { - end += float64(len(left)) + end += float64(leftLen) } if err := checkBounds(ast, left, int(start)); err != nil { return nil, err @@ -193,7 +194,7 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { if err := checkBounds(ast, left, int(end)); err != nil { return nil, err } - return left[int(start) : int(end)+1], nil + return stringSlice(left, int(start), int(end)), nil } if isNumber(resultRight) { idx, err := toNumber(ast, resultRight) @@ -211,12 +212,12 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { } left := toString(resultLeft) if idx < 0 { - idx += float64(len(left)) + idx += float64(stringLength(left)) } if err := checkBounds(ast, left, int(idx)); err != nil { return nil, err } - return string(left[int(idx)]), nil + return stringIndex(left, int(idx)), nil } return nil, NewError(ast.Offset, ast.Length, "array index must be number or slice %v", resultRight) case NodeSlice: @@ -228,9 +229,7 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { if err != nil { return nil, err } - ast.Value.([]any)[0] = resultLeft - ast.Value.([]any)[1] = resultRight - return ast.Value, nil + return []any{resultLeft, resultRight}, nil case NodeLiteral: return ast.Value, nil case NodeSign: @@ -294,7 +293,7 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { return math.Pow(left, right), nil } } - return nil, NewError(ast.Offset, ast.Length, "cannot add incompatible types %v and %v", resultLeft, resultRight) + return nil, NewError(ast.Offset, ast.Length, "cannot operate on incompatible types %v and %v", resultLeft, resultRight) case NodeEqual, NodeNotEqual, NodeLessThan, NodeLessThanEqual, NodeGreaterThan, NodeGreaterThanEqual: resultLeft, err := i.run(ast.Left, value) if err != nil { @@ -335,17 +334,26 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { if err != nil { return nil, err } - resultRight, err := i.run(ast.Right, value) - if err != nil { - return nil, err - } left := toBool(resultLeft) - right := toBool(resultRight) switch ast.Type { case NodeAnd: - return left && right, nil + if !left { + return false, nil + } + resultRight, err := i.run(ast.Right, value) + if err != nil { + return nil, err + } + return toBool(resultRight), nil case NodeOr: - return left || right, nil + if left { + return true, nil + } + resultRight, err := i.run(ast.Right, value) + if err != nil { + return nil, err + } + return toBool(resultRight), nil } case NodeBefore, NodeAfter: resultLeft, err := i.run(ast.Left, value) @@ -389,16 +397,12 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { return false, nil } if m, ok := resultRight.(map[string]any); ok { - if m[toString(resultLeft)] != nil { - return true, nil - } - return false, nil + _, ok := m[toString(resultLeft)] + return ok, nil } if m, ok := resultRight.(map[any]any); ok { - if m[resultLeft] != nil { - return true, nil - } - return false, nil + _, ok := m[resultLeft] + return ok, nil } return strings.Contains(toString(resultRight), toString(resultLeft)), nil case NodeContains: @@ -411,16 +415,12 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { return false, nil } if m, ok := resultLeft.(map[string]any); ok { - if m[toString(resultRight)] != nil { - return true, nil - } - return false, nil + _, ok := m[toString(resultRight)] + return ok, nil } if m, ok := resultLeft.(map[any]any); ok { - if m[resultRight] != nil { - return true, nil - } - return false, nil + _, ok := m[resultRight] + return ok, nil } return strings.Contains(toString(resultLeft), toString(resultRight)), nil case NodeStartsWith: diff --git a/interpreter_test.go b/interpreter_test.go index 9ff9a0a..706e91d 100644 --- a/interpreter_test.go +++ b/interpreter_test.go @@ -55,6 +55,8 @@ func TestInterpreter(t *testing.T) { {expr: "1 < 2 or 1 > 2", output: true}, {expr: "1 < 2 or 2 > 1", output: true}, {expr: `1 and "a"`, output: true}, + {expr: `0 and missing`, input: `{}`, skipTC: true, opts: []InterpreterOption{StrictMode}, output: false}, + {expr: `1 or missing`, input: `{}`, skipTC: true, opts: []InterpreterOption{StrictMode}, output: true}, // Negation {expr: "not (1 < 2)", output: false}, {expr: "not (1 < 2) and (3 < 4)", output: false}, @@ -64,12 +66,16 @@ func TestInterpreter(t *testing.T) { {expr: `"foo" == "foo"`, output: true}, {expr: `"foo" == "bar"`, output: false}, {expr: `"foo\"bar"`, output: `foo"bar`}, + {expr: `"foo`, err: "unterminated string"}, {expr: `"foo" + "bar" == "foobar"`, output: true}, {expr: `foo + "a"`, input: `{"foo": 1}`, output: "1a"}, {expr: `foo + bar`, input: `{"foo": "id", "bar": 1}`, output: "id1"}, {expr: `foo[0]`, input: `{"foo": "hello"}`, output: "h"}, {expr: `foo[-1]`, input: `{"foo": "hello"}`, output: "o"}, {expr: `foo[0:-3]`, input: `{"foo": "hello"}`, output: "hel"}, + {expr: `"é".length`, output: 1}, + {expr: `foo[0]`, input: `{"foo": "éclair"}`, output: "é"}, + {expr: `foo[1:2]`, input: `{"foo": "héllo"}`, output: "él"}, // Unquoted strings {expr: `"foo" == foo`, skipTC: true, output: false}, {expr: `"foo" == foo`, opts: []InterpreterOption{UnquotedStrings}, output: true}, @@ -77,7 +83,7 @@ func TestInterpreter(t *testing.T) { {expr: `foo == foo`, opts: []InterpreterOption{UnquotedStrings}, output: true}, {expr: `foo == foo`, opts: []InterpreterOption{UnquotedStrings, StrictMode}, output: true}, {expr: `foo + 1`, opts: []InterpreterOption{UnquotedStrings}, output: "foo1"}, - {expr: `@.foo + 1`, opts: []InterpreterOption{UnquotedStrings}, err: "cannot add incompatible types"}, + {expr: `@.foo + 1`, opts: []InterpreterOption{UnquotedStrings}, err: "cannot operate on incompatible types"}, {expr: `@.foo + 1`, opts: []InterpreterOption{UnquotedStrings, StrictMode}, err: "cannot get foo"}, {expr: `foo.bar == bar`, opts: []InterpreterOption{UnquotedStrings}, output: false}, {expr: `foo.bar == bar`, skipTC: true, opts: []InterpreterOption{UnquotedStrings}, input: `{"foo": {}}`, output: false}, @@ -96,6 +102,7 @@ func TestInterpreter(t *testing.T) { {expr: "foo[0]", input: `{"foo": [1, 2]}`, output: 1.0}, {expr: "foo[-1]", input: `{"foo": [1, 2]}`, output: 2.0}, {expr: "foo[:1]", input: `{"foo": [1, 2, 3]}`, output: []interface{}{1.0, 2.0}}, + {expr: "foo[:]", input: `{"foo": [1, 2, 3]}`, output: []interface{}{1.0, 2.0, 3.0}}, {expr: "foo[2:]", input: `{"foo": [1, 2, 3]}`, output: []interface{}{3.0}}, {expr: "foo[:-1]", input: `{"foo": [1, 2, 3]}`, output: []interface{}{1.0, 2.0, 3.0}}, {expr: "foo[1 + 2 / 2]", input: `{"foo": [1, 2, 3]}`, output: 3.0}, @@ -111,10 +118,12 @@ func TestInterpreter(t *testing.T) { {expr: `1 < 2 in "this is true"`, output: true}, {expr: `1 < 2 in "this is false"`, output: false}, {expr: `"bar" in foo`, input: `{"foo": {"bar": 1}}`, output: true}, + {expr: `"bar" in foo`, input: `{"foo": {"bar": null}}`, output: true}, // Contains {expr: `"foobar" contains "foo"`, output: true}, {expr: `"foobar" contains "baz"`, output: false}, {expr: `labels contains "foo"`, input: `{"labels": ["foo", "bar"]}`, output: true}, + {expr: `foo contains "bar"`, input: `{"foo": {"bar": null}}`, output: true}, // Starts / ends with {expr: `"foo" startsWith "f"`, output: true}, {expr: `"foo" startsWith "o"`, output: false}, @@ -144,6 +153,8 @@ func TestInterpreter(t *testing.T) { {expr: `items where id > 3`, input: `{}`, skipTC: true, output: nil}, {expr: `foo where method == "GET"`, input: `{"foo": {"op1": {"method": "GET", "path": "/op1"}, "op2": {"method": "PUT", "path": "/op2"}, "op3": {"method": "DELETE", "path": "/op3"}}}`, output: []any{map[string]any{"method": "GET", "path": "/op1"}}}, {expr: `foo where method == "GET"`, inputParsed: map[any]any{"foo": map[any]any{"op1": map[any]any{"method": "GET", "path": "/op1"}, "op2": map[any]any{"method": "PUT", "path": "/op2"}, "op3": map[any]any{"method": "DELETE", "path": "/op3"}}}, output: []any{map[any]any{"method": "GET", "path": "/op1"}}}, + {expr: `items where id > 0`, input: `{"items": [{"id": 1}, "x", {"id": 2}]}`, output: []any{map[string]any{"id": 1.0}, map[string]any{"id": 2.0}}}, + {expr: `foo where id > 0`, input: `{"foo": {"a": "x", "b": {"id": 1}, "c": {"id": 2}}}`, output: []any{map[string]any{"id": 1.0}, map[string]any{"id": 2.0}}}, {expr: `items where id > 3`, input: `{"items": []}`, err: "where clause requires a non-empty array or object"}, {expr: `items where id > 3`, input: `{"items": 1}`, skipTC: true, output: []any{}}, // Order of operations @@ -165,7 +176,7 @@ func TestInterpreter(t *testing.T) { {expr: `1 <`, err: "incomplete expression"}, {expr: `1 +`, err: "incomplete expression"}, {expr: `1 ]`, err: "expected eof but found right-bracket"}, - {expr: `0.5 + 1"`, err: "expected eof but found string"}, + {expr: `0.5 + 1"`, err: "unterminated string"}, {expr: `0.5 > "some kind of string"`, err: "unable to convert to number"}, {expr: `foo beginswith "bar"`, input: `{"foo": "bar"}`, err: "expected eof"}, {expr: `1 / (foo * 1)`, input: `{"foo": 0}`, err: "cannot divide by zero"}, diff --git a/lexer.go b/lexer.go index aacb7fd..f25d98b 100644 --- a/lexer.go +++ b/lexer.go @@ -229,7 +229,7 @@ func (l *lexer) consumeIdentifier() *Token { // consumeString reads runes from the expression until a non-escaped double // quote is encountered. Only double-quoted strings are supported. -func (l *lexer) consumeString() *Token { +func (l *lexer) consumeString() (*Token, Error) { buf := bytes.NewBuffer(make([]byte, 0, 8)) for { r := l.next() @@ -238,12 +238,15 @@ func (l *lexer) consumeString() *Token { buf.WriteRune('"') continue } - if r == -1 || r == '"' { + if r == -1 { + return nil, NewError(l.pos, 1, "unterminated string") + } + if r == '"' { break } buf.WriteRune(r) } - return l.newToken(TokenString, buf.String()) + return l.newToken(TokenString, buf.String()), nil } func (l *lexer) Next() (*Token, Error) { @@ -291,7 +294,7 @@ func (l *lexer) Next() (*Token, Error) { } if r == '"' { - return l.consumeString(), nil + return l.consumeString() } return l.consumeIdentifier(), nil diff --git a/parser.go b/parser.go index 72f8fb7..258ae7b 100644 --- a/parser.go +++ b/parser.go @@ -300,6 +300,16 @@ func (p *parser) nud(t *Token) (*Node, Error) { return &Node{Type: NodeSign, Value: value, Offset: offset, Length: uint8(t.Offset + uint16(t.Length) - offset), Right: result}, nil case TokenSlice: offset := t.Offset + if p.token.Type == TokenRightBracket { + return &Node{ + Type: NodeSlice, + Offset: offset, + Length: t.Length, + Left: &Node{Type: NodeLiteral, Value: 0.0, Offset: offset}, + Right: &Node{Type: NodeLiteral, Value: -1.0, Offset: offset}, + Value: []interface{}{0.0, 0.0}, + }, nil + } result, err := p.parse(bindingPowers[t.Type]) if err != nil { return nil, err diff --git a/typecheck.go b/typecheck.go index fe75a7a..a3034e9 100644 --- a/typecheck.go +++ b/typecheck.go @@ -69,6 +69,46 @@ func newSchema(t valueType) *schema { return &schema{typeName: t} } +func mergeSchema(a, b *schema) *schema { + if a == nil { + return b + } + if b == nil { + return a + } + if a.typeName == typeUnknown || b.typeName == typeUnknown { + return newSchema(typeUnknown) + } + if a.typeName != b.typeName { + return newSchema(typeUnknown) + } + switch a.typeName { + case typeArray: + return &schema{ + typeName: typeArray, + items: mergeSchema(a.items, b.items), + } + case typeObject: + merged := &schema{ + typeName: typeObject, + properties: map[string]*schema{}, + } + for k, v := range a.properties { + merged.properties[k] = v + } + for k, v := range b.properties { + if existing, ok := merged.properties[k]; ok { + merged.properties[k] = mergeSchema(existing, v) + continue + } + merged.properties[k] = v + } + return merged + default: + return a + } +} + func getSchema(v any) *schema { switch i := v.(type) { case bool: @@ -79,8 +119,8 @@ func getSchema(v any) *schema { return schemaString case []any: s := newSchema(typeArray) - if len(i) > 0 { - s.items = getSchema(i[0]) + for _, item := range i { + s.items = mergeSchema(s.items, getSchema(item)) } return s case map[string]any: @@ -165,6 +205,9 @@ func (i *typeChecker) run(ast *Node, value any) (*schema, Error) { } errValue := value if s, ok := value.(*schema); ok { + if s.typeName == typeUnknown { + return newSchema(typeUnknown), nil + } if v, ok := s.properties[ast.Value.(string)]; ok { return v, nil } @@ -213,6 +256,9 @@ func (i *typeChecker) run(ast *Node, value any) (*schema, Error) { if err != nil { return nil, err } + if leftType.typeName == typeUnknown || rightType.typeName == typeUnknown { + return newSchema(typeUnknown), nil + } if !(leftType.isString() || leftType.isArray()) { return nil, NewError(ast.Offset, ast.Length, "can only index strings or arrays but got %v", leftType) } @@ -232,6 +278,11 @@ func (i *typeChecker) run(ast *Node, value any) (*schema, Error) { if err != nil { return nil, err } + if leftType.typeName == typeUnknown || rightType.typeName == typeUnknown { + s := newSchema(typeArray) + s.items = newSchema(typeUnknown) + return s, nil + } if !leftType.isNumber() { return nil, NewError(ast.Offset, ast.Length, "slice index must be a number but found %s", leftType) } @@ -257,6 +308,9 @@ func (i *typeChecker) run(ast *Node, value any) (*schema, Error) { if err != nil { return nil, err } + if leftType.typeName == typeUnknown || rightType.typeName == typeUnknown { + return newSchema(typeUnknown), nil + } if ast.Type == NodeAdd { if leftType.isString() || rightType.isString() { return schemaString, nil @@ -277,6 +331,9 @@ func (i *typeChecker) run(ast *Node, value any) (*schema, Error) { if err != nil { return nil, err } + if leftType.typeName == typeUnknown || rightType.typeName == typeUnknown { + return schemaBool, nil + } if !leftType.isNumber() || !rightType.isNumber() { return nil, NewError(ast.Offset, ast.Length, "cannot compare %s with %s", leftType, rightType) } @@ -293,13 +350,14 @@ func (i *typeChecker) run(ast *Node, value any) (*schema, Error) { return nil, err } if leftType.isObject() { - keys := mapKeys(leftType.properties) + objectType := leftType + keys := mapKeys(objectType.properties) sort.Strings(keys) if len(keys) > 0 { - // Pick the first prop as the representative item type. - prop := leftType.properties[keys[0]] leftType = newSchema(typeArray) - leftType.items = prop + for _, key := range keys { + leftType.items = mergeSchema(leftType.items, objectType.properties[key]) + } } } if !leftType.isArray() || leftType.items == nil { From 5590fc4d2d9b1694a2d7d05287678ac36440d396 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Tue, 21 Apr 2026 23:12:14 -0700 Subject: [PATCH 2/7] Modernize Go type aliases and examples --- README.md | 8 ++++---- conversions.go | 22 +++++++++++----------- error.go | 2 +- interpreter_test.go | 34 +++++++++++++++++----------------- parser.go | 10 +++++----- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 3fd6dd1..5c6fbf5 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Try it out on the [Go Playground](https://play.golang.org/p/Z0UcEBgfxu_r)! You c import "github.com/danielgtaylor/mexpr" // Convenience for lexing/parsing/running in one step: -result, err := mexpr.Eval("a > b", map[string]interface{}{ +result, err := mexpr.Eval("a > b", map[string]any{ "a": 2, "b": 1, }) @@ -42,17 +42,17 @@ result, err := mexpr.Eval("a > b", map[string]interface{}{ l := mexpr.NewLexer("a > b") p := mexpr.NewParser(l) ast, err := p.Parse() -typeExamples := map[string]interface{}{ +typeExamples := map[string]any{ "a": 2, "b": 1, } err := mexpr.TypeCheck(ast, typeExamples) interpreter := mexpr.NewInterpreter(ast) -result1, err := interpreter.Run(map[string]interface{}{ +result1, err := interpreter.Run(map[string]any{ "a": 1, "b": 2, }) -result2, err := interpreter.Run(map[string]interface{}{ +result2, err := interpreter.Run(map[string]any{ "a": 150, "b": 30, }) diff --git a/conversions.go b/conversions.go index c92ea17..1364822 100644 --- a/conversions.go +++ b/conversions.go @@ -7,7 +7,7 @@ import ( "unicode/utf8" ) -func isNumber(v interface{}) bool { +func isNumber(v any) bool { switch v.(type) { case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: return true @@ -17,7 +17,7 @@ func isNumber(v interface{}) bool { return false } -func toNumber(ast *Node, v interface{}) (float64, Error) { +func toNumber(ast *Node, v any) (float64, Error) { switch n := v.(type) { case float64: return n, nil @@ -47,7 +47,7 @@ func toNumber(ast *Node, v interface{}) (float64, Error) { return 0, NewError(ast.Offset, ast.Length, "unable to convert to number: %v", v) } -func isString(v interface{}) bool { +func isString(v any) bool { switch v.(type) { case string, rune, byte, []byte: return true @@ -55,7 +55,7 @@ func isString(v interface{}) bool { return false } -func toString(v interface{}) string { +func toString(v any) string { switch s := v.(type) { case string: return s @@ -83,7 +83,7 @@ func stringSlice(v string, start, end int) string { // toTime converts a string value into a time.Time if possible, otherwise // returns a zero time. -func toTime(v interface{}) time.Time { +func toTime(v any) time.Time { vStr := toString(v) if t, err := time.Parse(time.RFC3339, vStr); err == nil { return t @@ -97,14 +97,14 @@ func toTime(v interface{}) time.Time { return time.Time{} } -func isSlice(v interface{}) bool { - if _, ok := v.([]interface{}); ok { +func isSlice(v any) bool { + if _, ok := v.([]any); ok { return true } return false } -func toBool(v interface{}) bool { +func toBool(v any) bool { switch n := v.(type) { case bool: return n @@ -136,9 +136,9 @@ func toBool(v interface{}) bool { return len(n) > 0 case []byte: return len(n) > 0 - case []interface{}: + case []any: return len(n) > 0 - case map[string]interface{}: + case map[string]any: return len(n) > 0 case map[any]any: return len(n) > 0 @@ -149,7 +149,7 @@ func toBool(v interface{}) bool { // normalize an input for equality checks. All numbers -> float64, []byte to // string, etc. Since `rune` is an alias for int32, we can't differentiate it // for comparison with strings. -func normalize(v interface{}) interface{} { +func normalize(v any) any { switch n := v.(type) { case int: return float64(n) diff --git a/error.go b/error.go index e0be05d..873d4f7 100644 --- a/error.go +++ b/error.go @@ -47,7 +47,7 @@ func (e *exprErr) Pretty(source string) string { } // NewError creates a new error at a specific location. -func NewError(offset uint16, length uint8, format string, a ...interface{}) Error { +func NewError(offset uint16, length uint8, format string, a ...any) Error { return &exprErr{ offset: offset, length: length, diff --git a/interpreter_test.go b/interpreter_test.go index 706e91d..746d2cc 100644 --- a/interpreter_test.go +++ b/interpreter_test.go @@ -15,7 +15,7 @@ func TestInterpreter(t *testing.T) { skipTC bool opts []InterpreterOption err string - output interface{} + output any } cases := []test{ // Add/sub @@ -96,20 +96,20 @@ func TestInterpreter(t *testing.T) { {expr: "foo.bar.baz", input: `{"foo": {"bar": {"baz": 1.0}}}`, output: 1.0}, {expr: `foo == "foo"`, input: `{"foo": "foo"}`, output: true}, {expr: `foo.in.not`, input: `{"foo": {"in": {"not": 1}}}`, output: 1.0}, - {expr: `@`, input: `{"hello": "world"}`, output: map[string]interface{}{"hello": "world"}}, + {expr: `@`, input: `{"hello": "world"}`, output: map[string]any{"hello": "world"}}, {expr: `hello.@`, input: `{"hello": "world"}`, output: "world"}, // Arrays {expr: "foo[0]", input: `{"foo": [1, 2]}`, output: 1.0}, {expr: "foo[-1]", input: `{"foo": [1, 2]}`, output: 2.0}, - {expr: "foo[:1]", input: `{"foo": [1, 2, 3]}`, output: []interface{}{1.0, 2.0}}, - {expr: "foo[:]", input: `{"foo": [1, 2, 3]}`, output: []interface{}{1.0, 2.0, 3.0}}, - {expr: "foo[2:]", input: `{"foo": [1, 2, 3]}`, output: []interface{}{3.0}}, - {expr: "foo[:-1]", input: `{"foo": [1, 2, 3]}`, output: []interface{}{1.0, 2.0, 3.0}}, + {expr: "foo[:1]", input: `{"foo": [1, 2, 3]}`, output: []any{1.0, 2.0}}, + {expr: "foo[:]", input: `{"foo": [1, 2, 3]}`, output: []any{1.0, 2.0, 3.0}}, + {expr: "foo[2:]", input: `{"foo": [1, 2, 3]}`, output: []any{3.0}}, + {expr: "foo[:-1]", input: `{"foo": [1, 2, 3]}`, output: []any{1.0, 2.0, 3.0}}, {expr: "foo[1 + 2 / 2]", input: `{"foo": [1, 2, 3]}`, output: 3.0}, - {expr: "foo[1:1 + 2]", input: `{"foo": [1, 2, 3, 4]}`, output: []interface{}{2.0, 3.0, 4.0}}, - {expr: "foo[foo[0]:bar.baz * 1^2]", input: `{"foo": [1, 2, 3, 4], "bar": {"baz": 3}}`, output: []interface{}{2.0, 3.0, 4.0}}, - {expr: "foo + bar", input: `{"foo": [1, 2], "bar": [3, 4]}`, output: []interface{}{1.0, 2.0, 3.0, 4.0}}, - {expr: "foo[bar]", input: `{"foo": [1, 2, 3], "bar": [0, 1]}`, output: []interface{}{1.0, 2.0}}, + {expr: "foo[1:1 + 2]", input: `{"foo": [1, 2, 3, 4]}`, output: []any{2.0, 3.0, 4.0}}, + {expr: "foo[foo[0]:bar.baz * 1^2]", input: `{"foo": [1, 2, 3, 4], "bar": {"baz": 3}}`, output: []any{2.0, 3.0, 4.0}}, + {expr: "foo + bar", input: `{"foo": [1, 2], "bar": [3, 4]}`, output: []any{1.0, 2.0, 3.0, 4.0}}, + {expr: "foo[bar]", input: `{"foo": [1, 2, 3], "bar": [0, 1]}`, output: []any{1.0, 2.0}}, // In {expr: `"foo" in "foobar"`, output: true}, {expr: `"foo" in bar`, input: `{"bar": ["foo", "other"]}`, output: true}, @@ -146,8 +146,8 @@ func TestInterpreter(t *testing.T) { {expr: `str.lower`, input: `{"str": "ABCD"}`, output: "abcd"}, {expr: `str.lower == abcd`, input: `{"str": "ABCD"}`, opts: []InterpreterOption{UnquotedStrings}, skipTC: true, output: true}, // Where - {expr: `items where id > 3`, input: `{"items": [{"id": 1}, {"id": 3}, {"id": 5}, {"id": 7}]}`, output: []interface{}{map[string]interface{}{"id": 5.0}, map[string]interface{}{"id": 7.0}}}, - {expr: `items where id > 3 where labels contains "foo"`, input: `{"items": [{"id": 1, "labels": ["foo"]}, {"id": 3}, {"id": 5, "labels": ["foo"]}, {"id": 7}]}`, output: []interface{}{map[string]interface{}{"id": 5.0, "labels": []interface{}{"foo"}}}}, + {expr: `items where id > 3`, input: `{"items": [{"id": 1}, {"id": 3}, {"id": 5}, {"id": 7}]}`, output: []any{map[string]any{"id": 5.0}, map[string]any{"id": 7.0}}}, + {expr: `items where id > 3 where labels contains "foo"`, input: `{"items": [{"id": 1, "labels": ["foo"]}, {"id": 3}, {"id": 5, "labels": ["foo"]}, {"id": 7}]}`, output: []any{map[string]any{"id": 5.0, "labels": []any{"foo"}}}}, {expr: `(items where id > 3).length == 2`, input: `{"items": [{"id": 1}, {"id": 3}, {"id": 5}, {"id": 7}]}`, output: true}, {expr: `not (items where id > 3)`, input: `{"items": [{"id": 1}, {"id": 3}, {"id": 5}, {"id": 7}]}`, output: false}, {expr: `items where id > 3`, input: `{}`, skipTC: true, output: nil}, @@ -264,7 +264,7 @@ func Benchmark(b *testing.B) { name string mexpr string expr string - result interface{} + result any }{ {"field", `baz`, `baz`, "value"}, {"comparison", `foo.bar > 1000`, `foo.bar > 1000`, true}, @@ -280,13 +280,13 @@ func Benchmark(b *testing.B) { }, } - var r interface{} - input := map[string]interface{}{ - "foo": map[string]interface{}{ + var r any + input := map[string]any{ + "foo": map[string]any{ "bar": 1000000000.0, }, "baz": "value", - "arr": []interface{}{1, 2, 3}, + "arr": []any{1, 2, 3}, } for _, bm := range benchmarks { diff --git a/parser.go b/parser.go index 258ae7b..d3a82d1 100644 --- a/parser.go +++ b/parser.go @@ -48,7 +48,7 @@ type Node struct { Offset uint16 Left *Node Right *Node - Value interface{} + Value any } // String converts the node to a string representation (basically the node name @@ -307,7 +307,7 @@ func (p *parser) nud(t *Token) (*Node, Error) { Length: t.Length, Left: &Node{Type: NodeLiteral, Value: 0.0, Offset: offset}, Right: &Node{Type: NodeLiteral, Value: -1.0, Offset: offset}, - Value: []interface{}{0.0, 0.0}, + Value: []any{0.0, 0.0}, }, nil } result, err := p.parse(bindingPowers[t.Type]) @@ -317,7 +317,7 @@ func (p *parser) nud(t *Token) (*Node, Error) { // Create a dummy left node with value 0, the start of the slice. This also // sets the parent node's value to a pre-allocated list of [0, 0] which is // used later by the interpreter. It prevents additional allocations. - return &Node{Type: NodeSlice, Offset: offset, Length: uint8(t.Offset + uint16(t.Length) - offset), Left: &Node{Type: NodeLiteral, Value: 0.0, Offset: offset}, Right: result, Value: []interface{}{0.0, 0.0}}, nil + return &Node{Type: NodeSlice, Offset: offset, Length: uint8(t.Offset + uint16(t.Length) - offset), Left: &Node{Type: NodeLiteral, Value: 0.0, Offset: offset}, Right: result, Value: []any{0.0, 0.0}}, nil case TokenRightParen: return nil, NewError(t.Offset, t.Length, "unexpected right-paren") case TokenRightBracket: @@ -432,13 +432,13 @@ func (p *parser) led(t *Token, n *Node) (*Node, Error) { // This sets the parent node's value to a pre-allocated list of [0, 0] // which is used later by the interpreter. It prevents additional // allocations. - return &Node{Type: NodeSlice, Offset: t.Offset, Length: t.Length, Left: n, Right: &Node{Type: NodeLiteral, Offset: t.Offset, Value: -1.0}, Value: []interface{}{0.0, 0.0}}, nil + return &Node{Type: NodeSlice, Offset: t.Offset, Length: t.Length, Left: n, Right: &Node{Type: NodeLiteral, Offset: t.Offset, Value: -1.0}, Value: []any{0.0, 0.0}}, nil } nn, err := p.newNodeParseRight(n, t, NodeSlice, bindingPowers[t.Type]) if err != nil { return nil, err } - nn.Value = []interface{}{0.0, 0.0} + nn.Value = []any{0.0, 0.0} return nn, nil } return nil, NewError(t.Offset, t.Length, "unexpected token %s", t.Type) From 4170b9866cdc76fa18f114c3ddd53d59571febf1 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Tue, 21 Apr 2026 23:15:19 -0700 Subject: [PATCH 3/7] Modernize GitHub Actions workflow --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bfc1182..0730951 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest name: Build & Test steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v5 - name: Setup go - uses: actions/setup-go@v1 + uses: actions/setup-go@v6 with: - go-version: "1.18" + go-version-file: go.mod - run: go test -coverprofile=coverage.txt -covermode=atomic ./... - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v5 From 85378e1653803095806977406f3b77430332fd78 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Tue, 21 Apr 2026 23:21:32 -0700 Subject: [PATCH 4/7] Fix flaky map where test ordering --- interpreter_test.go | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/interpreter_test.go b/interpreter_test.go index b78ff21..d260086 100644 --- a/interpreter_test.go +++ b/interpreter_test.go @@ -13,6 +13,7 @@ func TestInterpreter(t *testing.T) { input string inputParsed any skipTC bool + unordered bool opts []InterpreterOption err string output any @@ -154,7 +155,7 @@ func TestInterpreter(t *testing.T) { {expr: `foo where method == "GET"`, input: `{"foo": {"op1": {"method": "GET", "path": "/op1"}, "op2": {"method": "PUT", "path": "/op2"}, "op3": {"method": "DELETE", "path": "/op3"}}}`, output: []any{map[string]any{"method": "GET", "path": "/op1"}}}, {expr: `foo where method == "GET"`, inputParsed: map[any]any{"foo": map[any]any{"op1": map[any]any{"method": "GET", "path": "/op1"}, "op2": map[any]any{"method": "PUT", "path": "/op2"}, "op3": map[any]any{"method": "DELETE", "path": "/op3"}}}, output: []any{map[any]any{"method": "GET", "path": "/op1"}}}, {expr: `items where id > 0`, input: `{"items": [{"id": 1}, "x", {"id": 2}]}`, output: []any{map[string]any{"id": 1.0}, map[string]any{"id": 2.0}}}, - {expr: `foo where id > 0`, input: `{"foo": {"a": "x", "b": {"id": 1}, "c": {"id": 2}}}`, output: []any{map[string]any{"id": 1.0}, map[string]any{"id": 2.0}}}, + {expr: `foo where id > 0`, input: `{"foo": {"a": "x", "b": {"id": 1}, "c": {"id": 2}}}`, unordered: true, output: []any{map[string]any{"id": 1.0}, map[string]any{"id": 2.0}}}, {expr: `items where id > 3`, input: `{"items": []}`, err: "where clause requires a non-empty array or object"}, {expr: `items where id > 3`, input: `{"items": 1}`, skipTC: true, output: []any{}}, // Order of operations @@ -235,6 +236,37 @@ func TestInterpreter(t *testing.T) { if err != nil { t.Fatal(err.Pretty(tc.expr)) } + if tc.unordered { + expectedSlice, ok := tc.output.([]any) + if !ok { + t.Fatalf("unordered test expected []any output, got %T", tc.output) + } + resultSlice, ok := result.([]any) + if !ok { + t.Fatalf("unordered test expected []any result, got %T", result) + } + if len(expectedSlice) != len(resultSlice) { + t.Fatalf("expected %v but found %v", tc.output, result) + } + used := make([]bool, len(resultSlice)) + for _, expected := range expectedSlice { + matched := false + for idx, actual := range resultSlice { + if used[idx] { + continue + } + if reflect.DeepEqual(expected, actual) { + used[idx] = true + matched = true + break + } + } + if !matched { + t.Fatalf("expected %v but found %v", tc.output, result) + } + } + return + } if !reflect.DeepEqual(tc.output, result) { t.Fatalf("expected %v but found %v", tc.output, result) } From 606b7a81c931844a9204c64a43ffea9e5016b1df Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Tue, 21 Apr 2026 23:30:35 -0700 Subject: [PATCH 5/7] Address PR review feedback --- README.md | 2 +- conversions.go | 22 ++++++++++++++++++++-- interpreter.go | 5 +++-- lexer.go | 3 ++- parser.go | 13 +++---------- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2042778..2df9146 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ typeExamples := map[string]any{ "a": 2, "b": 1, } -err := mexpr.TypeCheck(ast, typeExamples) +err = mexpr.TypeCheck(ast, typeExamples) interpreter := mexpr.NewInterpreter(ast) result1, err := interpreter.Run(map[string]any{ "a": 1, diff --git a/conversions.go b/conversions.go index dffa3b0..0f3d562 100644 --- a/conversions.go +++ b/conversions.go @@ -107,12 +107,30 @@ func stringLength(v string) int { return utf8.RuneCountInString(v) } +func runeIndexToByteOffset(v string, idx int) int { + if idx <= 0 { + return 0 + } + + offset := 0 + for i := 0; i < idx && offset < len(v); i++ { + _, size := utf8.DecodeRuneInString(v[offset:]) + offset += size + } + + return offset +} + func stringIndex(v string, idx int) string { - return string([]rune(v)[idx]) + start := runeIndexToByteOffset(v, idx) + end := runeIndexToByteOffset(v, idx+1) + return v[start:end] } func stringSlice(v string, start, end int) string { - return string([]rune(v)[start : end+1]) + from := runeIndexToByteOffset(v, start) + to := runeIndexToByteOffset(v, end+1) + return v[from:to] } // toTime converts a string value into a time.Time if possible, otherwise diff --git a/interpreter.go b/interpreter.go index 9f96c23..5484946 100644 --- a/interpreter.go +++ b/interpreter.go @@ -38,8 +38,9 @@ func checkBounds(ast *Node, input any, idx int) Error { } } if v, ok := input.(string); ok { - if idx < 0 || idx >= stringLength(v) { - return NewError(ast.Offset, ast.Length, "invalid index %d for string of length %d", int(idx), stringLength(v)) + length := stringLength(v) + if idx < 0 || idx >= length { + return NewError(ast.Offset, ast.Length, "invalid index %d for string of length %d", int(idx), length) } } return nil diff --git a/lexer.go b/lexer.go index 77edabe..55ac07e 100644 --- a/lexer.go +++ b/lexer.go @@ -236,6 +236,7 @@ func (l *lexer) consumeIdentifier() *Token { // quote is encountered. Only double-quoted strings are supported. func (l *lexer) consumeString() (*Token, Error) { buf := bytes.NewBuffer(make([]byte, 0, 8)) + offset := l.pos - l.lastWidth for { r := l.next() if r == '\\' && l.peek() == '"' { @@ -244,7 +245,7 @@ func (l *lexer) consumeString() (*Token, Error) { continue } if r == -1 { - return nil, NewError(l.pos, 1, "unterminated string") + return nil, NewError(offset, 1, "unterminated string") } if r == '"' { break diff --git a/parser.go b/parser.go index be45f58..7d7c0b7 100644 --- a/parser.go +++ b/parser.go @@ -311,17 +311,14 @@ func (p *parser) nud(t *Token) (*Node, Error) { Length: t.Length, Left: &Node{Type: NodeLiteral, Value: 0.0, Offset: offset}, Right: &Node{Type: NodeLiteral, Value: -1.0, Offset: offset}, - Value: []any{0.0, 0.0}, }, nil } result, err := p.parse(bindingPowers[t.Type]) if err != nil { return nil, err } - // Create a dummy left node with value 0, the start of the slice. This also - // sets the parent node's value to a pre-allocated list of [0, 0] which is - // used later by the interpreter. It prevents additional allocations. - return &Node{Type: NodeSlice, Offset: offset, Length: uint8(t.Offset + uint16(t.Length) - offset), Left: &Node{Type: NodeLiteral, Value: 0.0, Offset: offset}, Right: result, Value: []any{0.0, 0.0}}, nil + // Create a dummy left node with value 0, the start of the slice. + return &Node{Type: NodeSlice, Offset: offset, Length: uint8(t.Offset + uint16(t.Length) - offset), Left: &Node{Type: NodeLiteral, Value: 0.0, Offset: offset}, Right: result}, nil case TokenRightParen: return nil, NewError(t.Offset, t.Length, "unexpected right-paren") case TokenRightBracket: @@ -433,16 +430,12 @@ func (p *parser) led(t *Token, n *Node) (*Node, Error) { return p.ensure(n, err, TokenRightBracket) case TokenSlice: if p.token.Type == TokenRightBracket { - // This sets the parent node's value to a pre-allocated list of [0, 0] - // which is used later by the interpreter. It prevents additional - // allocations. - return &Node{Type: NodeSlice, Offset: t.Offset, Length: t.Length, Left: n, Right: &Node{Type: NodeLiteral, Offset: t.Offset, Value: -1.0}, Value: []any{0.0, 0.0}}, nil + return &Node{Type: NodeSlice, Offset: t.Offset, Length: t.Length, Left: n, Right: &Node{Type: NodeLiteral, Offset: t.Offset, Value: -1.0}}, nil } nn, err := p.newNodeParseRight(n, t, NodeSlice, bindingPowers[t.Type]) if err != nil { return nil, err } - nn.Value = []any{0.0, 0.0} return nn, nil case TokenLeftParen: if n.Type != NodeIdentifier { From a1167bebabf89f2e76aac8a8dfae237d3038e7e2 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Wed, 22 Apr 2026 08:51:35 -0700 Subject: [PATCH 6/7] Optimize slice and string indexing paths --- conversions.go | 25 +++++++++++------ interpreter.go | 75 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/conversions.go b/conversions.go index 0f3d562..31810c2 100644 --- a/conversions.go +++ b/conversions.go @@ -107,29 +107,36 @@ func stringLength(v string) int { return utf8.RuneCountInString(v) } -func runeIndexToByteOffset(v string, idx int) int { - if idx <= 0 { - return 0 +func runeRangeToByteOffsets(v string, startIdx, endIdx int) (int, int) { + if startIdx <= 0 { + startIdx = 0 + } + if endIdx < startIdx { + endIdx = startIdx } offset := 0 - for i := 0; i < idx && offset < len(v); i++ { + for i := 0; i < startIdx && offset < len(v); i++ { + _, size := utf8.DecodeRuneInString(v[offset:]) + offset += size + } + + startOffset := offset + for i := startIdx; i < endIdx && offset < len(v); i++ { _, size := utf8.DecodeRuneInString(v[offset:]) offset += size } - return offset + return startOffset, offset } func stringIndex(v string, idx int) string { - start := runeIndexToByteOffset(v, idx) - end := runeIndexToByteOffset(v, idx+1) + start, end := runeRangeToByteOffsets(v, idx, idx+1) return v[start:end] } func stringSlice(v string, start, end int) string { - from := runeIndexToByteOffset(v, start) - to := runeIndexToByteOffset(v, end+1) + from, to := runeRangeToByteOffsets(v, start, end+1) return v[from:to] } diff --git a/interpreter.go b/interpreter.go index 5484946..cbdea15 100644 --- a/interpreter.go +++ b/interpreter.go @@ -38,10 +38,14 @@ func checkBounds(ast *Node, input any, idx int) Error { } } if v, ok := input.(string); ok { - length := stringLength(v) - if idx < 0 || idx >= length { - return NewError(ast.Offset, ast.Length, "invalid index %d for string of length %d", int(idx), length) - } + return checkStringBounds(ast, stringLength(v), idx) + } + return nil +} + +func checkStringBounds(ast *Node, length, idx int) Error { + if idx < 0 || idx >= length { + return NewError(ast.Offset, ast.Length, "invalid index %d for string of length %d", idx, length) } return nil } @@ -166,6 +170,60 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { if !isSlice(resultLeft) && !isString(resultLeft) { return nil, NewError(ast.Offset, ast.Length, "can only index strings or arrays but got %v", resultLeft) } + if ast.Right != nil && ast.Right.Type == NodeSlice { + startValue, err := i.run(ast.Right.Left, value) + if err != nil { + return nil, err + } + endValue, err := i.run(ast.Right.Right, value) + if err != nil { + return nil, err + } + start, err := toNumber(ast.Right.Left, startValue) + if err != nil { + return nil, err + } + end, err := toNumber(ast.Right.Right, endValue) + if err != nil { + return nil, err + } + if left, ok := resultLeft.([]any); ok { + if start < 0 { + start += float64(len(left)) + } + if end < 0 { + end += float64(len(left)) + } + if err := checkBounds(ast, left, int(start)); err != nil { + return nil, err + } + if err := checkBounds(ast, left, int(end)); err != nil { + return nil, err + } + if int(start) > int(end) { + return nil, NewError(ast.Offset, ast.Length, "slice start cannot be greater than end") + } + return left[int(start) : int(end)+1], nil + } + left := toString(resultLeft) + leftLen := stringLength(left) + if start < 0 { + start += float64(leftLen) + } + if end < 0 { + end += float64(leftLen) + } + if err := checkStringBounds(ast, leftLen, int(start)); err != nil { + return nil, err + } + if int(start) > int(end) { + return nil, NewError(ast.Offset, ast.Length, "string slice start cannot be greater than end") + } + if err := checkStringBounds(ast, leftLen, int(end)); err != nil { + return nil, err + } + return stringSlice(left, int(start), int(end)), nil + } resultRight, err := i.run(ast.Right, value) if err != nil { return nil, err @@ -205,13 +263,13 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { if end < 0 { end += float64(leftLen) } - if err := checkBounds(ast, left, int(start)); err != nil { + if err := checkStringBounds(ast, leftLen, int(start)); err != nil { return nil, err } if int(start) > int(end) { return nil, NewError(ast.Offset, ast.Length, "string slice start cannot be greater than end") } - if err := checkBounds(ast, left, int(end)); err != nil { + if err := checkStringBounds(ast, leftLen, int(end)); err != nil { return nil, err } return stringSlice(left, int(start), int(end)), nil @@ -231,10 +289,11 @@ func (i *interpreter) run(ast *Node, value any) (any, Error) { return left[int(idx)], nil } left := toString(resultLeft) + leftLen := stringLength(left) if idx < 0 { - idx += float64(stringLength(left)) + idx += float64(leftLen) } - if err := checkBounds(ast, left, int(idx)); err != nil { + if err := checkStringBounds(ast, leftLen, int(idx)); err != nil { return nil, err } return stringIndex(left, int(idx)), nil From 604a957c7ae62f6410d290609745555f29655518 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Wed, 22 Apr 2026 09:23:08 -0700 Subject: [PATCH 7/7] Fix escaped string token offsets --- lexer.go | 5 ++++- lexer_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 lexer_test.go diff --git a/lexer.go b/lexer.go index 55ac07e..7a72944 100644 --- a/lexer.go +++ b/lexer.go @@ -252,7 +252,10 @@ func (l *lexer) consumeString() (*Token, Error) { } buf.WriteRune(r) } - return l.newToken(TokenString, buf.String()), nil + tok := l.newToken(TokenString, buf.String()) + tok.Offset = offset + tok.Length = uint8(l.pos - offset) + return tok, nil } func (l *lexer) Next() (*Token, Error) { diff --git a/lexer_test.go b/lexer_test.go new file mode 100644 index 0000000..9b613a0 --- /dev/null +++ b/lexer_test.go @@ -0,0 +1,40 @@ +package mexpr + +import "testing" + +func TestEscapedStringTokenOffsets(t *testing.T) { + expr := `"a\"b"` + + l := NewLexer(expr) + tok, err := l.Next() + if err != nil { + t.Fatal(err) + } + + if tok.Type != TokenString { + t.Fatalf("expected string token, got %v", tok.Type) + } + if tok.Value != `a"b` { + t.Fatalf("expected decoded string value %q, got %q", `a"b`, tok.Value) + } + if tok.Offset != 0 { + t.Fatalf("expected token offset 0, got %d", tok.Offset) + } + if tok.Length != uint8(len(expr)) { + t.Fatalf("expected token length %d, got %d", len(expr), tok.Length) + } + + ast, err := Parse(expr, nil) + if err != nil { + t.Fatal(err) + } + if ast == nil { + t.Fatal("expected ast, got nil") + } + if ast.Offset != 0 { + t.Fatalf("expected node offset 0, got %d", ast.Offset) + } + if ast.Length != uint8(len(expr)) { + t.Fatalf("expected node length %d, got %d", len(expr), ast.Length) + } +}