From 59083c49b7b0d35116aca49d276a4dfd5569489e Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 31 Jul 2025 11:51:25 +1000 Subject: [PATCH 1/2] Added VarInt parser --- array.go | 19 +++++++++-- int.go | 49 +++++++++++++++++++++++++++ models.go | 3 ++ parser.go | 45 +++---------------------- parser_test.go | 16 ++++++++- pointer.go | 2 +- protocol.go | 15 +++++++++ utils.go | 8 +++++ varint.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 204 insertions(+), 44 deletions(-) create mode 100644 int.go create mode 100644 varint.go diff --git a/array.go b/array.go index 57d2ae8..a211d44 100644 --- a/array.go +++ b/array.go @@ -11,6 +11,10 @@ import ( "www.velocidex.com/golang/vfilter" ) +var ( + NotFoundError = errors.New("NotFoundError") +) + type ArrayParserOptions struct { Type string TypeOptions *ordereddict.Dict @@ -184,7 +188,18 @@ func (self *ArrayObject) SetParent(parent *StructObject) { } func (self *ArrayObject) Contents() []interface{} { - return self.contents + res := make([]interface{}, 0, len(self.contents)) + for _, v := range self.contents { + res = append(res, ValueOf(v)) + } + return res +} + +func (self *ArrayObject) Get(i int64) (interface{}, error) { + if i < 0 || i > int64(len(self.contents)) { + return nil, NotFoundError + } + return self.contents[i], nil } func (self *ArrayObject) Size() int { @@ -200,5 +215,5 @@ func (self *ArrayObject) End() int64 { } func (self *ArrayObject) MarshalJSON() ([]byte, error) { - return json.Marshal(self.contents) + return json.Marshal(self.Contents()) } diff --git a/int.go b/int.go new file mode 100644 index 0000000..510a02d --- /dev/null +++ b/int.go @@ -0,0 +1,49 @@ +package vtypes + +import ( + "errors" + "fmt" + "io" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/vfilter" +) + +// Parse various sizes of ints. +type IntParser struct { + type_name string + size int + converter func(buf []byte) interface{} +} + +// IntParser does not take options +func (self *IntParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { + return self, nil +} + +func (self *IntParser) Size() int { + return self.size +} + +func (self *IntParser) DebugString(scope vfilter.Scope, offset int64, reader io.ReaderAt) string { + return fmt.Sprintf("[%s] %#0x", + self.type_name, self.Parse(scope, reader, offset)) +} + +func (self *IntParser) Parse(scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { + buf := make([]byte, 8) + + n, err := reader.ReadAt(buf, offset) + if n == 0 || (err != nil && !errors.Is(err, io.EOF)) { + return 0 + } + return self.converter(buf) +} + +func NewIntParser(type_name string, size int, converter func(buf []byte) interface{}) *IntParser { + return &IntParser{ + type_name: type_name, + size: size, + converter: converter, + } +} diff --git a/models.go b/models.go index 2640c74..0cb5153 100644 --- a/models.go +++ b/models.go @@ -104,6 +104,9 @@ func AddModel(profile *Profile) { return int64(binary.BigEndian.Uint64(buf)) }) + // Var ints like in protobufs. + profile.types["leb128"] = &Leb128Parser{} + profile.types["sleb128"] = &Sleb128Parser{} profile.types["Array"] = &ArrayParser{} profile.types["String"] = &StringParser{} profile.types["Value"] = &ValueParser{} diff --git a/parser.go b/parser.go index bf86cbf..17fd999 100644 --- a/parser.go +++ b/parser.go @@ -2,7 +2,6 @@ package vtypes import ( - "fmt" "io" "github.com/Velocidex/ordereddict" @@ -22,6 +21,11 @@ type Sizer interface { Size() int } +// Allows psuedo elements to reveal their own value. +type Valuer interface { + Value() interface{} +} + // Return the start and end of the object type Starter interface { Start() int64 @@ -30,42 +34,3 @@ type Starter interface { type Ender interface { End() int64 } - -// Parse various sizes of ints. -type IntParser struct { - type_name string - size int - converter func(buf []byte) interface{} -} - -// IntParser does not take options -func (self *IntParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { - return self, nil -} - -func (self *IntParser) Size() int { - return self.size -} - -func (self *IntParser) DebugString(scope vfilter.Scope, offset int64, reader io.ReaderAt) string { - return fmt.Sprintf("[%s] %#0x", - self.type_name, self.Parse(scope, reader, offset)) -} - -func (self *IntParser) Parse(scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { - buf := make([]byte, 8) - - n, err := reader.ReadAt(buf, offset) - if n == 0 || err != nil { - return 0 - } - return self.converter(buf) -} - -func NewIntParser(type_name string, size int, converter func(buf []byte) interface{}) *IntParser { - return &IntParser{ - type_name: type_name, - size: size, - converter: converter, - } -} diff --git a/parser_test.go b/parser_test.go index f7ae79a..0d62b14 100644 --- a/parser_test.go +++ b/parser_test.go @@ -1,4 +1,3 @@ -// package vtypes import ( @@ -42,6 +41,9 @@ var ( // Offset 87 - uint64 WinFileTime from 4th bit 0x10, 0x08, 0x10, 0xe6, 0x54, 0x71, 0xa1, 0x1d, + + // offset 95 - E58E26 -> 624485 + 0xe5, 0x8e, 0x26, } ) @@ -58,6 +60,18 @@ func TestIntegerParser(t *testing.T) { assert.Equal(t, uint64(0x0807060504030201), obj) } +func TestLeb128Parser(t *testing.T) { + reader := bytes.NewReader(sample) + profile := NewProfile() + AddModel(profile) + + scope := vfilter.NewScope() + obj, err := profile.Parse(scope, "leb128", reader, 95) + assert.NoError(t, err) + + assert.Equal(t, uint64(624485), obj) +} + func TestStructParser(t *testing.T) { profile := NewProfile() AddModel(profile) diff --git a/pointer.go b/pointer.go index e9efe48..8f7aac2 100644 --- a/pointer.go +++ b/pointer.go @@ -65,7 +65,7 @@ func (self *PointerParser) Parse( buf := make([]byte, 8) n, err := reader.ReadAt(buf, offset) - if n == 0 || err != nil { + if n == 0 || (err != nil && !errors.Is(err, io.EOF)) { return vfilter.Null{} } diff --git a/protocol.go b/protocol.go index 6d5cc8b..0f80903 100644 --- a/protocol.go +++ b/protocol.go @@ -68,6 +68,11 @@ func (self ArrayAssociative) Applicable(a vfilter.Any, b vfilter.Any) bool { if ok { return true } + + _, ok = to_int64(b) + if ok { + return true + } } return false } @@ -79,6 +84,16 @@ func (self ArrayAssociative) Associative(scope vfilter.Scope, return vfilter.Null{}, false } + // Indexing the array + idx, ok := to_int64(b) + if ok { + res, err := lhs.Get(idx) + if err != nil { + return nil, false + } + return res, true + } + rhs, ok := b.(string) if !ok { return vfilter.Null{}, false diff --git a/utils.go b/utils.go index 9eab930..f6cbfef 100644 --- a/utils.go +++ b/utils.go @@ -73,6 +73,14 @@ func SizeOf(obj interface{}) int { return 0 } +func ValueOf(obj interface{}) interface{} { + v, ok := obj.(Valuer) + if ok { + return v.Value() + } + return obj +} + func StartOf(obj interface{}) int64 { start, ok := obj.(Starter) if ok { diff --git a/varint.go b/varint.go new file mode 100644 index 0000000..9210f2d --- /dev/null +++ b/varint.go @@ -0,0 +1,91 @@ +package vtypes + +import ( + "encoding/json" + "errors" + "fmt" + "io" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/vfilter" +) + +type VarInt struct { + base uint64 + size int +} + +func (self VarInt) Size() int { + return self.size +} + +func (self VarInt) Value() interface{} { + return self.base +} + +func (self VarInt) MarshalJSON() ([]byte, error) { + return json.Marshal(self.base) +} + +type SVarInt struct { + VarInt +} + +func (self SVarInt) Value() interface{} { + return int64(self.base) +} + +func (self SVarInt) MarshalJSON() ([]byte, error) { + return json.Marshal(int64(self.base)) +} + +type Leb128Parser struct{} + +func (self *Leb128Parser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { + return &Leb128Parser{}, nil +} + +func (self *Leb128Parser) DebugString(scope vfilter.Scope, offset int64, reader io.ReaderAt) string { + return fmt.Sprintf("[Leb128] %#0x", self.Parse(scope, reader, offset)) +} + +func (self *Leb128Parser) Parse(scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { + // We only support uint64 - max size 64 / 7 = 10 bytes + buf := make([]byte, 10) + + n, err := reader.ReadAt(buf, offset) + if n == 0 || (err != nil && !errors.Is(err, io.EOF)) { + return 0 + } + + var res uint64 + for i := 0; i < len(buf); i++ { + next := buf[i] & 0x80 + value := uint64(buf[i] & 0x7f) + res |= value << (i * 7) + if next == 0 { + return VarInt{ + base: res, + size: i + 1, + } + } + } + + return VarInt{ + base: res, + size: len(buf), + } +} + +type Sleb128Parser struct { + Leb128Parser +} + +func (self *Sleb128Parser) Parse(scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { + res := self.Leb128Parser.Parse(scope, reader, offset) + res_vi, ok := res.(VarInt) + if ok { + return &SVarInt{res_vi} + } + return 0 +} From fc73d8ceb19992a006031963d7558a706e7d9e45 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 31 Jul 2025 11:54:32 +1000 Subject: [PATCH 2/2] Fixed test --- parser_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parser_test.go b/parser_test.go index 0d62b14..5dfbe13 100644 --- a/parser_test.go +++ b/parser_test.go @@ -69,7 +69,10 @@ func TestLeb128Parser(t *testing.T) { obj, err := profile.Parse(scope, "leb128", reader, 95) assert.NoError(t, err) - assert.Equal(t, uint64(624485), obj) + obj_val, ok := obj.(VarInt) + assert.True(t, ok) + + assert.Equal(t, uint64(624485), obj_val.Value()) } func TestStructParser(t *testing.T) {