From b7fd2bdd5a7b4c46b1e5b510d0f47d0f868f3a80 Mon Sep 17 00:00:00 2001 From: Aidan Macdonald Date: Wed, 21 Jan 2026 21:32:00 +0300 Subject: [PATCH 01/14] Adding support for CAMT XML data imports --- ledger/camt/camt.go | 91 +++++++++++++++++++++++ ledger/camt/camt_test.go | 22 ++++++ ledger/camt/sample.xml | 155 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 268 insertions(+) create mode 100644 ledger/camt/camt.go create mode 100644 ledger/camt/camt_test.go create mode 100644 ledger/camt/sample.xml diff --git a/ledger/camt/camt.go b/ledger/camt/camt.go new file mode 100644 index 0000000..fa26f6a --- /dev/null +++ b/ledger/camt/camt.go @@ -0,0 +1,91 @@ +package camt + +import ( + "encoding/xml" + "io" +) + +// XML structures for CAMT.053 format +type Document struct { + XMLName xml.Name `xml:"Document"` + BkToCstmrStmt BkToCstmrStmt `xml:"BkToCstmrStmt"` +} + +type BkToCstmrStmt struct { + Stmt Stmt `xml:"Stmt"` +} + +type Stmt struct { + Acct Acct `xml:"Acct"` + Ntry []Ntry `xml:"Ntry"` +} + +type Acct struct { + Id Id `xml:"Id"` + Ccy string `xml:"Ccy"` + Ownr Ownr `xml:"Ownr"` +} + +type Id struct { + IBAN string `xml:"IBAN"` +} + +type Ownr struct { + Nm string `xml:"Nm"` +} + +type Ntry struct { + Amt Amount `xml:"Amt"` + CdtDbtInd string `xml:"CdtDbtInd"` + BookgDt BookgDt `xml:"BookgDt"` + BkTxCd BkTxCd `xml:"BkTxCd"` + NtryRef string `xml:"NtryRef"` + AddtlNtryInf string `xml:"AddtlNtryInf"` + NtryDtls *NtryDtls `xml:"NtryDtls"` +} + +type Amount struct { + Value string `xml:",chardata"` + Ccy string `xml:"Ccy,attr"` +} + +type BookgDt struct { + DtTm string `xml:"DtTm"` +} + +type BkTxCd struct { + Prtry Prtry `xml:"Prtry"` +} + +type Prtry struct { + Cd string `xml:"Cd"` +} + +type NtryDtls struct { + TxDtls TxDtls `xml:"TxDtls"` +} + +type TxDtls struct { + RltdPties RltdPties `xml:"RltdPties"` +} + +type RltdPties struct { + Cdtr *Cdtr `xml:"Cdtr"` +} + +type Cdtr struct { + Pty Pty `xml:"Pty"` +} + +type Pty struct { + Nm string `xml:"Nm"` +} + +func ParseCamt(reader io.Reader) ([]Ntry, error) { + var doc Document + if err := xml.NewDecoder(reader).Decode(&doc); err != nil { + return nil, err + } + + return doc.BkToCstmrStmt.Stmt.Ntry, nil +} diff --git a/ledger/camt/camt_test.go b/ledger/camt/camt_test.go new file mode 100644 index 0000000..602fd98 --- /dev/null +++ b/ledger/camt/camt_test.go @@ -0,0 +1,22 @@ +package camt_test + +import ( + "bytes" + _ "embed" + "testing" + + "github.com/howeyc/ledger/ledger/camt" +) + +//go:embed sample.xml +var camtSample []byte + +func TestParseCamt(t *testing.T) { + entries, err := camt.ParseCamt(bytes.NewBuffer(camtSample)) + if err != nil { + t.Error(err) + } + if len(entries) != 2 { + t.Error("Expected 2 got ", len(entries)) + } +} diff --git a/ledger/camt/sample.xml b/ledger/camt/sample.xml new file mode 100644 index 0000000..11456d1 --- /dev/null +++ b/ledger/camt/sample.xml @@ -0,0 +1,155 @@ + + + + + 1111111-000000 + 2025-07-31T12:37:01.152446900Z + + + 1111111-000000-99999999 + 2025-07-31T12:37:01.152446900Z + + 2025-07-12T00:00:00+01:00 + 2025-07-14T00:00:00+01:00 + + + + BE00000000000 + + EUR + + Sample + + + ADDR + + EU-0000 + Fake + Happy lane + + + + + 0000000001234 + + COID + + + + + + + + Wise Europe SA + + + ADDR + + 1050 + Brussels + Rue du Trône 100, 3rd floor + + + + + + + + CLBD + + + 67.71 + CRDT +
+ 2025-07-14T00:00:00+01:00 +
+ + + + + OPBD + + + 306.61 + CRDT +
+ 2025-07-12T00:00:00+01:00 +
+ + + + + Unrealised gains and losses + + + 2.26 + CRDT +
+ 2025-07-14T00:00:00+01:00 +
+ + + + 2 + -38.90 + + 38.90 + DBIT + + + + 0 + 0 + + + 2 + -38.90 + + + + 3.90 + DBIT + + BOOK + + + 2025-07-13T05:32:45.916737+01:00 + + + + CARD-675 + + + Card transaction of EUR issued + + + 00001/2025 + 35.00 + DBIT + + BOOK + + + 2025-07-12T08:58:01.327701+01:00 + + + + TRANSFER-0000 + + + + + + + + LLC Company + + + + + + Sent money to LLC Company + + + + From 7a34afacdb0b2caca3f598c1d2dadc53afadc004 Mon Sep 17 00:00:00 2001 From: Aidan Macdonald Date: Fri, 23 Jan 2026 18:13:00 +0300 Subject: [PATCH 02/14] Support for conversion rates as per hledger adding test for conversion per ledger standards. test is correctly failing separating posting parsing and adding test cases posting parsing passes tests passes more testing posting parsing passes all tests all tests passing --- balances_test.go | 22 +++++++ parse.go | 74 ++++++++++++++++++----- parseFuzz_test.go | 10 ++- parse_test.go | 151 ++++++++++++++++++++++++++++++++++++++++++++++ types.go | 5 ++ 5 files changed, 245 insertions(+), 17 deletions(-) diff --git a/balances_test.go b/balances_test.go index 927f891..94eed8a 100644 --- a/balances_test.go +++ b/balances_test.go @@ -71,6 +71,28 @@ var testBalCases = []testBalCase{ }, nil, }, + { + "conversion", + `2026/01/21 Converted CZK to EUR + CZK -2000.00 @ 0.5 + EUR 1000.00 + +2026/01/21 Converted CZK to EUR + CZK -2000.00 @@ 1000.00 + EUR 1000.00 +`, + []Account{ + { + Name: "CZK", + Balance: decimal.NewFromFloat(-4000), + }, + { + Name: "EUR", + Balance: decimal.NewFromFloat(2000), + }, + }, + nil, + }, } func TestBalanceLedger(t *testing.T) { diff --git a/parse.go b/parse.go index 20b1ca6..f64191d 100644 --- a/parse.go +++ b/parse.go @@ -6,10 +6,10 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" "sync" "time" - "unicode" "github.com/alfredxing/calc/compute" "github.com/howeyc/ledger/decimal" @@ -224,6 +224,52 @@ func (lp *parser) parseDate(dateString string) (transDate time.Time, err error) return } +func (a *Account) parsePosting(trimmedLine string) (err error) { + trimmedLine = strings.TrimSpace(trimmedLine) + + // Regex groups: + // 1: account name + // 2: amount (number or parenthesized expression) + // 3: @@ converted amount + // 4: @ conversion rate + re := regexp.MustCompile( + `^(.+?)(?:(?:\s{2,}|\t)([\-]?\d+(?:\.\d+)?|\([0-9+\-*\/. ]+\))(?:\s*(?:@@\s*([\-]?\d+(?:\.\d+)?)|@\s*([\-]?\d+(?:\.\d+)?)))?)?\s*$`, + ) + + m := re.FindStringSubmatch(trimmedLine) + if m == nil { + return fmt.Errorf("invalid posting: %q", trimmedLine) + } + + a.Name = m[1] + if m[2] != "" { + bal, err := compute.Evaluate(m[2]) + if err != nil { + return err + } + a.Balance = decimal.NewFromFloat(bal) + } + + // @@ explicit converted amount + if m[3] != "" { + conv, err := decimal.NewFromString(m[3]) + if err != nil { + return err + } + a.Converted = &conv + } + + // @ rate-based conversion + if m[4] != "" { + rate, err := decimal.NewFromString(m[4]) + if err != nil { + return err + } + a.ConversionFactor = &rate + } + return +} + func (lp *parser) parseTransaction(dateString, payeeString, payeeComment string) (trans *Transaction, err error) { transDate, derr := lp.parseDate(dateString) if derr != nil { @@ -254,26 +300,22 @@ func (lp *parser) parseTransaction(dateString, payeeString, payeeComment string) break } - if iSpace := strings.LastIndexFunc(trimmedLine, unicode.IsSpace); iSpace >= 0 { - if decbal, derr := decimal.NewFromString(trimmedLine[iSpace+1:]); derr == nil { - lp.postings[lp.cpIdx+accIndex].Name = strings.TrimSpace(trimmedLine[:iSpace]) - lp.postings[lp.cpIdx+accIndex].Balance = decbal - } else if iParen := strings.Index(trimmedLine, "("); iParen >= 0 { - lp.postings[lp.cpIdx+accIndex].Name = strings.TrimSpace(trimmedLine[:iParen]) - f, _ := compute.Evaluate(trimmedLine[iParen+1 : len(trimmedLine)-1]) - lp.postings[lp.cpIdx+accIndex].Balance = decimal.NewFromFloat(f) - } else { - lp.postings[lp.cpIdx+accIndex].Name = strings.TrimSpace(trimmedLine) - } - } else { - lp.postings[lp.cpIdx+accIndex].Name = strings.TrimSpace(trimmedLine) - } + _ = lp.postings[lp.cpIdx+accIndex].parsePosting(trimmedLine) if lp.postings[lp.cpIdx+accIndex].Balance.IsZero() { numEmpty++ emptyAccIndex = accIndex } - transBal = transBal.Add(lp.postings[lp.cpIdx+accIndex].Balance) + + if lp.postings[lp.cpIdx+accIndex].Converted != nil { + transBal = transBal.Add(lp.postings[lp.cpIdx+accIndex].Converted.Neg()) + } else if lp.postings[lp.cpIdx+accIndex].ConversionFactor != nil { + transBal = transBal.Add(lp.postings[lp.cpIdx+accIndex].Balance.Mul( + *lp.postings[lp.cpIdx+accIndex].ConversionFactor, + )) + } else { + transBal = transBal.Add(lp.postings[lp.cpIdx+accIndex].Balance) + } accIndex++ } diff --git a/parseFuzz_test.go b/parseFuzz_test.go index 39d91ac..47e86d4 100644 --- a/parseFuzz_test.go +++ b/parseFuzz_test.go @@ -21,7 +21,15 @@ func FuzzParseLedger(f *testing.F) { overall := decimal.Zero for _, t := range trans { for _, p := range t.AccountChanges { - overall = overall.Add(p.Balance) + if p.Converted != nil { + overall = overall.Add(p.Converted.Neg()) + } else if p.ConversionFactor != nil { + overall = overall.Add(p.Balance.Mul( + *p.ConversionFactor, + )) + } else { + overall = overall.Add(p.Balance) + } } } if !overall.IsZero() { diff --git a/parse_test.go b/parse_test.go index e15bb70..cdc5df6 100644 --- a/parse_test.go +++ b/parse_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "reflect" "sync" "testing" "time" @@ -513,6 +514,60 @@ account Assets }, nil, }, + { + "conversion factor", + `1970/01/01 Converted CZK to EUR + Assets:Wise:CZK -2000.00 @ 0.5 + Assets:Wise:EUR 1000.00 +`, + []*Transaction{ + { + Payee: "Converted CZK to EUR", + Date: time.Unix(0, 0).UTC(), + AccountChanges: []Account{ + { + Name: "Assets:Wise:CZK", + Balance: decimal.NewFromFloat(-2000.0), + ConversionFactor: p(decimal.NewFromFloat(0.5)), + }, + { + Name: "Assets:Wise:EUR", + Balance: decimal.NewFromFloat(1000.0), + }, + }, + }, + }, + nil, + }, + { + "conversion", + `1970/01/01 Converted CZK to EUR + Assets:Wise:CZK -2000.00 @@ 1000.00 + Assets:Wise:EUR 1000.00 +`, + []*Transaction{ + { + Payee: "Converted CZK to EUR", + Date: time.Unix(0, 0).UTC(), + AccountChanges: []Account{ + { + Name: "Assets:Wise:CZK", + Balance: decimal.NewFromFloat(-2000.0), + Converted: p(decimal.NewFromFloat(1000)), + }, + { + Name: "Assets:Wise:EUR", + Balance: decimal.NewFromFloat(1000.0), + }, + }, + }, + }, + nil, + }, +} + +func p(d decimal.Decimal) *decimal.Decimal { + return &d } func TestParseLedger(t *testing.T) { @@ -581,3 +636,99 @@ func BenchmarkParseLedger(b *testing.B) { _, _ = ParseLedgerFile("testdata/ledgerBench.dat") } } + +func TestAccount_parsePosting(t *testing.T) { + tests := []struct { + name string + trimmedLine string + want Account + wantErr bool + }{ + { + "simple", + "Expense 123", + Account{Name: "Expense", Balance: decimal.NewFromFloat(123.0)}, + false, + }, + { + "empty", + "Expense", + Account{Name: "Expense", Balance: decimal.NewFromFloat(0.0)}, + false, + }, + { + "spaces", + "Expense:Cranks Unlimited 10", + Account{Name: "Expense:Cranks Unlimited", Balance: decimal.NewFromFloat(10.0)}, + false, + }, + { + "multiply", + "Expense (123*2)", + Account{Name: "Expense", Balance: decimal.NewFromFloat(246.0)}, + false, + }, + { + "slash", + "Expense/test 158", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(158.0)}, + false, + }, + { + "negative", + "Expense/test -158", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(-158.0)}, + false, + }, + { + "math", + "Expense:Bank of:Money (123*2+3)", + Account{Name: "Expense:Bank of:Money", Balance: decimal.NewFromFloat(249.0)}, + false, + }, + { + "math with spaces", + "Expense/test (123 * 3)", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(123 * 3)}, + false, + }, + { + "converted", + "Expense/test 158 @@ 200", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(158.0), Converted: p(decimal.NewFromFloat(200.0))}, + false, + }, + { + "conversion", + "Expense/test 100 @ 2", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(100.0), ConversionFactor: p(decimal.NewFromFloat(2.0))}, + false, + }, + { + "conversion heirarchy", + "Assets:Wise:CZK -2000.00 @ 0.5", + Account{Name: "Assets:Wise:CZK", Balance: decimal.NewFromFloat(-2000.0), ConversionFactor: p(decimal.NewFromFloat(0.5))}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := Account{} + gotErr := a.parsePosting(tt.trimmedLine) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("parsePosting() failed: %v", gotErr) + } + return + } + if !reflect.DeepEqual(a, tt.want) { + aJson, _ := json.Marshal(a) + wantJson, _ := json.Marshal(tt.want) + t.Errorf("got %+v wanted %+v", string(aJson), string(wantJson)) + } + if tt.wantErr { + t.Fatal("parsePosting() succeeded unexpectedly") + } + }) + } +} diff --git a/types.go b/types.go index ccfc0ef..35b4848 100644 --- a/types.go +++ b/types.go @@ -11,6 +11,11 @@ type Account struct { Name string Balance decimal.Decimal Comment string + + // Balance converted using @@ notation + Converted *decimal.Decimal + // Conversion factor using @ notation + ConversionFactor *decimal.Decimal } // Transaction is the basis of a ledger. The ledger holds a list of transactions. From 474e961fe48eaff087ddf9a8ef12530f36482663 Mon Sep 17 00:00:00 2001 From: Aidan Macdonald Date: Sun, 25 Jan 2026 11:28:49 +0200 Subject: [PATCH 03/14] currency parsing passes tests --- parse.go | 22 +++++++++++++++------- parse_test.go | 32 +++++++++++++++++++++++++++++++- types.go | 8 +++++--- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/parse.go b/parse.go index f64191d..8257c8f 100644 --- a/parse.go +++ b/parse.go @@ -233,7 +233,13 @@ func (a *Account) parsePosting(trimmedLine string) (err error) { // 3: @@ converted amount // 4: @ conversion rate re := regexp.MustCompile( - `^(.+?)(?:(?:\s{2,}|\t)([\-]?\d+(?:\.\d+)?|\([0-9+\-*\/. ]+\))(?:\s*(?:@@\s*([\-]?\d+(?:\.\d+)?)|@\s*([\-]?\d+(?:\.\d+)?)))?)?\s*$`, + `^(?P.+?)` + + `(?:(?:\s{2,}|\t)` + + `(?:(?P[A-Z\$]+)\s+)?` + + `(?P[\-]?\d+(?:\.\d+)?|\([0-9+\-*\/. ]+\))` + + `(?:\s*(?:@@\s*` + + `(?P[\-]?\d+(?:\.\d+)?)|@\s*` + + `(?P[\-]?\d+(?:\.\d+)?)))?)?\s*$`, ) m := re.FindStringSubmatch(trimmedLine) @@ -242,8 +248,10 @@ func (a *Account) parsePosting(trimmedLine string) (err error) { } a.Name = m[1] - if m[2] != "" { - bal, err := compute.Evaluate(m[2]) + a.Currency = m[2] + + if m[3] != "" { + bal, err := compute.Evaluate(m[3]) if err != nil { return err } @@ -251,8 +259,8 @@ func (a *Account) parsePosting(trimmedLine string) (err error) { } // @@ explicit converted amount - if m[3] != "" { - conv, err := decimal.NewFromString(m[3]) + if m[4] != "" { + conv, err := decimal.NewFromString(m[4]) if err != nil { return err } @@ -260,8 +268,8 @@ func (a *Account) parsePosting(trimmedLine string) (err error) { } // @ rate-based conversion - if m[4] != "" { - rate, err := decimal.NewFromString(m[4]) + if m[5] != "" { + rate, err := decimal.NewFromString(m[5]) if err != nil { return err } diff --git a/parse_test.go b/parse_test.go index cdc5df6..a66d53d 100644 --- a/parse_test.go +++ b/parse_test.go @@ -701,7 +701,7 @@ func TestAccount_parsePosting(t *testing.T) { { "conversion", "Expense/test 100 @ 2", - Account{Name: "Expense/test", Balance: decimal.NewFromFloat(100.0), ConversionFactor: p(decimal.NewFromFloat(2.0))}, + Account{Name: "Expense/test", Currency: "", Balance: decimal.NewFromFloat(100.0), ConversionFactor: p(decimal.NewFromFloat(2.0))}, false, }, { @@ -710,6 +710,36 @@ func TestAccount_parsePosting(t *testing.T) { Account{Name: "Assets:Wise:CZK", Balance: decimal.NewFromFloat(-2000.0), ConversionFactor: p(decimal.NewFromFloat(0.5))}, false, }, + { + "negative", + "Expense/test EUR -158", + Account{Name: "Expense/test", Currency: "EUR", Balance: decimal.NewFromFloat(-158.0)}, + false, + }, + { + "math", + "Expense:Bank of:Money USD (123*2+3)", + Account{Name: "Expense:Bank of:Money", Currency: "USD", Balance: decimal.NewFromFloat(249.0)}, + false, + }, + { + "math with spaces", + "Expense/test CZK (123 * 3)", + Account{Name: "Expense/test", Currency: "CZK", Balance: decimal.NewFromFloat(123 * 3)}, + false, + }, + { + "converted", + "Expense/test USD 158 @@ 200", + Account{Name: "Expense/test", Currency: "USD", Balance: decimal.NewFromFloat(158.0), Converted: p(decimal.NewFromFloat(200.0))}, + false, + }, + { + "conversion", + "Expense/test $ 100 @ 2", + Account{Name: "Expense/test", Currency: "$", Balance: decimal.NewFromFloat(100.0), ConversionFactor: p(decimal.NewFromFloat(2.0))}, + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/types.go b/types.go index 35b4848..0e2e78e 100644 --- a/types.go +++ b/types.go @@ -8,9 +8,11 @@ import ( // Account holds the name and balance type Account struct { - Name string - Balance decimal.Decimal - Comment string + Name string + // Default "" for no currency/token displayed + Currency string + Balance decimal.Decimal + Comment string // Balance converted using @@ notation Converted *decimal.Decimal From d9ae372744269e0680758a08ad4d45331322ff92 Mon Sep 17 00:00:00 2001 From: Aidan Macdonald Date: Sun, 25 Jan 2026 12:30:15 +0200 Subject: [PATCH 04/14] separating currencies in balance --- balances.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/balances.go b/balances.go index 6f539a3..32e1237 100644 --- a/balances.go +++ b/balances.go @@ -14,13 +14,13 @@ import ( // Accounts are sorted by name. func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { var accList []*Account - balances := make(map[string]*Account) + balances := make(map[string]map[string]*Account) // at every depth, for each account, track the parent account depthMap := make(map[int]map[string]string) var maxDepth int - incAccount := func(accName string, val decimal.Decimal) { + incAccount := func(accName string, currency string, val decimal.Decimal) { // track parent accDepth := strings.Count(accName, ":") + 1 pmap, pmapfound := depthMap[accDepth] @@ -35,10 +35,14 @@ func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { } // add to balance - if acc, ok := balances[accName]; !ok { + if _, ok := balances[accName]; !ok { + balances[accName] = make(map[string]*Account) + } + + if acc, ok := balances[accName][currency]; !ok { acc := &Account{Name: accName, Balance: val} accList = append(accList, acc) - balances[accName] = acc + balances[accName][currency] = acc } else { acc.Balance = acc.Balance.Add(val) } @@ -53,7 +57,7 @@ func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { } } if inFilter { - incAccount(accChange.Name, accChange.Balance) + incAccount(accChange.Name, accChange.Currency, accChange.Balance) } } } @@ -61,7 +65,9 @@ func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { // roll-up balances for curDepth := maxDepth; curDepth > 1; curDepth-- { for accName, parentName := range depthMap[curDepth] { - incAccount(parentName, balances[accName].Balance) + for currency, acc := range balances[accName] { + incAccount(parentName, currency, acc.Balance) + } } } From b8779350aa78c350604cb6f81ed9974378afe163 Mon Sep 17 00:00:00 2001 From: Aidan Macdonald Date: Sun, 25 Jan 2026 12:34:15 +0200 Subject: [PATCH 05/14] supports currency in balance --- balances.go | 2 +- balances_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/balances.go b/balances.go index 32e1237..8043ead 100644 --- a/balances.go +++ b/balances.go @@ -40,7 +40,7 @@ func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { } if acc, ok := balances[accName][currency]; !ok { - acc := &Account{Name: accName, Balance: val} + acc := &Account{Name: accName, Currency: currency, Balance: val} accList = append(accList, acc) balances[accName][currency] = acc } else { diff --git a/balances_test.go b/balances_test.go index 94eed8a..6b2d126 100644 --- a/balances_test.go +++ b/balances_test.go @@ -93,6 +93,30 @@ var testBalCases = []testBalCase{ }, nil, }, + { + "conversion", + `2026/01/21 Converted CZK to EUR + CZK CZK -2000.00 @ 0.5 + EUR EUR 1000.00 + +2026/01/21 Converted CZK to EUR + CZK CZK -2000.00 @@ 1000.00 + EUR EUR 1000.00 +`, + []Account{ + { + Name: "CZK", + Currency: "CZK", + Balance: decimal.NewFromFloat(-4000), + }, + { + Name: "EUR", + Currency: "EUR", + Balance: decimal.NewFromFloat(2000), + }, + }, + nil, + }, } func TestBalanceLedger(t *testing.T) { From 3f2245190151bf67cec852d417bfddfaaa068cf3 Mon Sep 17 00:00:00 2001 From: Aidan Macdonald Date: Thu, 5 Feb 2026 14:58:32 +0200 Subject: [PATCH 06/14] printing formatting --- ledger/cmd/print.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/cmd/print.go b/ledger/cmd/print.go index 70f4b43..0dff2d8 100644 --- a/ledger/cmd/print.go +++ b/ledger/cmd/print.go @@ -133,7 +133,7 @@ func PrintBalances(accountList []*ledger.Account, printZeroBalances bool, depth, overallBalance = overallBalance.Add(account.Balance) } if (printZeroBalances || account.Balance.Sign() != 0) && (depth < 0 || accDepth <= depth) { - outBalanceString := account.Balance.StringFixedBank() + outBalanceString := account.Currency + " " + account.Balance.StringFixedBank() amtColor := colorReset if account.Balance.Sign() < 0 { amtColor = colorNeg From d4b24256ac4c78ead16dceb5c6ac5393c8f1a919 Mon Sep 17 00:00:00 2001 From: Aidan Macdonald Date: Thu, 12 Feb 2026 18:26:09 +0200 Subject: [PATCH 07/14] bump go --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 81e1912..26b485f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/howeyc/ledger -go 1.22 +go 1.24 require ( github.com/alfredxing/calc v0.0.0-20180827002445-77daf576f976 From f9cffc05cbbcfa0d12092fc25e1e9bc946459886 Mon Sep 17 00:00:00 2001 From: Aidan Macdonald Date: Thu, 5 Feb 2026 21:10:27 +0200 Subject: [PATCH 08/14] removed local implementation of decimal. benchmarks are slower now Previous $ go test -bench=. goos: linux goarch: amd64 pkg: github.com/howeyc/ledger cpu: Intel(R) Core(TM) i7-6820HQ CPU @ 2.70GHz BenchmarkGetBalances-8 45 26721471 ns/op BenchmarkParseLedger-8 273 4209337 ns/op PASS ok github.com/howeyc/ledger 3.273s New $ go test -bench=. goos: linux goarch: amd64 pkg: github.com/howeyc/ledger cpu: Intel(R) Core(TM) i7-6820HQ CPU @ 2.70GHz BenchmarkGetBalances-8 12 99485602 ns/op BenchmarkParseLedger-8 248 4376757 ns/op PASS ok github.com/howeyc/ledger 4.825s --- .github/workflows/go.yml | 2 +- balances.go | 2 +- balances_test.go | 2 +- decimal/decimal.go | 314 ------------- decimal/decimal_test.go | 437 ------------------ include_test.go | 4 +- ledger/cmd/import.go | 2 +- ledger/cmd/print.go | 14 +- ledger/cmd/printEquity.go | 2 +- .../templates/template.leaderboardchart.html | 2 +- ledger/cmd/webHandlerReport.go | 4 +- parse.go | 2 +- parseFuzz_test.go | 2 +- parse_test.go | 9 +- types.go | 2 +- 15 files changed, 24 insertions(+), 776 deletions(-) delete mode 100644 decimal/decimal.go delete mode 100644 decimal/decimal_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0f8a770..8d6687d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -23,7 +23,7 @@ jobs: run: go build -v ./... - name: Test - run: go test -v -coverprofile=profile.cov . ./decimal + run: go test -v -coverprofile=profile.cov . - uses: shogo82148/actions-goveralls@v1 with: diff --git a/balances.go b/balances.go index 8043ead..6a06e0a 100644 --- a/balances.go +++ b/balances.go @@ -4,7 +4,7 @@ import ( "slices" "strings" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) // GetBalances provided a list of transactions and filter strings, returns account balances of diff --git a/balances_test.go b/balances_test.go index 6b2d126..af70a46 100644 --- a/balances_test.go +++ b/balances_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) type testBalCase struct { diff --git a/decimal/decimal.go b/decimal/decimal.go deleted file mode 100644 index 16f7fcb..0000000 --- a/decimal/decimal.go +++ /dev/null @@ -1,314 +0,0 @@ -// Package decimal implements fixed-point decimal with accuracy to 3 digits of -// precision after the decimal point. -// -// int64 is the underlying data type for speed of computation. However, using -// an int64 cast to Decimal will not work, one of the "New" functions must -// be used to get accurate results. -// -// The package multiplies every source value by 1000, and then does integer -// math from that point forward, maintaining all values at that scale over -// every operation. -// -// Note: For use in ledger. Cannot handle values over approx 900 trillion. -package decimal - -import ( - "errors" - "strconv" - "strings" -) - -// Decimal represents a fixed-point decimal. -type Decimal int64 - -// scaleFactor used for math operations, -const scaleFactor = 1000 - -// precision of 3 digits -const precision = 3 - -// Zero constant, to make initializations easier. -const Zero = Decimal(0) - -// One constant, to make initializations easier. -const One = Decimal(scaleFactor) - -// Parse max/min for whole number part -const parseMax = (1<<63 - 1) / scaleFactor -const parseMin = (-1 << 63) / scaleFactor - -// NewFromFloat converts a float64 to Decimal. Only 3 digits of precision after -// the decimal point are preserved. -func NewFromFloat(f float64) Decimal { - return Decimal(f * float64(scaleFactor)) -} - -// NewFromInt converts a int64 to Decimal. Multiplies by 1000 to get into -// Decimal scale. -func NewFromInt(i int64) Decimal { - return Decimal(i) * scaleFactor -} - -var errEmpty = errors.New("empty string") -var errTooBig = errors.New("number too big") -var errInvalid = errors.New("invalid syntax") - -// atoi64 is equivalent to strconv.Atoi -func atoi64(s string) (bool, int64, error) { - sLen := len(s) - if sLen < 1 { - return false, 0, errEmpty - } - if sLen > 18 { - return false, 0, errTooBig - } - - neg := false - if s[0] == '-' { - neg = true - s = s[1:] - if len(s) < 1 { - return neg, 0, errEmpty - } - } - - var n int64 - for _, ch := range []byte(s) { - ch -= '0' - if ch > 9 { - return neg, 0, errInvalid - } - n = n*10 + int64(ch) - } - if neg { - n = -n - } - return neg, n, nil -} - -// NewFromString returns a Decimal from a string representation. Throws an -// error if integer parsing fails. -func NewFromString(s string) (Decimal, error) { - if whole, frac, split := strings.Cut(s, "."); split { - neg, w, err := atoi64(whole) - // if fractional portion exists, whole part can be empty - if err != nil && err != errEmpty { - return Zero, err - } - - // overflow - if w > parseMax || w < parseMin { - return Zero, errTooBig - } - w *= int64(scaleFactor) - - // Parse up to *precision* digits and scale up - var f int64 - var seen int - for _, b := range frac { - f *= 10 - if b < '0' || b > '9' { - return Zero, errInvalid - } - f += int64(b - '0') - seen++ - if seen == precision { - break - } - } - for seen < precision { - f *= 10 - seen++ - } - - if neg { - f = -f - } - return Decimal(w + f), nil - } - - _, i, err := atoi64(s) - if i > parseMax || i < parseMin { - return Zero, errTooBig - } - i *= int64(scaleFactor) - return Decimal(i), err -} - -// IsZero returns true if d == 0 -func (d Decimal) IsZero() bool { - return d == Zero -} - -// Neg returns -d -func (d Decimal) Neg() Decimal { - return -d -} - -// Sign returns: -// -// -1 if d < 0 -// -// 0 if d == 0 -// -// +1 if d > 0 -func (d Decimal) Sign() int { - if d < 0 { - return -1 - } else if d > 0 { - return 1 - } - return 0 -} - -// Add returns d + d1 -func (d Decimal) Add(d1 Decimal) Decimal { - return d + d1 -} - -// Sub returns d - d1 -func (d Decimal) Sub(d1 Decimal) Decimal { - return d - d1 -} - -// Mul returns d * d1 -func (d Decimal) Mul(d1 Decimal) Decimal { - return (d * d1) / scaleFactor -} - -// Div returns d / d1 -func (d Decimal) Div(d1 Decimal) Decimal { - return (d * scaleFactor) / d1 -} - -// Abs returns the absolute value of the decimal -func (d Decimal) Abs() Decimal { - if d < 0 { - return d.Neg() - } - return d -} - -// Float64 returns the float64 value for d, and exact is always set to false. -// The signature is this way to match big.Rat -func (d Decimal) Float64() (f float64, exact bool) { - return float64(d) / float64(scaleFactor), false -} - -// Cmp compares the numbers represented by d and d1 and returns: -// -// -1 if d < d1 -// 0 if d == d1 -// +1 if d > d1 -func (d Decimal) Cmp(d1 Decimal) int { - if d < d1 { - return -1 - } else if d > d1 { - return 1 - } - return 0 -} - -// fmtFrac formats the fraction of v/10**prec (e.g., ".12345") into the -// tail of buf. It returns the index where the -// output bytes begin and the value v/10**prec. -func fmtFrac(buf []byte, v uint64, prec int) (nw int, nv uint64) { - w := len(buf) - for range prec { - digit := v % 10 - w-- - buf[w] = byte(digit) + '0' - v /= 10 - } - w-- - buf[w] = '.' - return w, v -} - -// fmtInt formats v into the tail of buf. -// It returns the index where the output begins. -func fmtInt(buf []byte, v uint64) int { - w := len(buf) - if v == 0 { - w-- - buf[w] = '0' - } else { - for v > 0 { - w-- - buf[w] = byte(v%10) + '0' - v /= 10 - } - } - return w -} - -// StringFixedBank returns a banker rounded fixed-point string with 2 digits -// after the decimal point. -// -// Example: -// -// NewFromFloat(5.455).StringFixedBank() == "5.46" -// NewFromFloat(5.445).StringFixedBank() == "5.44" -func (d Decimal) StringFixedBank() string { - var buf [24]byte - w := len(buf) - - u := uint64(d) - neg := d < 0 - if neg { - u = -u - } - - // Bank rounding - rem := u % 10 - u /= 10 - if rem > 5 || (rem == 5 && u%2 != 0) { - u++ - } - - // fmt functions from time.Duration - w, u = fmtFrac(buf[:w], u, precision-1) - w = fmtInt(buf[:w], u) - - if neg { - w-- - buf[w] = '-' - } - - return string(buf[w:]) -} - -// StringTruncate returns the whole-number (Int) part of d. -// -// Example: -// -// NewFromFloat(5.44).StringTruncate() == "5" -func (d Decimal) StringTruncate() string { - whole := d / scaleFactor - return strconv.FormatInt(int64(whole), 10) -} - -// StringRound returns the nearest rounded whole-number (Int) part of d. -// Example: -// -// NewFromFloat(5.5).StringRound() == "6" -// NewFromFloat(5.4).StringRound() == "5" -// NewFromFloat(-5.4).StringRound() == "5" -// NewFromFloat(-5.5).StringRound() == "6" -func (d Decimal) StringRound() string { - whole := d / scaleFactor - frac := (d % scaleFactor) - neg := false - if frac < 0 { - frac = -frac - neg = true - } - if frac >= (5 * (scaleFactor / 10)) { - if neg { - whole-- - } else { - whole++ - } - } - return strconv.FormatInt(int64(whole), 10) -} diff --git a/decimal/decimal_test.go b/decimal/decimal_test.go deleted file mode 100644 index 327125c..0000000 --- a/decimal/decimal_test.go +++ /dev/null @@ -1,437 +0,0 @@ -package decimal - -import ( - "math/rand" - "strings" - "testing" - - sdec "github.com/shopspring/decimal" -) - -type testCase struct { - name string - Result, Input string -} - -var testCases = []testCase{ - { - "multiply", - NewFromFloat(48.0).StringFixedBank(), - NewFromInt(6).Mul(NewFromInt(8)).StringFixedBank(), - }, - { - "divide", - NewFromFloat(6.0).StringFixedBank(), - NewFromInt(48).Div(NewFromInt(8)).StringFixedBank(), - }, - { - "divide-1", - NewFromFloat(11.111).StringFixedBank(), - NewFromInt(100).Div(NewFromInt(9)).StringFixedBank(), - }, - { - "sum", - NewFromFloat(234.56).StringFixedBank(), - NewFromFloat(123.12).Add(NewFromInt(111)).Add(NewFromFloat(0.44)).StringFixedBank(), - }, - { - "bankrounduppos", - NewFromFloat(234.56).StringFixedBank(), - NewFromFloat(234.555).StringFixedBank(), - }, - { - "bankrounddownpos", - NewFromFloat(234.54).StringFixedBank(), - NewFromFloat(234.545).StringFixedBank(), - }, - { - "bankroundupneg", - "-234.56", - NewFromFloat(-234.555).StringFixedBank(), - }, - { - "bankrounddownneg", - "-234.54", - NewFromFloat(-234.545).StringFixedBank(), - }, - { - "rounduppos", - NewFromFloat(234.56).StringFixedBank(), - NewFromFloat(234.556).StringFixedBank(), - }, - { - "rounddownpos", - NewFromFloat(234.55).StringFixedBank(), - NewFromFloat(234.554).StringFixedBank(), - }, - { - "roundupneg", - "-234.56", - NewFromFloat(-234.556).StringFixedBank(), - }, - { - "rounddownneg", - "-234.55", - NewFromFloat(-234.554).StringFixedBank(), - }, - { - "truncate", - NewFromInt(234).StringTruncate(), - NewFromFloat(234.554).StringTruncate(), - }, - { - "2digits-1", - "1.00", - One.StringFixedBank(), - }, - { - "2digits-4.5", - "4.50", - NewFromFloat(4.5).StringFixedBank(), - }, - { - "roundintuppos", - "6", - NewFromFloat(5.6).StringRound(), - }, - { - "roundintdownpos", - "5", - NewFromFloat(5.4).StringRound(), - }, - { - "roundintupneg", - "-5", - NewFromFloat(-5.4).StringRound(), - }, - { - "roundintdownneg", - "-6", - NewFromFloat(-5.6).StringRound(), - }, - { - "negfrac", - "-0.43", - NewFromFloat(-0.43).StringFixedBank(), - }, - { - "sub", - "5.12", - NewFromFloat(5.56).Sub(NewFromFloat(0.44)).StringFixedBank(), - }, - { - "neg", - "-5.12", - NewFromFloat(5.12).Neg().StringFixedBank(), - }, - { - "abs-1", - "5.12", - NewFromFloat(-5.12).Abs().StringFixedBank(), - }, - { - "abs-1", - "5.12", - NewFromFloat(5.12).Abs().StringFixedBank(), - }, -} - -func TestDecimal(t *testing.T) { - for _, tc := range testCases { - if tc.Result != tc.Input { - t.Errorf("Error(%s): expected \n`%s`, \ngot \n`%s`", tc.name, tc.Result, tc.Input) - } - } -} - -func TestFloat(t *testing.T) { - d := NewFromFloat(5.56) - f := float64(5.56) - if df, _ := d.Float64(); df != f { - t.Error("Float64 not exact") - } -} - -func TestCompare(t *testing.T) { - l := NewFromInt(5) - h := NewFromInt(10) - z := NewFromInt(0) - - if !z.IsZero() { - t.Error("zero failed") - } - - if h.Cmp(l) != 1 || l.Cmp(h) != -1 || z.Cmp(Zero) != 0 { - t.Error("compare fail") - } -} - -func TestSign(t *testing.T) { - n := NewFromInt(-5) - p := NewFromInt(5) - z := NewFromInt(0) - - if z.Sign() != 0 { - t.Error("zero failed") - } - - if n.Sign() != -1 || p.Sign() != 1 { - t.Error("sign fail") - } -} - -var testParseCases = []testCase{ - { - "negzero", - "-0.43", - "-0.43", - }, - { - "poszero", - "0.43", - "0.43", - }, - { - "3digit", - "5.56", - "5.564", - }, - { - "truncateinput", - "5.56", - "5.56432342", - }, - { - "precise", - "16.24", - "16.24", - }, - { - "fuzz-1", - "0.00", - "0.0051", - }, - { - "fuzz-2", - "8.00", - "8.005", - }, - { - "fuzz-3", - "0.00", - "0.005", - }, - { - "fuzz-4", - "1.00", - "0.997", - }, - { - "fuzz-5", - "2200000000000021.00", - "2200000000000021", - }, - { - "fuzz-6", - "0.01", - "0.010e1", - }, - { - "fuzz-7", - "-8.00", - "-7.995", - }, - { - "fuzz-8", - "-9.00", - "-8.995", - }, - { - "fuzz-9", - "8.00", - "7.995", - }, - { - "fuzz-10", - "9.00", - "8.995", - }, - { - "fuzz-11", - "-7.98", - "-7.985", - }, - { - "fuzz-12", - "-8.98", - "-8.985", - }, - { - "fuzz-13", - "7.98", - "7.985", - }, - { - "fuzz-14", - "8.98", - "8.984", - }, - { - "fuzz-15", - "-8.00", - "-7.999", - }, - { - "fuzz-16", - "-9.00", - "-8.999", - }, - { - "fuzz-17", - "8.00", - "7.999", - }, - { - "fuzz-18", - "9.00", - "8.999", - }, - { - "error-1", - errTooBig.Error(), - "100000000000000000", - }, - { - "error-2", - errTooBig.Error(), - "10000000000000000", - }, - { - "error-3", - errTooBig.Error(), - "10000000000000000.56", - }, - { - "error-4", - errInvalid.Error(), - "0.e0", - }, - { - "error-5", - errTooBig.Error(), - "5555555555555555555555555550000000000000000", - }, - { - "error-6", - errEmpty.Error(), - "-", - }, - { - "error-7", - errEmpty.Error(), - "", - }, - { - "error-badint-1", - errInvalid.Error(), - "1QZ.56", - }, - { - "error-expr-1", - errInvalid.Error(), - "(123 * 6)", - }, - { - "missingwhole", - "0.50", - ".50", - }, - { - "negmissingwhole", - "-0.50", - "-.50", - }, - { - "missingfrac", - "5.00", - "5.", - }, - { - "neg-missingfrac", - "-5.00", - "-5.", - }, - { - "just-a-decimal", - "0.00", - ".", - }, -} - -func TestStringParse(t *testing.T) { - for _, tc := range testParseCases { - d, err := NewFromString(tc.Input) - if strings.HasPrefix(tc.name, "error") { - if err == nil { - t.Fatalf("Error(%s): expected error `%s`", tc.name, tc.Result) - } - if err.Error() != tc.Result { - t.Fatalf("Error(%s): expected `%s`, got `%s`", tc.name, tc.Result, err) - } - } - if !strings.HasPrefix(tc.name, "error") && err != nil { - t.Fatalf("Error(%s): unexpected error `%s`", tc.name, err) - } - if !strings.HasPrefix(tc.name, "error") && tc.Result != d.StringFixedBank() { - t.Errorf("Error(%s): expected \n`%s`, \ngot \n`%s`", tc.name, tc.Result, d.StringFixedBank()) - } - } -} - -func FuzzStringParse(f *testing.F) { - f.Fuzz(func(t *testing.T, s string) { - if _, after, split := strings.Cut(s, "."); split { - if len(after) > 3 { - return - } - } - sd, serr := sdec.NewFromString(s) - if serr != nil { - return - } - d, err := NewFromString(s) - if err != nil { - return - } - ss := strings.TrimPrefix(sd.StringFixedBank(2), "-") - ds := strings.TrimPrefix(d.StringFixedBank(), "-") - - if ds != ss { - t.Fatalf("no match: decimal \n`%s`, \nsdec \n `%s`", ds, ss) - } - }) -} - -func BenchmarkNewFromString(b *testing.B) { - numbers := []string{"10.0", "245.6", "354", "2.456", "-31.2"} - for b.Loop() { - for _, numStr := range numbers { - NewFromString(numStr) - } - } -} - -func BenchmarkStringFixedBank(b *testing.B) { - var numbers [1000]Decimal - for i := range len(numbers) { - numbers[i] = NewFromFloat(rand.Float64() * 100000) - if i%2 == 0 { - numbers[i] *= -1 - } - } - for b.Loop() { - for _, num := range numbers { - num.StringFixedBank() - } - } -} diff --git a/include_test.go b/include_test.go index 344ec8c..aef6a53 100644 --- a/include_test.go +++ b/include_test.go @@ -11,7 +11,7 @@ func TestIncludeSimple(t *testing.T) { t.Fatal(err) } bals := GetBalances(trans, []string{"Assets"}) - if bals[0].Balance.StringRound() != "50" { + if bals[0].Balance.StringFixed(0) != "50" { t.Fatal(errors.New("should be 50")) } } @@ -22,7 +22,7 @@ func TestIncludeGlob(t *testing.T) { t.Fatal(err) } bals := GetBalances(trans, []string{"Assets"}) - if bals[0].Balance.StringRound() != "80" { + if bals[0].Balance.StringFixed(0) != "80" { t.Fatal(errors.New("should be 80")) } } diff --git a/ledger/cmd/import.go b/ledger/cmd/import.go index da47902..1592c34 100644 --- a/ledger/cmd/import.go +++ b/ledger/cmd/import.go @@ -11,10 +11,10 @@ import ( "unicode/utf8" "github.com/howeyc/ledger" - "github.com/howeyc/ledger/decimal" "github.com/howeyc/ledger/ledger/cmd/internal/import/camt" "github.com/howeyc/ledger/ledger/cmd/internal/import/qfx" "github.com/jbrukh/bayesian" + "github.com/shopspring/decimal" "github.com/spf13/cobra" ) diff --git a/ledger/cmd/print.go b/ledger/cmd/print.go index 0dff2d8..49ff88a 100644 --- a/ledger/cmd/print.go +++ b/ledger/cmd/print.go @@ -14,9 +14,9 @@ import ( "unicode/utf8" "github.com/howeyc/ledger" - "github.com/howeyc/ledger/decimal" "github.com/howeyc/ledger/ledger/cmd/internal/fastcolor" date "github.com/joyt/godate" + "github.com/shopspring/decimal" "github.com/spf13/cobra" "golang.org/x/term" ) @@ -133,7 +133,7 @@ func PrintBalances(accountList []*ledger.Account, printZeroBalances bool, depth, overallBalance = overallBalance.Add(account.Balance) } if (printZeroBalances || account.Balance.Sign() != 0) && (depth < 0 || accDepth <= depth) { - outBalanceString := account.Currency + " " + account.Balance.StringFixedBank() + outBalanceString := account.Currency + " " + account.Balance.StringFixedBank(2) amtColor := colorReset if account.Balance.Sign() < 0 { amtColor = colorNeg @@ -145,7 +145,7 @@ func PrintBalances(accountList []*ledger.Account, printZeroBalances bool, depth, } } fmt.Fprintln(buf, strings.Repeat("-", columns)) - outBalanceString := overallBalance.StringFixedBank() + outBalanceString := overallBalance.StringFixedBank(2) amtColor := colorReset if overallBalance.Sign() < 0 { amtColor = colorNeg @@ -186,7 +186,7 @@ func WriteTransaction(w io.StringWriter, trans *ledger.Transaction, columns int) } w.WriteString(newLine) for _, accChange := range trans.AccountChanges { - outBalanceString := accChange.Balance.StringFixedBank() + outBalanceString := accChange.Balance.StringFixedBank(2) spaceCount := columns - 4 - utf8.RuneCountInString(accChange.Name) - utf8.RuneCountInString(outBalanceString) if spaceCount < 1 { spaceCount = 1 @@ -253,8 +253,8 @@ func PrintRegister(generalLedger []*ledger.Transaction, filterArr []string, colu } if inFilter { runningBalance = runningBalance.Add(accChange.Balance) - outBalanceString := accChange.Balance.StringFixedBank() - outRunningBalanceString := runningBalance.StringFixedBank() + outBalanceString := accChange.Balance.StringFixedBank(2) + outRunningBalanceString := runningBalance.StringFixedBank(2) balamtColor := colorReset if accChange.Balance.Sign() < 0 { @@ -297,7 +297,7 @@ func PrintCSV(generalLedger []*ledger.Transaction, filterArr []string) { } if inFilter { runningBalance = runningBalance.Add(accChange.Balance) - outBalanceString := accChange.Balance.StringFixedBank() + outBalanceString := accChange.Balance.StringFixedBank(2) record := []string{trans.Date.Format(transactionDateFormat), trans.Payee, accChange.Name, diff --git a/ledger/cmd/printEquity.go b/ledger/cmd/printEquity.go index 753986c..1d527be 100644 --- a/ledger/cmd/printEquity.go +++ b/ledger/cmd/printEquity.go @@ -8,7 +8,7 @@ import ( "time" "github.com/howeyc/ledger" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" "github.com/spf13/cobra" ) diff --git a/ledger/cmd/templates/template.leaderboardchart.html b/ledger/cmd/templates/template.leaderboardchart.html index 51b848e..287f3c5 100644 --- a/ledger/cmd/templates/template.leaderboardchart.html +++ b/ledger/cmd/templates/template.leaderboardchart.html @@ -42,7 +42,7 @@

{{.ReportName}} : {{.RangeStart.Format "2006-01-02"}} - {{.RangeEnd.Format "
- {{$acc.Balance.StringRound}} ({{$acc.Percentage}}%) + {{$acc.Balance.StringFixed(0)}} ({{$acc.Percentage}}%)
diff --git a/ledger/cmd/webHandlerReport.go b/ledger/cmd/webHandlerReport.go index 63e64ee..23681d1 100644 --- a/ledger/cmd/webHandlerReport.go +++ b/ledger/cmd/webHandlerReport.go @@ -9,9 +9,9 @@ import ( "time" "github.com/howeyc/ledger" - "github.com/howeyc/ledger/decimal" "github.com/howeyc/ledger/ledger/cmd/internal/pdr" colorful "github.com/lucasb-eyer/go-colorful" + "github.com/shopspring/decimal" ) func getRangeAndPeriod(dateRange, dateFreq string) (start, end time.Time, period ledger.Period, err error) { @@ -82,7 +82,7 @@ func calcBalances(calcAccts []calculatedAccount, balances []*ledger.Account) (re factor := decimal.NewFromFloat(acctOp.MultiplicationFactor) fval = fval.Mul(factor) } - oval := decimal.One + oval := decimal.NewFromInt(1) if acctOp.SubAccount != "" { for _, obal := range balances { if acctOp.SubAccount == obal.Name { diff --git a/parse.go b/parse.go index 8257c8f..e3164d7 100644 --- a/parse.go +++ b/parse.go @@ -12,8 +12,8 @@ import ( "time" "github.com/alfredxing/calc/compute" - "github.com/howeyc/ledger/decimal" date "github.com/joyt/godate" + "github.com/shopspring/decimal" ) // ParseLedgerFile parses a ledger file and returns a list of Transactions. diff --git a/parseFuzz_test.go b/parseFuzz_test.go index 47e86d4..40116c3 100644 --- a/parseFuzz_test.go +++ b/parseFuzz_test.go @@ -6,7 +6,7 @@ import ( "bytes" "testing" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) func FuzzParseLedger(f *testing.F) { diff --git a/parse_test.go b/parse_test.go index a66d53d..8815743 100644 --- a/parse_test.go +++ b/parse_test.go @@ -4,12 +4,11 @@ import ( "bytes" "encoding/json" "errors" - "reflect" "sync" "testing" "time" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) type testCase struct { @@ -751,9 +750,9 @@ func TestAccount_parsePosting(t *testing.T) { } return } - if !reflect.DeepEqual(a, tt.want) { - aJson, _ := json.Marshal(a) - wantJson, _ := json.Marshal(tt.want) + aJson, _ := json.Marshal(a) + wantJson, _ := json.Marshal(tt.want) + if string(aJson) != string(wantJson) { t.Errorf("got %+v wanted %+v", string(aJson), string(wantJson)) } if tt.wantErr { diff --git a/types.go b/types.go index 0e2e78e..f141afc 100644 --- a/types.go +++ b/types.go @@ -3,7 +3,7 @@ package ledger import ( "time" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) // Account holds the name and balance From dcf32174b7c3a4904881bfd46b0d72d956fa699e Mon Sep 17 00:00:00 2001 From: Aidan <6690599+aodhan-domhnaill@users.noreply.github.com> Date: Thu, 12 Feb 2026 22:24:39 +0200 Subject: [PATCH 09/14] Implicit rates support (#6) * moving balance computation into separate function for maintainability * implicit conversion factor computation * testing raw transaction block * testing raw transaction block * separated comment capture from parser * remove postings from parser * remove excess indexes * separate out include * separating block parsing from transaction parsing * fixing decimal precision bug * print currency --------- Co-authored-by: Aidan Macdonald --- ledger/cmd/print.go | 134 +++++++++++++++++++++++----- parse.go | 194 +++++++++++++++++++---------------------- parse_test.go | 80 ++++++++++++++++- transaction.go | 179 ++++++++++++++++++++++++++++++++++++++ transaction_test.go | 207 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 666 insertions(+), 128 deletions(-) create mode 100644 transaction.go create mode 100644 transaction_test.go diff --git a/ledger/cmd/print.go b/ledger/cmd/print.go index 49ff88a..fb2f714 100644 --- a/ledger/cmd/print.go +++ b/ledger/cmd/print.go @@ -187,6 +187,15 @@ func WriteTransaction(w io.StringWriter, trans *ledger.Transaction, columns int) w.WriteString(newLine) for _, accChange := range trans.AccountChanges { outBalanceString := accChange.Balance.StringFixedBank(2) + if accChange.Currency != "" { + outBalanceString = accChange.Currency + " " + outBalanceString + } + // Show converted amount (@@) or conversion factor (@) similar to hledger + if accChange.Converted != nil { + outBalanceString = outBalanceString + " @@ " + accChange.Converted.StringFixedBank(2) + } else if accChange.ConversionFactor != nil { + outBalanceString = outBalanceString + " @ " + accChange.ConversionFactor.String() + } spaceCount := columns - 4 - utf8.RuneCountInString(accChange.Name) - utf8.RuneCountInString(outBalanceString) if spaceCount < 1 { spaceCount = 1 @@ -242,7 +251,9 @@ func PrintRegister(generalLedger []*ledger.Transaction, filterArr []string, colu colorReset := fastcolor.Reset buf := bufio.NewWriter(os.Stdout) - runningBalance := decimal.Zero + // runningBalance keeps the total per currency + runningBalance := make(map[string]decimal.Decimal) + for _, trans := range generalLedger { for _, accChange := range trans.AccountChanges { inFilter := len(filterArr) == 0 @@ -251,30 +262,104 @@ func PrintRegister(generalLedger []*ledger.Transaction, filterArr []string, colu inFilter = true } } - if inFilter { - runningBalance = runningBalance.Add(accChange.Balance) - outBalanceString := accChange.Balance.StringFixedBank(2) - outRunningBalanceString := runningBalance.StringFixedBank(2) + if !inFilter { + continue + } - balamtColor := colorReset - if accChange.Balance.Sign() < 0 { - balamtColor = colorNeg + // Update running totals per currency + cur := accChange.Currency + if cur == "" { + cur = "_" // treat empty currency as its own bucket + } + runningBalance[cur] = runningBalance[cur].Add(accChange.Balance) + + // Current posting amount string + outBalanceString := accChange.Balance.StringFixedBank(2) + if accChange.Currency != "" { + outBalanceString = accChange.Currency + " " + outBalanceString + } + + // Build primary running total string (first currency: the one for this posting) + type curTotal struct { + currency string + amount decimal.Decimal + } + totals := make([]curTotal, 0, len(runningBalance)) + for k, v := range runningBalance { + totals = append(totals, curTotal{currency: k, amount: v}) + } + // Sort for deterministic output: primary currency first, then by name + slices.SortFunc(totals, func(a, b curTotal) int { + // primary currency first + if a.currency == cur && b.currency != cur { + return -1 + } + if b.currency == cur && a.currency != cur { + return 1 } - runamtColor := colorReset - if runningBalance.Sign() < 0 { - runamtColor = colorNeg + // "_" (no currency) should sort last + if a.currency == "_" && b.currency != "_" { + return 1 } + if b.currency == "_" && a.currency != "_" { + return -1 + } + return strings.Compare(a.currency, b.currency) + }) + + formatTotal := func(ct curTotal) string { + amtStr := ct.amount.StringFixedBank(2) + if ct.currency == "_" { + return amtStr + } + return ct.currency + " " + amtStr + } + + primaryTotal := formatTotal(totals[0]) + + // Colors + balamtColor := colorReset + if accChange.Balance.Sign() < 0 { + balamtColor = colorNeg + } + runamtColor := colorReset + if totals[0].amount.Sign() < 0 { + runamtColor = colorNeg + } - buf.WriteString(trans.Date.Format(transactionDateFormat)) - buf.WriteString(" ") - colorPayee.WriteStringFixed(buf, trans.Payee, col1width, false) - buf.WriteString(" ") - colorAccount.WriteStringFixed(buf, accChange.Name, col2width, false) - buf.WriteString(" ") - balamtColor.WriteStringFixed(buf, outBalanceString, 10, true) - buf.WriteString(" ") - runamtColor.WriteStringFixed(buf, outRunningBalanceString, 10, true) - buf.WriteString(newLine) + // First line with primary total + buf.WriteString(trans.Date.Format(transactionDateFormat)) + buf.WriteString(" ") + colorPayee.WriteStringFixed(buf, trans.Payee, col1width, false) + buf.WriteString(" ") + colorAccount.WriteStringFixed(buf, accChange.Name, col2width, false) + buf.WriteString(" ") + balamtColor.WriteStringFixed(buf, outBalanceString, 10, true) + buf.WriteString(" ") + runamtColor.WriteStringFixed(buf, primaryTotal, 10, true) + buf.WriteString(newLine) + + // Additional lines for other currencies in running total + if len(totals) > 1 { + for _, ct := range totals[1:] { + otherTotal := formatTotal(ct) + otherColor := colorReset + if ct.amount.Sign() < 0 { + otherColor = colorNeg + } + + // Empty date/payee/account/amount columns, only total column + buf.WriteString(strings.Repeat(" ", 10)) // date + buf.WriteString(" ") + colorPayee.WriteStringFixed(buf, "", col1width, false) + buf.WriteString(" ") + colorAccount.WriteStringFixed(buf, "", col2width, false) + buf.WriteString(" ") + balamtColor.WriteStringFixed(buf, "", 10, true) + buf.WriteString(" ") + otherColor.WriteStringFixed(buf, otherTotal, 10, true) + buf.WriteString(newLine) + } } } } @@ -301,7 +386,12 @@ func PrintCSV(generalLedger []*ledger.Transaction, filterArr []string) { record := []string{trans.Date.Format(transactionDateFormat), trans.Payee, accChange.Name, - outBalanceString, + func() string { + if accChange.Currency != "" { + return accChange.Currency + " " + outBalanceString + } + return outBalanceString + }(), } if err := csvWriter.Write(record); err != nil { fmt.Fprintf(os.Stderr, "error writing record to CSV: %s", err) diff --git a/parse.go b/parse.go index e3164d7..0e64411 100644 --- a/parse.go +++ b/parse.go @@ -89,37 +89,16 @@ type parser struct { strPrevDate string prevDateErr error prevDate time.Time - - transactions []Transaction - ctIdx int - postings []Account - cpIdx int -} - -const preAllocSize = 100000 -const preAllocWarn = 10 - -func (p *parser) init() { - p.transactions = make([]Transaction, preAllocSize) - p.postings = make([]Account, preAllocSize*3) - p.ctIdx = 0 - p.cpIdx = 0 -} - -func (p *parser) grow() { - if len(p.transactions)-p.ctIdx < preAllocWarn || - len(p.postings)-p.cpIdx < (preAllocWarn*3) { - p.init() - } } func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Transaction, err error) (stop bool)) (stop bool) { var lp parser - lp.init() lp.scanner = newLineScanner(filename, ledgerReader) var tlist []*Transaction + blocks := []block{} + comments := []string{} for lp.scanner.Scan() { // remove heading and tailing space from the line trimmedLine := strings.TrimSpace(lp.scanner.Text()) @@ -135,7 +114,7 @@ func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Tra // Skip empty lines if len(trimmedLine) == 0 { if len(currentComment) > 0 { - lp.comments = append(lp.comments, currentComment) + comments = append(comments, currentComment) } continue } @@ -147,7 +126,7 @@ func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Tra return true } if len(currentComment) > 0 { - lp.comments = append(lp.comments, currentComment) + comments = append(comments, currentComment) } continue } @@ -155,37 +134,33 @@ func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Tra case "account": lp.skipAccount() case "include": - paths, _ := filepath.Glob(filepath.Join(filepath.Dir(lp.scanner.Name()), after)) - if len(paths) < 1 { - callback(nil, fmt.Errorf("%s:%d: unable to include file(%s): %w", lp.scanner.Name(), lp.scanner.LineNumber(), after, errors.New("not found"))) - return true - } - var wg sync.WaitGroup - for _, incpath := range paths { - wg.Add(1) - go func(ipath string) { - ifile, _ := os.Open(ipath) - defer ifile.Close() - if parseLedger(ipath, ifile, callback) { - stop = true - } - wg.Done() - }(incpath) - } - wg.Wait() + stop := lp.include(after, callback) if stop { return stop } default: - trans, transErr := lp.parseTransaction(before, after, currentComment) - if transErr != nil { - if callback(nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", lp.scanner.Name(), lp.scanner.LineNumber(), transErr)) { + transDate, derr := lp.parseDate(before) + if derr != nil { + if callback(nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", lp.scanner.Name(), lp.scanner.LineNumber(), derr)) { return true } continue } - tlist = append(tlist, trans) + + blocks = append(blocks, lp.parseBlock(transDate, after, currentComment, comments)) + comments = []string{} + } + } + + for _, block := range blocks { + trans, transErr := block.parseTransaction() + if transErr != nil { + if callback(nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", block.filename, block.lineNum, transErr)) { + return true + } + continue } + tlist = append(tlist, trans) } callback(tlist, nil) return false @@ -200,6 +175,28 @@ func (lp *parser) skipAccount() { } } +func (lp *parser) include(after string, callback func(t []*Transaction, err error) (stop bool)) (stop bool) { + paths, _ := filepath.Glob(filepath.Join(filepath.Dir(lp.scanner.Name()), after)) + if len(paths) < 1 { + callback(nil, fmt.Errorf("%s:%d: unable to include file(%s): %w", lp.scanner.Name(), lp.scanner.LineNumber(), after, errors.New("not found"))) + return true + } + var wg sync.WaitGroup + for _, incpath := range paths { + wg.Add(1) + go func(ipath string) { + ifile, _ := os.Open(ipath) + defer ifile.Close() + if parseLedger(ipath, ifile, callback) { + stop = true + } + wg.Done() + }(incpath) + } + wg.Wait() + return +} + func (lp *parser) parseDate(dateString string) (transDate time.Time, err error) { // seen before, skip parse if lp.strPrevDate == dateString { @@ -224,7 +221,7 @@ func (lp *parser) parseDate(dateString string) (transDate time.Time, err error) return } -func (a *Account) parsePosting(trimmedLine string) (err error) { +func (a *Account) parsePosting(trimmedLine string, comment string) (err error) { trimmedLine = strings.TrimSpace(trimmedLine) // Regex groups: @@ -249,6 +246,7 @@ func (a *Account) parsePosting(trimmedLine string) (err error) { a.Name = m[1] a.Currency = m[2] + a.Comment = comment if m[3] != "" { bal, err := compute.Evaluate(m[3]) @@ -278,86 +276,72 @@ func (a *Account) parsePosting(trimmedLine string) (err error) { return } -func (lp *parser) parseTransaction(dateString, payeeString, payeeComment string) (trans *Transaction, err error) { - transDate, derr := lp.parseDate(dateString) - if derr != nil { - return nil, derr - } - - transBal := decimal.Zero - var numEmpty int - var emptyAccIndex int - var accIndex int +type block struct { + transDate time.Time + payeeString string + payeeComment string + comments []string + lines []string + filename string + lineNum int +} +func (lp *parser) parseBlock(transDate time.Time, payeeString, payeeComment string, comments []string) block { + lines := []string{} for lp.scanner.Scan() { trimmedLine := lp.scanner.Text() + lines = append(lines, trimmedLine) + if len(trimmedLine) == 0 { + break + } + } + + return block{ + transDate: transDate, + payeeString: payeeString, + payeeComment: payeeComment, + comments: comments, + lines: lines, + filename: lp.scanner.Name(), + lineNum: lp.scanner.LineNumber(), + } +} +func (b *block) parseTransaction() (trans *Transaction, err error) { + trans = &Transaction{} + for _, trimmedLine := range b.lines { + postingComment := "" // handle comments if commentIdx := strings.Index(trimmedLine, ";"); commentIdx >= 0 { currentComment := trimmedLine[commentIdx:] trimmedLine = trimmedLine[:commentIdx] trimmedLine = strings.TrimSpace(trimmedLine) if len(trimmedLine) == 0 { - lp.comments = append(lp.comments, currentComment) + b.comments = append(b.comments, currentComment) continue } - lp.postings[lp.cpIdx+accIndex].Comment = currentComment + postingComment = currentComment } if len(trimmedLine) == 0 { break } - _ = lp.postings[lp.cpIdx+accIndex].parsePosting(trimmedLine) - - if lp.postings[lp.cpIdx+accIndex].Balance.IsZero() { - numEmpty++ - emptyAccIndex = accIndex - } - - if lp.postings[lp.cpIdx+accIndex].Converted != nil { - transBal = transBal.Add(lp.postings[lp.cpIdx+accIndex].Converted.Neg()) - } else if lp.postings[lp.cpIdx+accIndex].ConversionFactor != nil { - transBal = transBal.Add(lp.postings[lp.cpIdx+accIndex].Balance.Mul( - *lp.postings[lp.cpIdx+accIndex].ConversionFactor, - )) - } else { - transBal = transBal.Add(lp.postings[lp.cpIdx+accIndex].Balance) - } - accIndex++ + posting := Account{} + posting.parsePosting(trimmedLine, postingComment) + trans.AccountChanges = append(trans.AccountChanges, posting) } - if accIndex < 2 { - err = errors.New("need at least two postings") - return + trans.Payee = b.payeeString + trans.Date = b.transDate + trans.PayeeComment = b.payeeComment + if len(b.comments) > 0 { + trans.Comments = b.comments } - if !transBal.IsZero() { - switch numEmpty { - case 0: - return nil, errors.New("unable to balance transaction: no empty account to place extra balance") - case 1: - // If there is a single empty account, then it is obvious where to - // place the remaining balance. - lp.postings[lp.cpIdx+emptyAccIndex].Balance = transBal.Neg() - default: - return nil, errors.New("unable to balance transaction: more than one account empty") - } + if err = trans.IsBalanced(); err != nil { + return nil, err } - lp.transactions[lp.ctIdx].Payee = payeeString - lp.transactions[lp.ctIdx].Date = transDate - lp.transactions[lp.ctIdx].PayeeComment = payeeComment - lp.transactions[lp.ctIdx].AccountChanges = lp.postings[lp.cpIdx : lp.cpIdx+accIndex] - lp.transactions[lp.ctIdx].Comments = lp.comments - - trans = &lp.transactions[lp.ctIdx] - - lp.comments = nil - lp.cpIdx += accIndex - lp.ctIdx++ - - lp.grow() - return } diff --git a/parse_test.go b/parse_test.go index 8815743..ab2cb36 100644 --- a/parse_test.go +++ b/parse_test.go @@ -563,6 +563,84 @@ account Assets }, nil, }, + { + "conversion implicit rate", + `1970/01/01 Converted CZK to EUR + Assets:Wise:CZK CZK -2000.00 + Assets:Wise:EUR EUR 1000.00 +`, + []*Transaction{ + { + Payee: "Converted CZK to EUR", + Date: time.Unix(0, 0).UTC(), + AccountChanges: []Account{ + { + Name: "Assets:Wise:CZK", + Currency: "CZK", + Balance: decimal.NewFromFloat(-2000.0), + }, + { + Name: "Assets:Wise:EUR", + Currency: "EUR", + Balance: decimal.NewFromFloat(1000.0), + Converted: p(decimal.NewFromFloat(-2000.0)), + }, + }, + }, + }, + nil, + }, + { + "conversion implicit rate USD", + `; test comment +1970/01/01 Wise Charges for: BALANCE + assets:wise EUR -8 + expenses:bank:fees EUR 8 + +; test comment +1970/01/01 Converted EUR to USD + assets:wise EUR -1000 + assets:wise USD 2060 +`, + []*Transaction{ + { + Payee: "Wise Charges for: BALANCE", + Date: time.Unix(0, 0).UTC(), + Comments: []string{"; test comment"}, + AccountChanges: []Account{ + { + Name: "assets:wise", + Currency: "EUR", + Balance: decimal.NewFromFloat(-8.0), + }, + { + Name: "expenses:bank:fees", + Currency: "EUR", + Balance: decimal.NewFromFloat(8.0), + }, + }, + }, + { + Payee: "Converted EUR to USD", + Date: time.Unix(0, 0).UTC(), + Comments: []string{"; test comment"}, + AccountChanges: []Account{ + { + Name: "assets:wise", + Currency: "EUR", + Balance: decimal.NewFromFloat(-1000.0), + }, + { + Name: "assets:wise", + Currency: "USD", + Balance: decimal.NewFromFloat(2060.0), + Converted: p(decimal.NewFromFloat(-1000)), + }, + }, + }, + }, + nil, + }, } func p(d decimal.Decimal) *decimal.Decimal { @@ -743,7 +821,7 @@ func TestAccount_parsePosting(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { a := Account{} - gotErr := a.parsePosting(tt.trimmedLine) + gotErr := a.parsePosting(tt.trimmedLine, "") if gotErr != nil { if !tt.wantErr { t.Errorf("parsePosting() failed: %v", gotErr) diff --git a/transaction.go b/transaction.go new file mode 100644 index 0000000..4e89a67 --- /dev/null +++ b/transaction.go @@ -0,0 +1,179 @@ +package ledger + +import ( + "errors" + + "github.com/shopspring/decimal" +) + +var ( + ErrNeedAtLeastTwoPostings = errors.New("need at least two postings") + ErrNoEmptyAccountForExtraBalance = errors.New("unable to balance transaction: no empty account to place extra balance") + ErrMoreThanOneEmptyAccountInTx = errors.New("unable to balance transaction: more than one account empty") +) + +func (t *Transaction) IsBalanced() error { + if len(t.AccountChanges) < 2 { + return ErrNeedAtLeastTwoPostings + } + + if err := t.inferConversionFactorForTwoCurrencyTx(); err != nil { + return err + } + + transBal := decimal.Zero + var numEmpty int + var emptyAccIndex int + + for i, acc := range t.AccountChanges { + if acc.Balance.IsZero() { + numEmpty++ + emptyAccIndex = i + } + + if acc.Converted != nil { + transBal = transBal.Add(acc.Converted.Neg()) + } else if acc.ConversionFactor != nil { + transBal = transBal.Add(acc.Balance.Mul(*acc.ConversionFactor)) + } else { + transBal = transBal.Add(acc.Balance) + } + } + + if !transBal.IsZero() { + switch numEmpty { + case 0: + return ErrNoEmptyAccountForExtraBalance + case 1: + // If there is a single empty account, then it is obvious where to + // place the remaining balance. + t.AccountChanges[emptyAccIndex].Balance = transBal.Neg() + default: + return ErrMoreThanOneEmptyAccountInTx + } + } + + return nil +} + +func (t *Transaction) inferConversionFactorForTwoCurrencyTx() error { + type currencyGroup struct { + indices []int + } + + currencyMap := make(map[string]*currencyGroup) + + getCurrencyKey := func(a *Account) string { + if a.Converted != nil { + // TODO: explicit currency for conversion? + return a.Currency + } + return a.Currency + } + + for i := range t.AccountChanges { + acc := &t.AccountChanges[i] + key := getCurrencyKey(acc) + if key == "" { + return nil + } + group, ok := currencyMap[key] + if !ok { + group = ¤cyGroup{} + currencyMap[key] = group + } + group.indices = append(group.indices, i) + } + + if len(currencyMap) != 2 { + return nil + } + + var ( + curKeys [2]string + groups [2]*currencyGroup + i int + ) + for k, g := range currencyMap { + if i >= 2 { + break + } + curKeys[i] = k + groups[i] = g + i++ + } + + var baseCurIdx, otherCurIdx int + hasConv0 := false + for _, idx := range groups[0].indices { + if t.AccountChanges[idx].ConversionFactor != nil { + hasConv0 = true + break + } + } + hasConv1 := false + for _, idx := range groups[1].indices { + if t.AccountChanges[idx].ConversionFactor != nil { + hasConv1 = true + break + } + } + + switch { + case hasConv0 && hasConv1: + return nil + case hasConv0: + baseCurIdx, otherCurIdx = 1, 0 + case hasConv1: + baseCurIdx, otherCurIdx = 0, 1 + default: + baseCurIdx, otherCurIdx = 0, 1 + } + + sumForGroup := func(g *currencyGroup) (decimal.Decimal, error) { + total := decimal.Zero + for _, idx := range g.indices { + acc := &t.AccountChanges[idx] + if acc.Converted != nil { + total = total.Add(acc.Converted.Neg()) + } else if acc.ConversionFactor != nil { + total = total.Add(acc.Balance.Mul(*acc.ConversionFactor)) + } else { + total = total.Add(acc.Balance) + } + } + return total, nil + } + + sumBase, _ := sumForGroup(groups[baseCurIdx]) + sumOtherRaw := decimal.Zero + for _, idx := range groups[otherCurIdx].indices { + acc := &t.AccountChanges[idx] + if acc.Converted != nil || acc.ConversionFactor != nil { + if acc.Converted != nil { + sumOtherRaw = sumOtherRaw.Add(acc.Converted.Neg()) + } else if acc.ConversionFactor != nil { + sumOtherRaw = sumOtherRaw.Add(acc.Balance.Mul(*acc.ConversionFactor)) + } + } else { + sumOtherRaw = sumOtherRaw.Add(acc.Balance) + } + } + + if sumOtherRaw.IsZero() { + return nil + } + if sumBase.Add(sumOtherRaw).IsZero() { + return nil + } + + for _, idx := range groups[otherCurIdx].indices { + acc := &t.AccountChanges[idx] + if acc.ConversionFactor == nil && acc.Converted == nil { + conv := acc.Balance.Mul(sumBase).Div(sumOtherRaw) + acc.Converted = &conv + } + } + + return nil +} diff --git a/transaction_test.go b/transaction_test.go new file mode 100644 index 0000000..2a63031 --- /dev/null +++ b/transaction_test.go @@ -0,0 +1,207 @@ +package ledger + +import ( + "testing" + + "github.com/shopspring/decimal" +) + +func TestIsBalanced(t *testing.T) { + tests := []struct { + name string + tx *Transaction + wantErr error + wantBalances []decimal.Decimal + }{ + { + name: "errors on too few postings", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(10), + }, + }, + }, + wantErr: ErrNeedAtLeastTwoPostings, + wantBalances: nil, + }, + { + name: "no empty account error", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(10), + }, + { + Name: "Expenses:Food", + Balance: decimal.NewFromInt(-5), + }, + }, + }, + wantErr: ErrNoEmptyAccountForExtraBalance, + wantBalances: nil, + }, + { + name: "more than one empty account error", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(10), + }, + { + Name: "Expenses:Food", + }, + { + Name: "Equity:OpeningBalances", + }, + }, + }, + wantErr: ErrMoreThanOneEmptyAccountInTx, + wantBalances: nil, + }, + { + name: "single empty account gets balancing amount", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Expenses:Food", + }, + }, + }, + wantErr: nil, + wantBalances: []decimal.Decimal{decimal.NewFromInt(-10), decimal.NewFromInt(10)}, + }, + { + name: "already balanced with no empty account", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Expenses:Food", + Balance: decimal.NewFromInt(10), + }, + }, + }, + wantErr: nil, + wantBalances: []decimal.Decimal{decimal.NewFromInt(-10), decimal.NewFromInt(10)}, + }, + { + name: "two currency implicit conversion factor inferred", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank:USD", + Currency: "USD", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Assets:Bank:EUR", + Currency: "EUR", + Balance: decimal.NewFromInt(5), + }, + }, + }, + wantErr: nil, + wantBalances: nil, + }, + { + name: "two currency implicit conversion factor inferred multiple", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank:USD", + Currency: "USD", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Assets:Bank:EUR", + Currency: "EUR", + Balance: decimal.NewFromInt(5), + }, + { + Name: "Assets:otherBank:EUR", + Currency: "EUR", + Balance: decimal.NewFromInt(3), + }, + }, + }, + wantErr: nil, + wantBalances: nil, + }, + { + name: "does not infer conversion factor for three currencies", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank:USD", + Currency: "USD", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Assets:Bank:EUR", + Currency: "EUR", + Balance: decimal.NewFromInt(5), + }, + { + Name: "Assets:Bank:GBP", + Currency: "GBP", + Balance: decimal.NewFromInt(3), + }, + }, + }, + wantErr: ErrNoEmptyAccountForExtraBalance, + wantBalances: nil, + }, + { + name: "decimal precision bug", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Wise:CZK", + Currency: "CZK", + Balance: decimal.NewFromFloat(-2003.0), + }, + { + Name: "Assets:Wise:EUR", + Currency: "EUR", + Balance: decimal.NewFromFloat(1000.0), + }, + }, + }, + wantErr: nil, + wantBalances: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := tt.tx.IsBalanced() + if err != tt.wantErr { + t.Fatalf("expected error %v, got %v", tt.wantErr, err) + } + + if tt.wantBalances != nil { + if len(tt.tx.AccountChanges) != len(tt.wantBalances) { + t.Fatalf("expected %d account balances, got %d", len(tt.wantBalances), len(tt.tx.AccountChanges)) + } + for i, want := range tt.wantBalances { + if !tt.tx.AccountChanges[i].Balance.Equal(want) { + t.Fatalf("account %d: expected balance %s, got %s", i, want.String(), tt.tx.AccountChanges[i].Balance.String()) + } + } + } + }) + } +} From d8992e5bd5645af5304358fcb6228ba47d3e003c Mon Sep 17 00:00:00 2001 From: Aidan <6690599+aodhan-domhnaill@users.noreply.github.com> Date: Fri, 20 Feb 2026 18:49:41 +0200 Subject: [PATCH 10/14] Qif support (#7) * adding qif code copied from qfx * add QIF support and replace ParseQFX with ParseQIF * add QIF import support * fix unique accounts bug * add override-currency flag and apply currency to imports * fix currency override --------- Co-authored-by: Aidan Macdonald --- ledger/cmd/import.go | 125 +++++++++++-- ledger/cmd/internal/import/qif/qif.go | 200 +++++++++++++++++++++ ledger/cmd/internal/import/qif/qif_test.go | 101 +++++++++++ ledger/cmd/internal/import/qif/sample.qif | 35 ++++ 4 files changed, 450 insertions(+), 11 deletions(-) create mode 100644 ledger/cmd/internal/import/qif/qif.go create mode 100644 ledger/cmd/internal/import/qif/qif_test.go create mode 100644 ledger/cmd/internal/import/qif/sample.qif diff --git a/ledger/cmd/import.go b/ledger/cmd/import.go index 1592c34..75cd91d 100644 --- a/ledger/cmd/import.go +++ b/ledger/cmd/import.go @@ -13,6 +13,7 @@ import ( "github.com/howeyc/ledger" "github.com/howeyc/ledger/ledger/cmd/internal/import/camt" "github.com/howeyc/ledger/ledger/cmd/internal/import/qfx" + "github.com/howeyc/ledger/ledger/cmd/internal/import/qif" "github.com/jbrukh/bayesian" "github.com/shopspring/decimal" "github.com/spf13/cobra" @@ -27,13 +28,22 @@ var negateAmount bool var allowMatching bool var fieldDelimiter string var scaleFactor float64 +var overrideCurrency string func trainClassifier(generalLedger []*ledger.Transaction, matchingAccount string) *bayesian.Classifier { allAccounts := ledger.GetBalances(generalLedger, []string{}) - classes := make([]bayesian.Class, len(allAccounts)) - for i, bal := range allAccounts { - classes[i] = bayesian.Class(bal.Name) + uniqueAccounts := make(map[string]bool) + for _, acc := range allAccounts { + if ok, _ := uniqueAccounts[acc.Name]; !ok { + uniqueAccounts[acc.Name] = true + } + } + + classes := []bayesian.Class{} + for name := range uniqueAccounts { + classes = append(classes, bayesian.Class(name)) } + classifier := bayesian.NewClassifier(classes...) for _, tran := range generalLedger { payeeWords := strings.Fields(tran.Payee) @@ -185,11 +195,14 @@ func importCSV(accountSubstring, csvFileName string) { // Csv amount is the negative of the expense amount csvAccount.Balance = expenseAccount.Balance.Neg() - // Create valid transaction for print in ledger format trans := &ledger.Transaction{Date: csvDate, Payee: record[payeeColumn]} trans.AccountChanges = []ledger.Account{csvAccount, expenseAccount} - // Comment + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } if commentColumn >= 0 && record[commentColumn] != "" { trans.Comments = []string{";" + record[commentColumn]} } @@ -274,11 +287,17 @@ func importCamt(accountSubstring, camtFileName string) { // Csv amount is the negative of the expense amount camtAccount.Balance = expenseAccount.Balance.Neg() - // Create valid transaction for print in ledger format trans := &ledger.Transaction{Date: dateTime, Payee: payee} trans.AccountChanges = []ledger.Account{camtAccount, expenseAccount} - - // Comment + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } else if entry.Amt.Ccy != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = entry.Amt.Ccy + } + } if reference != "" { trans.Comments = []string{";" + reference} } @@ -286,6 +305,85 @@ func importCamt(accountSubstring, camtFileName string) { } } +func importQIF(accountSubstring, qifFileName string) { + decScale := decimal.NewFromFloat(scaleFactor) + + fileReader, err := os.Open(qifFileName) + if err != nil { + fmt.Println("QIF: ", err, qifFileName) + return + } + defer fileReader.Close() + + generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) + if parseError != nil { + fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) + return + } + + matchingAccount, err := findMatchingAccount(generalLedger, accountSubstring) + if err != nil { + fmt.Println(err) + return + } + + classifier := trainClassifier(generalLedger, matchingAccount) + + entries, err := qif.ParseQIF(fileReader) + if err != nil { + fmt.Println("QIF parse error:", err.Error()) + return + } + + expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} + qifAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero} + for _, entry := range entries { + // Parse date (QIF dates are often locale-specific; assume mm/dd/yyyy here) + dateTime, err := time.Parse("01/02/2006", entry.Date) + if err != nil { + // Try an alternate common QIF date format (dd/mm/yyyy) + dateTime, err = time.Parse("02/01/2006", entry.Date) + if err != nil { + fmt.Println("QIF date parse error:", err.Error()) + continue + } + } + + // Parse amount + amount, err := decimal.NewFromString(entry.Amount) + if err != nil { + fmt.Println("QIF amount parse error:", err.Error()) + continue + } + + payee := entry.Payee + inputPayeeWords := strings.Fields(payee) + + expenseAccount.Name = predictAccount(classifier, inputPayeeWords) + expenseAccount.Balance = amount + + // Apply scale + expenseAccount.Balance = expenseAccount.Balance.Mul(decScale) + + // Account side is the opposite of expense + qifAccount.Balance = expenseAccount.Balance.Neg() + + trans := &ledger.Transaction{Date: dateTime, Payee: payee} + trans.AccountChanges = []ledger.Account{qifAccount, expenseAccount} + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } + if len(entry.RawLines) > 0 { + // Join all raw lines except header/type line + comment := strings.Join(entry.RawLines, " ") + trans.Comments = []string{";" + comment} + } + WriteTransaction(os.Stdout, trans, 80) + } +} + func importQFX(accountSubstring, qfxFileName string) { decScale := decimal.NewFromFloat(scaleFactor) @@ -350,11 +448,13 @@ func importQFX(accountSubstring, qfxFileName string) { // Account side is the opposite of expense qfxAccount.Balance = expenseAccount.Balance.Neg() - // Create valid transaction for print in ledger format trans := &ledger.Transaction{Date: dateTime, Payee: payee} trans.AccountChanges = []ledger.Account{qfxAccount, expenseAccount} - - // Comment with FITID if present + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } if entry.FitID != "" { trans.Comments = []string{";" + entry.FitID} } @@ -376,6 +476,8 @@ var importCmd = &cobra.Command{ importCamt(accountSubstring, fileName) } else if strings.HasSuffix(lower, ".qfx") || strings.HasSuffix(lower, ".ofx") { importQFX(accountSubstring, fileName) + } else if strings.HasSuffix(lower, ".qif") { + importQIF(accountSubstring, fileName) } else { importCSV(accountSubstring, fileName) } @@ -391,6 +493,7 @@ func init() { importCmd.Flags().Float64Var(&scaleFactor, "scale", 1.0, "Scale factor to multiply against every imported amount.") importCmd.Flags().StringVar(&csvDateFormat, "date-format", "01/02/2006", "Date format.") importCmd.Flags().StringVar(&fieldDelimiter, "delimiter", ",", "Field delimiter.") + importCmd.Flags().StringVar(&overrideCurrency, "override-currency", "", "Override detected currency for imported transactions.") } func existingTransaction(generalLedger []*ledger.Transaction, transDate time.Time, payee string) bool { diff --git a/ledger/cmd/internal/import/qif/qif.go b/ledger/cmd/internal/import/qif/qif.go new file mode 100644 index 0000000..ec855a2 --- /dev/null +++ b/ledger/cmd/internal/import/qif/qif.go @@ -0,0 +1,200 @@ +package qif + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +// Non-investment QIF transaction, based on the "Non-investment transaction format" +// from the GnuCash documentation. Only a subset of fields is modeled for now. +type Transaction struct { + // Header/type line, e.g. "!Type:Cash" + Type string `qif:"header"` + + // Core transaction fields + Date string `qif:"D"` // D - Date + Amount string `qif:"T"` // T - Amount + Num string `qif:"N"` // N - Number (check/reference) + Payee string `qif:"P"` // P - Payee/description + Memo string `qif:"M"` // M - Memo + Addr string `qif:"A"` // A - Address (multi-line; kept concatenated with '\n') + Cleared string `qif:"C"` // C - Cleared status + Category string `qif:"L"` // L - Category (or transfer/class) + + // Split fields – repeated groups, flattened for now to first occurrence + SplitCategory string `qif:"S"` // S - Category in split + SplitMemo string `qif:"E"` // E - Memo in split + SplitAmount string `qif:"$"` // $ - Dollar amount of split + + // RawLines contains the raw QIF lines (without trailing newline) that + // composed this transaction, excluding the header and trailing '^'. + RawLines []string `qif:"-"` +} + +// Decoder reads QIF data from an input stream. +type Decoder struct { + r *bufio.Reader +} + +// NewDecoder returns a new QIF decoder that reads from r. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + r: bufio.NewReader(r), + } +} + +// Decode reads QIF data from the underlying reader and returns all parsed +// non-investment transactions. For now this is a convenience wrapper around +// a streaming decode; it reads the whole file. +func (d *Decoder) Decode() ([]*Transaction, error) { + var ( + transactions []*Transaction + currentType string + ) + + for { + line, err := d.readLine() + if err == io.EOF { + // No partial transaction handling – QIF files should end with '^' + return transactions, nil + } + if err != nil { + return nil, err + } + + if len(line) == 0 { + continue + } + + // Header / account-type line: !Type:Cash, !Type:Bank, ... + if strings.HasPrefix(line, "!Type:") { + currentType = strings.TrimSpace(line[len("!Type:"):]) + continue + } + + // A transaction must start with 'D' (date) according to the spec. + if line[0] == 'D' { + tx, err := d.decodeTransaction(currentType, line) + if err != nil { + return nil, err + } + transactions = append(transactions, tx) + continue + } + + // Lines outside of transactions are currently ignored. + } + +} + +// decodeTransaction parses a single transaction, given that the first line +// (already read) is a 'D' date line. It continues reading until the '^' end +// marker has been consumed. +func (d *Decoder) decodeTransaction(txType string, firstLine string) (*Transaction, error) { + tx := &Transaction{ + Type: txType, + } + + assignField(tx, firstLine) + + for { + line, err := d.readLine() + if err != nil { + if err == io.EOF { + return nil, fmt.Errorf("unexpected EOF while reading transaction") + } + return nil, err + } + if len(line) == 0 { + // empty lines inside a transaction are preserved in RawLines but + // don't correspond to any field. + tx.RawLines = append(tx.RawLines, line) + continue + } + if line[0] == '^' { + // end of transaction + return tx, nil + } + + assignField(tx, line) + } +} + +// assignField updates tx based on a single QIF field line. +// It also appends the raw line (minus trailing newline) to RawLines. +func assignField(tx *Transaction, line string) { + if len(line) == 0 { + return + } + // Store raw line + tx.RawLines = append(tx.RawLines, line) + + prefix := line[0] + value := line[1:] + + switch prefix { + case 'D': + tx.Date = value + case 'T': + tx.Amount = value + case 'U': + // Higher precision amount; if present, prefer it over T. + tx.Amount = value + case 'N': + tx.Num = value + case 'P': + tx.Payee = value + case 'M': + if tx.Memo == "" { + tx.Memo = value + } else { + // Multiple memo lines – concatenate with newline. + tx.Memo += "\n" + value + } + case 'A': + if tx.Addr == "" { + tx.Addr = value + } else { + tx.Addr += "\n" + value + } + case 'C': + tx.Cleared = value + case 'L': + tx.Category = value + case 'S': + // For now we keep only first split; real-world usage may need a slice. + if tx.SplitCategory == "" { + tx.SplitCategory = value + } + case 'E': + if tx.SplitMemo == "" { + tx.SplitMemo = value + } + case '$': + if tx.SplitAmount == "" { + tx.SplitAmount = value + } + } +} + +// readLine reads a single logical line without the trailing '\n' or '\r\n'. +func (d *Decoder) readLine() (string, error) { + line, err := d.r.ReadString('\n') + if err != nil && err != io.EOF { + return "", err + } + // Trim CRLF and LF. + line = strings.TrimRight(line, "\r\n") + if err == io.EOF && len(line) == 0 { + return "", io.EOF + } + return line, err +} + +// ParseQIF is a convenience helper that parses all transactions from a QIF +// stream and returns them. +func ParseQIF(reader io.Reader) ([]*Transaction, error) { + return NewDecoder(reader).Decode() +} diff --git a/ledger/cmd/internal/import/qif/qif_test.go b/ledger/cmd/internal/import/qif/qif_test.go new file mode 100644 index 0000000..224e858 --- /dev/null +++ b/ledger/cmd/internal/import/qif/qif_test.go @@ -0,0 +1,101 @@ +package qif_test + +import ( + "bytes" + _ "embed" + "testing" + + "github.com/howeyc/ledger/ledger/cmd/internal/import/qif" +) + +//go:embed sample.qif +var qifSample []byte + +func TestParseQIF(t *testing.T) { + entries, err := qif.ParseQIF(bytes.NewBuffer(qifSample)) + if err != nil { + t.Fatal(err) + } + + if len(entries) != 3 { + t.Fatalf("Expected 3 entries, got %d", len(entries)) + } + + tests := []struct { + index int + typ string + date string + amount string + payee string + memo string + cat string + splitCt string + splitAm string + }{ + { + index: 0, + typ: "Cash", + date: "08/14/2024", + amount: "15.00", + payee: "", + memo: "~@~CLD:1723446000~@~", + cat: "Bank Deposit to PP Account ", + splitCt: "Bank Deposit to PP Account ", + splitAm: "15.00", + }, + { + index: 1, + typ: "Cash", + date: "08/14/2024", + amount: "-15.00", + payee: "9171-5573 Quebec Inc", + memo: "VOIPMS15", + cat: "PreApproved Payment Bill User Payment", + splitCt: "PreApproved Payment Bill User Payment", + splitAm: "-15.00", + }, + { + index: 2, + typ: "Cash", + date: "08/27/2024", + amount: "80.00", + payee: "", + memo: "", + cat: "Bank Deposit to PP Account ", + splitCt: "Bank Deposit to PP Account ", + splitAm: "80.00", + }, + } + + for _, tt := range tests { + if tt.index >= len(entries) { + t.Fatalf("test index %d out of range, len(entries)=%d", tt.index, len(entries)) + } + e := entries[tt.index] + + if e.Type != tt.typ { + t.Errorf("entry %d: expected Type %q, got %q", tt.index, tt.typ, e.Type) + } + if e.Date != tt.date { + t.Errorf("entry %d: expected Date %q, got %q", tt.index, tt.date, e.Date) + } + if e.Amount != tt.amount { + t.Errorf("entry %d: expected Amount %q, got %q", tt.index, tt.amount, e.Amount) + } + if e.Payee != tt.payee { + t.Errorf("entry %d: expected Payee %q, got %q", tt.index, tt.payee, e.Payee) + } + if e.Memo != tt.memo { + t.Errorf("entry %d: expected Memo %q, got %q", tt.index, tt.memo, e.Memo) + } + if e.Category != tt.cat { + t.Errorf("entry %d: expected Category %q, got %q", tt.index, tt.cat, e.Category) + } + if e.SplitCategory != tt.splitCt { + t.Errorf("entry %d: expected SplitCategory %q, got %q", tt.index, tt.splitCt, e.SplitCategory) + } + if e.SplitAmount != tt.splitAm { + t.Errorf("entry %d: expected SplitAmount %q, got %q", tt.index, tt.splitAm, e.SplitAmount) + } + } +} diff --git a/ledger/cmd/internal/import/qif/sample.qif b/ledger/cmd/internal/import/qif/sample.qif new file mode 100644 index 0000000..0c35604 --- /dev/null +++ b/ledger/cmd/internal/import/qif/sample.qif @@ -0,0 +1,35 @@ +!Type:Cash +D08/14/2024 +T15.00 +LBank Deposit to PP Account +SBank Deposit to PP Account +$15.00 +SFee +$0.00 +CX +M~@~CLD:1723446000~@~ +P +^ +!Type:Cash +D08/14/2024 +T-15.00 +LPreApproved Payment Bill User Payment +SPreApproved Payment Bill User Payment +$-15.00 +SFee +$0.00 +CX +MVOIPMS15 +P9171-5573 Quebec Inc +^ +!Type:Cash +D08/27/2024 +T80.00 +LBank Deposit to PP Account +SBank Deposit to PP Account +$80.00 +SFee +$0.00 +CX +P +^ From 53e8ebf1227a4861441cdd9daa0585dd5f328e44 Mon Sep 17 00:00:00 2001 From: Aidan <6690599+aodhan-domhnaill@users.noreply.github.com> Date: Fri, 20 Feb 2026 19:46:12 +0200 Subject: [PATCH 11/14] Optional predict import (#8) * reducing duplicate code * ledger optional on import --------- Co-authored-by: Aidan Macdonald --- ledger/cmd/import.go | 220 ++++++++++++++++---------------------- ledger/cmd/import_test.go | 3 +- 2 files changed, 95 insertions(+), 128 deletions(-) diff --git a/ledger/cmd/import.go b/ledger/cmd/import.go index 75cd91d..9225777 100644 --- a/ledger/cmd/import.go +++ b/ledger/cmd/import.go @@ -30,8 +30,59 @@ var fieldDelimiter string var scaleFactor float64 var overrideCurrency string -func trainClassifier(generalLedger []*ledger.Transaction, matchingAccount string) *bayesian.Classifier { - allAccounts := ledger.GetBalances(generalLedger, []string{}) +type Importer struct { + filename string + reader *os.File + decScale decimal.Decimal + matchingAccount string + generalLedger []*ledger.Transaction + classifier *bayesian.Classifier +} + +func NewImporter(accountSubstring, filename string) *Importer { + imp := Importer{ + filename: filename, + decScale: decimal.NewFromFloat(scaleFactor), + } + + fileReader, err := os.Open(filename) + if err != nil { + fmt.Println("CSV: ", err) + return nil + } + imp.reader = fileReader + + // If a ledger file path is provided, load it and train the classifier. + // Otherwise, skip loading and prediction will fall back to "unknown:unknown". + if ledgerFilePath != "" { + generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) + if parseError != nil { + fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) + return nil + } + imp.generalLedger = generalLedger + + matchingAccount, err := imp.findMatchingAccount(accountSubstring) + if err != nil { + fmt.Println(err) + return nil + } + imp.matchingAccount = matchingAccount + + imp.classifier = imp.trainClassifier(imp.matchingAccount) + } else { + imp.matchingAccount = accountSubstring + } + + return &imp +} + +func (imp *Importer) Close() { + imp.reader.Close() +} + +func (imp *Importer) trainClassifier(matchingAccount string) *bayesian.Classifier { + allAccounts := ledger.GetBalances(imp.generalLedger, []string{}) uniqueAccounts := make(map[string]bool) for _, acc := range allAccounts { if ok, _ := uniqueAccounts[acc.Name]; !ok { @@ -45,7 +96,7 @@ func trainClassifier(generalLedger []*ledger.Transaction, matchingAccount string } classifier := bayesian.NewClassifier(classes...) - for _, tran := range generalLedger { + for _, tran := range imp.generalLedger { payeeWords := strings.Fields(tran.Payee) // learn accounts names (except matchingAccount) for transactions where matchingAccount is present learnName := false @@ -67,14 +118,18 @@ func trainClassifier(generalLedger []*ledger.Transaction, matchingAccount string return classifier } -func predictAccount(classifier *bayesian.Classifier, inputPayeeWords []string) string { +func (imp *Importer) predictAccount(inputPayeeWords []string) string { + if imp.classifier == nil { + return "unknown:unknown" + } + // Classify into expense account // Find the highest and second highest scores highScore1 := math.Inf(-1) highScore2 := math.Inf(-1) matchIdx := 0 - scores, _, _ := classifier.LogScores(inputPayeeWords) + scores, _, _ := imp.classifier.LogScores(inputPayeeWords) for j, score := range scores { if score > highScore1 { highScore2 = highScore1 @@ -85,15 +140,15 @@ func predictAccount(classifier *bayesian.Classifier, inputPayeeWords []string) s // If the difference between the highest and second highest scores is greater than 10 // then it indicates that highscore is a high confidence match if highScore1-highScore2 > 10 { - return string(classifier.Classes[matchIdx]) + return string(imp.classifier.Classes[matchIdx]) } else { return "unknown:unknown" } } -func findMatchingAccount(generalLedger []*ledger.Transaction, accountSubstring string) (string, error) { +func (imp *Importer) findMatchingAccount(accountSubstring string) (string, error) { var matchingAccount string - matchingAccounts := ledger.GetBalances(generalLedger, []string{accountSubstring}) + matchingAccounts := ledger.GetBalances(imp.generalLedger, []string{accountSubstring}) if len(matchingAccounts) < 1 { return "", ErrNoMatchingAccount } @@ -110,29 +165,8 @@ func findMatchingAccount(generalLedger []*ledger.Transaction, accountSubstring s return matchingAccount, nil } -func importCSV(accountSubstring, csvFileName string) { - decScale := decimal.NewFromFloat(scaleFactor) - - csvFileReader, err := os.Open(csvFileName) - if err != nil { - fmt.Println("CSV: ", err) - return - } - defer csvFileReader.Close() - - generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) - if parseError != nil { - fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) - return - } - - matchingAccount, err := findMatchingAccount(generalLedger, accountSubstring) - if err != nil { - fmt.Println(err) - return - } - - csvReader := csv.NewReader(csvFileReader) +func (imp *Importer) importCSV() { + csvReader := csv.NewReader(imp.reader) csvReader.Comma, _ = utf8.DecodeRuneInString(fieldDelimiter) csvRecords, cerr := csvReader.ReadAll() if cerr != nil { @@ -140,8 +174,6 @@ func importCSV(accountSubstring, csvFileName string) { return } - classifier := trainClassifier(generalLedger, matchingAccount) - // Find columns from header var dateColumn, payeeColumn, amountColumn, commentColumn int dateColumn, payeeColumn, amountColumn, commentColumn = -1, -1, -1, -1 @@ -170,12 +202,12 @@ func importCSV(accountSubstring, csvFileName string) { } expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} - csvAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero} + csvAccount := ledger.Account{Name: imp.matchingAccount, Balance: decimal.Zero} for _, record := range csvRecords[1:] { inputPayeeWords := strings.Fields(record[payeeColumn]) csvDate, _ := time.Parse(csvDateFormat, record[dateColumn]) - if allowMatching || !existingTransaction(generalLedger, csvDate, record[payeeColumn]) { - expenseAccount.Name = predictAccount(classifier, inputPayeeWords) + if allowMatching || !imp.existingTransaction(csvDate, record[payeeColumn]) { + expenseAccount.Name = imp.predictAccount(inputPayeeWords) // Parse error, set to zero if dec, derr := decimal.NewFromString(record[amountColumn]); derr != nil { @@ -190,7 +222,7 @@ func importCSV(accountSubstring, csvFileName string) { } // Apply scale - expenseAccount.Balance = expenseAccount.Balance.Mul(decScale) + expenseAccount.Balance = expenseAccount.Balance.Mul(imp.decScale) // Csv amount is the negative of the expense amount csvAccount.Balance = expenseAccount.Balance.Neg() @@ -211,38 +243,15 @@ func importCSV(accountSubstring, csvFileName string) { } } -func importCamt(accountSubstring, camtFileName string) { - decScale := decimal.NewFromFloat(scaleFactor) - - fileReader, err := os.Open(camtFileName) - if err != nil { - fmt.Println("CAMT: ", err, camtFileName) - return - } - defer fileReader.Close() - - generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) - if parseError != nil { - fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) - return - } - - matchingAccount, err := findMatchingAccount(generalLedger, accountSubstring) - if err != nil { - fmt.Println(err) - return - } - - classifier := trainClassifier(generalLedger, matchingAccount) - - entries, err := camt.ParseCamt(fileReader) +func (imp *Importer) importCamt() { + entries, err := camt.ParseCamt(imp.reader) if err != nil { fmt.Println("CAMT parse error:", err.Error()) return } expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} - camtAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero} + camtAccount := ledger.Account{Name: imp.matchingAccount, Balance: decimal.Zero} for _, entry := range entries { dateTime, err := time.Parse(time.RFC3339, entry.BookgDt.DtTm) if err != nil { @@ -272,7 +281,7 @@ func importCamt(accountSubstring, camtFileName string) { } inputPayeeWords := strings.Fields(payee) - expenseAccount.Name = predictAccount(classifier, inputPayeeWords) + expenseAccount.Name = imp.predictAccount(inputPayeeWords) expenseAccount.Balance = amount // Determine if debit @@ -282,7 +291,7 @@ func importCamt(accountSubstring, camtFileName string) { } // Apply scale - expenseAccount.Balance = expenseAccount.Balance.Mul(decScale) + expenseAccount.Balance = expenseAccount.Balance.Mul(imp.decScale) // Csv amount is the negative of the expense amount camtAccount.Balance = expenseAccount.Balance.Neg() @@ -305,38 +314,15 @@ func importCamt(accountSubstring, camtFileName string) { } } -func importQIF(accountSubstring, qifFileName string) { - decScale := decimal.NewFromFloat(scaleFactor) - - fileReader, err := os.Open(qifFileName) - if err != nil { - fmt.Println("QIF: ", err, qifFileName) - return - } - defer fileReader.Close() - - generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) - if parseError != nil { - fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) - return - } - - matchingAccount, err := findMatchingAccount(generalLedger, accountSubstring) - if err != nil { - fmt.Println(err) - return - } - - classifier := trainClassifier(generalLedger, matchingAccount) - - entries, err := qif.ParseQIF(fileReader) +func (imp *Importer) importQIF() { + entries, err := qif.ParseQIF(imp.reader) if err != nil { fmt.Println("QIF parse error:", err.Error()) return } expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} - qifAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero} + qifAccount := ledger.Account{Name: imp.matchingAccount, Balance: decimal.Zero} for _, entry := range entries { // Parse date (QIF dates are often locale-specific; assume mm/dd/yyyy here) dateTime, err := time.Parse("01/02/2006", entry.Date) @@ -359,11 +345,11 @@ func importQIF(accountSubstring, qifFileName string) { payee := entry.Payee inputPayeeWords := strings.Fields(payee) - expenseAccount.Name = predictAccount(classifier, inputPayeeWords) + expenseAccount.Name = imp.predictAccount(inputPayeeWords) expenseAccount.Balance = amount // Apply scale - expenseAccount.Balance = expenseAccount.Balance.Mul(decScale) + expenseAccount.Balance = expenseAccount.Balance.Mul(imp.decScale) // Account side is the opposite of expense qifAccount.Balance = expenseAccount.Balance.Neg() @@ -384,38 +370,15 @@ func importQIF(accountSubstring, qifFileName string) { } } -func importQFX(accountSubstring, qfxFileName string) { - decScale := decimal.NewFromFloat(scaleFactor) - - fileReader, err := os.Open(qfxFileName) - if err != nil { - fmt.Println("QFX: ", err, qfxFileName) - return - } - defer fileReader.Close() - - generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) - if parseError != nil { - fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) - return - } - - matchingAccount, err := findMatchingAccount(generalLedger, accountSubstring) - if err != nil { - fmt.Println(err) - return - } - - classifier := trainClassifier(generalLedger, matchingAccount) - - entries, err := qfx.ParseQFX(fileReader) +func (imp *Importer) importQFX() { + entries, err := qfx.ParseQFX(imp.reader) if err != nil { fmt.Println("QFX parse error:", err.Error()) return } expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} - qfxAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero} + qfxAccount := ledger.Account{Name: imp.matchingAccount, Balance: decimal.Zero} for _, entry := range entries { // QFX DTPOSTED is typically YYYYMMDDHHMMSS.XXX; we only care about the date. // Take the first 8 characters as YYYYMMDD. @@ -439,11 +402,11 @@ func importQFX(accountSubstring, qfxFileName string) { payee := entry.Memo inputPayeeWords := strings.Fields(payee) - expenseAccount.Name = predictAccount(classifier, inputPayeeWords) + expenseAccount.Name = imp.predictAccount(inputPayeeWords) expenseAccount.Balance = amount // Apply scale - expenseAccount.Balance = expenseAccount.Balance.Mul(decScale) + expenseAccount.Balance = expenseAccount.Balance.Mul(imp.decScale) // Account side is the opposite of expense qfxAccount.Balance = expenseAccount.Balance.Neg() @@ -471,15 +434,18 @@ var importCmd = &cobra.Command{ accountSubstring := args[0] fileName := args[1] + imp := NewImporter(accountSubstring, fileName) + defer imp.Close() + lower := strings.ToLower(fileName) if strings.HasSuffix(lower, ".xml") { - importCamt(accountSubstring, fileName) + imp.importCamt() } else if strings.HasSuffix(lower, ".qfx") || strings.HasSuffix(lower, ".ofx") { - importQFX(accountSubstring, fileName) + imp.importQFX() } else if strings.HasSuffix(lower, ".qif") { - importQIF(accountSubstring, fileName) + imp.importQIF() } else { - importCSV(accountSubstring, fileName) + imp.importCSV() } }, @@ -496,8 +462,8 @@ func init() { importCmd.Flags().StringVar(&overrideCurrency, "override-currency", "", "Override detected currency for imported transactions.") } -func existingTransaction(generalLedger []*ledger.Transaction, transDate time.Time, payee string) bool { - for _, trans := range generalLedger { +func (imp *Importer) existingTransaction(transDate time.Time, payee string) bool { + for _, trans := range imp.generalLedger { if trans.Date == transDate && strings.TrimSpace(trans.Payee) == strings.TrimSpace(payee) { return true } diff --git a/ledger/cmd/import_test.go b/ledger/cmd/import_test.go index b78105c..8ce2bc8 100644 --- a/ledger/cmd/import_test.go +++ b/ledger/cmd/import_test.go @@ -32,7 +32,8 @@ func Test_findMatchingAccount(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, gotErr := findMatchingAccount(tt.generalLedger, tt.accountSubstring) + imp := &Importer{generalLedger: tt.generalLedger} + got, gotErr := imp.findMatchingAccount(tt.accountSubstring) if gotErr != nil { if !tt.wantErr { t.Errorf("findMatchingAccount() failed: %v", gotErr) From aa292f70da0407a885071e1a228acdc4a5958a90 Mon Sep 17 00:00:00 2001 From: Aidan <6690599+aodhan-domhnaill@users.noreply.github.com> Date: Sat, 21 Feb 2026 15:31:09 +0200 Subject: [PATCH 12/14] IIF support (tested on Paypal data) (#9) * iif format header parsing works * record parsing working * cleaner parser * refactor: use type tag instead of iff in iif_trns.go * iif parser into transactions passes tests * iif works. tested on export from paypal --------- Co-authored-by: Aidan Macdonald --- ledger/cmd/import.go | 53 +++++ .../internal/import/iif/Full Bill payment.iif | 11 + .../cmd/internal/import/iif/Full Deposit.iif | 15 ++ .../cmd/internal/import/iif/Full Invoice.iif | 24 +++ .../import/iif/Full Sales Tax Payment.iif | 13 ++ .../cmd/internal/import/iif/Full Transfer.iif | 9 + ledger/cmd/internal/import/iif/iif.go | 188 ++++++++++++++++++ ledger/cmd/internal/import/iif/iif_test.go | 151 ++++++++++++++ ledger/cmd/internal/import/iif/iif_trns.go | 167 ++++++++++++++++ .../cmd/internal/import/iif/iif_trns_test.go | 115 +++++++++++ 10 files changed, 746 insertions(+) create mode 100644 ledger/cmd/internal/import/iif/Full Bill payment.iif create mode 100644 ledger/cmd/internal/import/iif/Full Deposit.iif create mode 100644 ledger/cmd/internal/import/iif/Full Invoice.iif create mode 100644 ledger/cmd/internal/import/iif/Full Sales Tax Payment.iif create mode 100644 ledger/cmd/internal/import/iif/Full Transfer.iif create mode 100644 ledger/cmd/internal/import/iif/iif.go create mode 100644 ledger/cmd/internal/import/iif/iif_test.go create mode 100644 ledger/cmd/internal/import/iif/iif_trns.go create mode 100644 ledger/cmd/internal/import/iif/iif_trns_test.go diff --git a/ledger/cmd/import.go b/ledger/cmd/import.go index 9225777..3b69595 100644 --- a/ledger/cmd/import.go +++ b/ledger/cmd/import.go @@ -4,6 +4,7 @@ import ( "encoding/csv" "errors" "fmt" + "log" "math" "os" "strings" @@ -12,6 +13,7 @@ import ( "github.com/howeyc/ledger" "github.com/howeyc/ledger/ledger/cmd/internal/import/camt" + "github.com/howeyc/ledger/ledger/cmd/internal/import/iif" "github.com/howeyc/ledger/ledger/cmd/internal/import/qfx" "github.com/howeyc/ledger/ledger/cmd/internal/import/qif" "github.com/jbrukh/bayesian" @@ -370,6 +372,55 @@ func (imp *Importer) importQIF() { } } +func (imp *Importer) importIIF() { + f, err := iif.NewDecoder(imp.reader).Decode() + if err != nil { + log.Fatal(err) + return + } + + tx := []iif.Transaction{} + for _, b := range f.Blocks { + tr, err := iif.DeserializeTransactions(b) + if err != nil { + log.Fatal(err) + return + } + tx = append(tx, tr...) + } + + for _, itx := range tx { + trans := &ledger.Transaction{ + Date: itx.Tr.Date, + Payee: itx.Tr.Class + " " + itx.Tr.Memo, + } + trans.AccountChanges = []ledger.Account{ + { + Name: itx.Tr.Account, + Balance: itx.Tr.Amount, + }, + } + + for _, split := range itx.Splits { + trans.AccountChanges = append( + trans.AccountChanges, + ledger.Account{ + Name: split.Account, + Balance: split.Amount, + }, + ) + } + + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } + WriteTransaction(os.Stdout, trans, 80) + } + +} + func (imp *Importer) importQFX() { entries, err := qfx.ParseQFX(imp.reader) if err != nil { @@ -444,6 +495,8 @@ var importCmd = &cobra.Command{ imp.importQFX() } else if strings.HasSuffix(lower, ".qif") { imp.importQIF() + } else if strings.HasSuffix(lower, ".iif") { + imp.importIIF() } else { imp.importCSV() } diff --git a/ledger/cmd/internal/import/iif/Full Bill payment.iif b/ledger/cmd/internal/import/iif/Full Bill payment.iif new file mode 100644 index 0000000..a90d629 --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Bill payment.iif @@ -0,0 +1,11 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Checking BANK +ACCNT Accounts Payable AP 2000 +!VEND NAME REFNUM PRINTAS ADDR1 ADDR2 ADDR3 ADDR4 ADDR5 VTYPE CONT1 CONT2 PHONE1 PHONE2 FAXNUM EMAIL NOTE TAXID LIMIT TERMS NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +VEND Vendor 1 Jon Vendor 555 Street St "Anywhere, AZ 85730" USA Jon Vendor 5555555555 Jon Vendor +!TRNS TRNSID TRNSTYPE DATE ACCNT NAME AMOUNT DOCNUM MEMO CLEAR TOPRINT +!SPL SPLID TRNSTYPE DATE ACCNT NAME AMOUNT DOCNUM MEMO CLEAR QNTY +!ENDTRNS +TRNS BILLPMT 7/16/1998 Checking Vendor -35 Test Memo N Y +SPL BILLPMT 7/16/1998 Accounts Payable Vendor 35 N +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/Full Deposit.iif b/ledger/cmd/internal/import/iif/Full Deposit.iif new file mode 100644 index 0000000..cd01b6f --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Deposit.iif @@ -0,0 +1,15 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Checking BANK +ACCNT Income INC +!CLASS NAME +CLASS class +!CUST NAME BADDR1 BADDR2 BADDR3 BADDR4 BADDR5 SADDR1 SADDR2 SADDR3 SADDR4 SADDR5 PHONE1 PHONE2 FAXNUM EMAIL NOTE CONT1 CONT2 CTYPE TERMS TAXABLE LIMIT RESALENUM REP TAXITEM NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +CUST Customer Joe Customer 444 Road Rd "Anywhere, AZ 85740" USA 5554443333 Joe Customer N Joe Customer +!OTHERNAME NAME BADDR1 BADDR2 BADDR3 BADDR4 BADDR5 PHONE1 PHONE2 FAXNUM EMAIL NOTE CONT1 CONT2 NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +OTHERNAME Other Name Other Name 123 a Street "Somewhere, AZ 85730" USA 5555555555 Other Name Other Name +!TRNS TRNSID TRNSTYPE DATE ACCNT NAME CLASS AMOUNT DOCNUM MEMO CLEAR +!SPL SPLID TRNSTYPE DATE ACCNT NAME CLASS AMOUNT DOCNUM MEMO CLEAR +!ENDTRNS +TRNS DEPOSIT 7/1/1998 Checking 10000 N +SPL DEPOSIT 7/1/1998 Income Customer -10000 N +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/Full Invoice.iif b/ledger/cmd/internal/import/iif/Full Invoice.iif new file mode 100644 index 0000000..2825119 --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Invoice.iif @@ -0,0 +1,24 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Accounts Receivable AR 1200 +ACCNT Construction:Labor INC 4100 +ACCNT Construction:Materials INC 4200 +ACCNT Inventory Asset OCASSET 1120 INVENTORYASSET +ACCNT Cost of Goods Sold COGS Cost of Goods Sold 5000 COGS +!INVITEM NAME INVITEMTYPE DESC PURCHASEDESC ACCNT ASSETACCNT COGSACCNT PRICE COST TAXABLE PAYMETH TAXVEND TAXDIST PREFVEND REORDERPOINT EXTRA +INVITEM Framing SERV Framing labor Construction:Labor 55 0 N +INVITEM Wood Door:Exterior INVENTORY Exterior wood door Exterior door - #P-10981 Construction:Materials Inventory Asset Cost of Goods Sold 120 105 Y Perry Windows & Doors 5 +INVITEM Hardware:Doorknobs Std INVENTORY Standard Doorknobs Doorknobs Part # DK 3704 Construction:Materials Inventory Asset Cost of Goods Sold 30 27 Y Patton Hardware Supplies 50 +!CLASS NAME +CLASS class +!CUST NAME BADDR1 BADDR2 BADDR3 BADDR4 BADDR5 SADDR1 SADDR2 SADDR3 SADDR4 SADDR5 PHONE1 PHONE2 FAXNUM EMAIL NOTE CONT1 CONT2 CTYPE TERMS TAXABLE LIMIT RESALENUM REP TAXITEM NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +CUST Customer Joe Customer 444 Road Rd "Anywhere, AZ 85740" USA 5554443333 Joe Customer N Joe Customer +!VEND NAME PRINTAS ADDR1 ADDR2 ADDR3 ADDR4 ADDR5 VTYPE CONT1 CONT2 PHONE1 PHONE2 FAXNUM EMAIL NOTE TAXID LIMIT TERMS NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +VEND Vendor Jon Vendor 555 Street St "Anywhere, AZ 85730" USA Jon Vendor 5555555555 Jon Vendor +!TRNS TRNSID TRNSTYPE DATE ACCNT NAME CLASS AMOUNT DOCNUM MEMO CLEAR TOPRINT NAMEISTAXABLE ADDR1 ADDR3 TERMS SHIPVIA SHIPDATE +!SPL SPLID TRNSTYPE DATE ACCNT NAME CLASS AMOUNT DOCNUM MEMO CLEAR QNTY PRICE INVITEM TAXABLE OTHER2 YEARTODATE WAGEBASE +!ENDTRNS +TRNS INVOICE 7/18/98 Accounts Receivable Customer 205 1 N Y N 7/16/98 +SPL INVOICE 7/16/98 Construction:Labor -55 Framing labor N 55 Framing N 0 0 +SPL INVOICE 7/16/98 Construction:Materials -120 Exterior wood door N 120 Wood Door:Exterior Y 0 0 +SPL INVOICE 7/16/98 Construction:Materials -30 Standard Doorknobs N 30 Hardware:Doorknobs Std Y 0 0 +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/Full Sales Tax Payment.iif b/ledger/cmd/internal/import/iif/Full Sales Tax Payment.iif new file mode 100644 index 0000000..9d22a5c --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Sales Tax Payment.iif @@ -0,0 +1,13 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Checking BANK +ACCNT Sales Tax Payable OCLIAB 2200 SALESTAX +!INVITEM NAME INVITEMTYPE DESC PURCHASEDESC ACCNT ASSETACCNT COGSACCNT PRICE COST TAXABLE PAYMETH TAXVEND TAXDIST PREFVEND REORDERPOINT EXTRA +INVITEM San Domingo COMPTAX "CA sales tax, San Domingo County" Sales Tax Payable 7.50% 0 N Sales Tax Vendor +!VEND NAME PRINTAS ADDR1 ADDR2 ADDR3 ADDR4 ADDR5 VTYPE CONT1 CONT2 PHONE1 PHONE2 FAXNUM EMAIL NOTE TAXID LIMIT TERMS NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +VEND Sales Tax Vendor Jon Vendor 555 Street St "Anywhere, AZ 85730" USA Jon Vendor 5555555555 Jon Vendor +!TRNS TRNSID TRNSTYPE DATE ACCNT NAME AMOUNT DOCNUM CLEAR TOPRINT ADDR1 +!SPL SPLID TRNSTYPE DATE ACCNT NAME AMOUNT DOCNUM CLEAR QNTY INVITEM +!ENDTRNS +TRNS SALESTAXPMT 8/5/98 Checking Sales Tax Vendor -66.57 N Y Jon Vendor +SPL SALESTAXPMT 8/5/98 Sales Tax Payable Sales Tax Vendor 66.57 N San Domingo +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/Full Transfer.iif b/ledger/cmd/internal/import/iif/Full Transfer.iif new file mode 100644 index 0000000..1b01952 --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Transfer.iif @@ -0,0 +1,9 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Checking BANK +ACCNT Savings BANK +!TRNS TRNSID TRNSTYPE DATE ACCNT AMOUNT MEMO CLEAR +!SPL SPLID TRNSTYPE DATE ACCNT AMOUNT MEMO CLEAR +!ENDTRNS +TRNS TRANSFER 7/1/98 Checking -500 Funds Transfer N +SPL TRANSFER 7/1/98 Savings 500 N +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/iif.go b/ledger/cmd/internal/import/iif/iif.go new file mode 100644 index 0000000..92d41e7 --- /dev/null +++ b/ledger/cmd/internal/import/iif/iif.go @@ -0,0 +1,188 @@ +package iif + +import ( + "encoding/csv" + "errors" + "io" + "strings" +) + +var ( + ErrInvalidHeaderLine = errors.New("iif: invalid header line") + ErrMismatchedColumns = errors.New("iif: mismatched number of columns") + ErrMismatchedRecords = errors.New("iif: row does not match expected header") + ErrUnknownRecordType = errors.New("iif: unknown record type") + ErrUnexpectedSectionType = errors.New("iif: unexpected record type for current section") + ErrEmptyHeader = errors.New("iif: empty header") +) + +type RecordType string + +type Header struct { + Type RecordType + Fields []string +} + +type Record struct { + Type RecordType + Fields map[string]string +} + +type Block struct { + Records [][]Record + Headers []Header +} + +type File struct { + Blocks []Block +} + +type Decoder struct { + r *csv.Reader + err error + IsHeader bool + Type RecordType + Fields []string +} + +func NewDecoder(r io.Reader) *Decoder { + reader := csv.NewReader(r) + reader.Comma = '\t' + reader.LazyQuotes = true + reader.TrimLeadingSpace = false + reader.FieldsPerRecord = -1 + d := Decoder{r: reader} + d.Next() + return &d +} + +func (d *Decoder) Next() { + line, err := d.r.Read() + d.err = err + if err == nil { + d.IsHeader = strings.HasPrefix(line[0], "!") + if d.IsHeader { + d.Type = RecordType(line[0][1:]) + } else { + d.Type = RecordType(line[0]) + } + d.Fields = line[1:] + } +} + +func (d *Decoder) Error() error { + if d.err != io.EOF { + return d.err + } + return nil +} + +func (d *Decoder) Done() bool { + return d.err != nil +} + +func (f *File) Load(d *Decoder) error { + for !d.Done() { + if d.Error() != nil { + return d.Error() + } + b := Block{} + err := b.Load(d) + if err != nil { + return err + } + f.Blocks = append(f.Blocks, b) + } + return nil +} + +func (h Header) MapFields(fields []string) map[string]string { + m := make(map[string]string, len(fields)) + for i, f := range h.Fields { + if i >= len(fields) { + break + } + m[f] = fields[i] + } + return m +} + +func (b *Block) Load(d *Decoder) error { + if d.Done() { + return d.Error() + } + // Parse Headers + for !d.Done() && d.IsHeader { + b.Headers = append( + b.Headers, + Header{ + Type: RecordType(d.Type), + Fields: trimLine(d.Fields), + }, + ) + d.Next() + } + if d.Error() != nil { + return d.Error() + } + + // Parse Records + for !d.Done() && !d.IsHeader { + r := []Record{} + // At least one record per header + if len(b.Headers) == 0 { + return ErrEmptyHeader + } + for _, h := range b.Headers { + if d.Done() { + return d.Error() + } + if d.Done() || d.Type != h.Type { + return ErrMismatchedRecords + } + + for !d.Done() && !d.IsHeader && d.Type == h.Type { + r = append(r, Record{ + Type: d.Type, + Fields: h.MapFields(d.Fields), + }) + d.Next() + } + if len(r) == 0 { + return ErrMismatchedRecords + } + } + b.Records = append(b.Records, r) + } + return nil +} + +func trimLine(records []string) []string { + for i, r := range records { + if r == "" { + return records[:i] + } + } + return records +} + +func (d *Decoder) Decode() (*File, error) { + f := File{} + err := f.Load(d) + if err != nil && err != io.EOF { + return nil, err + } + return &f, nil +} + +type Encoder struct { + w io.Writer +} + +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +func (e *Encoder) Encode(f *File) error { + return errors.New("iif encoding not implemented") +} diff --git a/ledger/cmd/internal/import/iif/iif_test.go b/ledger/cmd/internal/import/iif/iif_test.go new file mode 100644 index 0000000..aecda94 --- /dev/null +++ b/ledger/cmd/internal/import/iif/iif_test.go @@ -0,0 +1,151 @@ +package iif_test + +import ( + "bytes" + "reflect" + "testing" + + _ "embed" + + "github.com/howeyc/ledger/ledger/cmd/internal/import/iif" +) + +var ( + //go:embed "Full Deposit.iif" + fullDepositIIF []byte + + //go:embed "Full Invoice.iif" + fullInvoiceIIF []byte + + //go:embed "Full Bill payment.iif" + fullBillPaymentIIF []byte + + //go:embed "Full Sales Tax Payment.iif" + fullSalesTaxPaymentIIF []byte + + //go:embed "Full Transfer.iif" + fullTransferIIF []byte +) + +func TestDecodeEncode(t *testing.T) { + tests := []struct { + name string + data []byte + blocks []iif.Block + }{ + { + name: "fullDepositIIF", + data: fullDepositIIF, + blocks: []iif.Block{ + { + Headers: []iif.Header{ + {Type: iif.RecordType("ACCNT"), Fields: []string{"NAME", "ACCNTTYPE", "DESC", "ACCNUM", "EXTRA"}}, + }, + }, + { + Headers: []iif.Header{ + {Type: iif.RecordType("CLASS"), Fields: []string{"NAME"}}, + }, + }, + { + Headers: []iif.Header{ + {Type: iif.RecordType("CUST"), Fields: []string{"NAME", "BADDR1", "BADDR2", "BADDR3", "BADDR4", "BADDR5", "SADDR1"}}, + }, + }, + { + Headers: []iif.Header{ + {Type: iif.RecordType("OTHERNAME"), Fields: []string{"NAME", "BADDR1", "BADDR2", "BADDR3", "BADDR4", "BADDR5", "PHONE1", "PHONE2", "FAXNUM", "EMAIL", "NOTE", "CONT1", "CONT2", "NOTEPAD", "SALUTATION", "COMPANYNAME", "FIRSTNAME", "MIDINIT", "LASTNAME"}}, + }, + }, + { + Headers: []iif.Header{ + {Type: iif.RecordType("TRNS"), Fields: []string{"TRNSID", "TRNSTYPE", "DATE", "ACCNT", "NAME", "CLASS", "AMOUNT", "DOCNUM", "MEMO", "CLEAR"}}, + {Type: iif.RecordType("SPL"), Fields: []string{"SPLID", "TRNSTYPE", "DATE", "ACCNT", "NAME", "CLASS", "AMOUNT", "DOCNUM", "MEMO", "CLEAR"}}, + {Type: iif.RecordType("ENDTRNS"), Fields: []string{}}, + }, + Records: [][]iif.Record{ + { + { + Type: iif.RecordType("TRNS"), + Fields: map[string]string{ + "TRNSID": " ", + "TRNSTYPE": "DEPOSIT", + "DATE": "7/1/1998", + "ACCNT": "Checking", + "NAME": "", + "CLASS": "", + "AMOUNT": "10000", + "DOCNUM": "", + "MEMO": "", + "CLEAR": "N", + }, + }, + { + Type: iif.RecordType("SPL"), + Fields: map[string]string{ + "SPLID": "", + "TRNSTYPE": "DEPOSIT", + "DATE": "7/1/1998", + "ACCNT": "Income", + "NAME": "Customer", + "CLASS": "", + "AMOUNT": "-10000", + "DOCNUM": "", + "MEMO": "", + "CLEAR": "N", + }, + }, + { + Type: iif.RecordType("ENDTRNS"), + Fields: map[string]string{}, + }, + }, + }, + }, + }, + }, + { + name: "fullInvoiceIIF", + data: fullInvoiceIIF, + }, + { + name: "fullBillPaymentIIF", + data: fullBillPaymentIIF, + }, + { + name: "fullSalesTaxPaymentIIF", + data: fullSalesTaxPaymentIIF, + }, + { + name: "fullTransferIIF", + data: fullTransferIIF, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dec := iif.NewDecoder(bytes.NewReader(tt.data)) + f, err := dec.Decode() + if err != nil { + t.Fatalf("Decode error: %v", err) + } + + if len(f.Blocks) == 0 { + t.Error("missing blocks from file") + } + + for i, b := range tt.blocks { + if i >= len(f.Blocks) { + t.Errorf("expected at least %d blocks, got %d", len(tt.blocks), len(f.Blocks)) + break + } + if !reflect.DeepEqual(b.Headers, f.Blocks[i].Headers) { + t.Errorf("expected headers to equal %+v != %+v", b.Headers, f.Blocks[i].Headers) + } + if b.Records != nil && !reflect.DeepEqual(b.Records, f.Blocks[i].Records) { + t.Errorf("expected records to equal %+v != %+v", b.Records, f.Blocks[i].Records) + } + } + }) + } +} diff --git a/ledger/cmd/internal/import/iif/iif_trns.go b/ledger/cmd/internal/import/iif/iif_trns.go new file mode 100644 index 0000000..ef79a55 --- /dev/null +++ b/ledger/cmd/internal/import/iif/iif_trns.go @@ -0,0 +1,167 @@ +package iif + +import ( + "fmt" + "reflect" + "time" + + "github.com/shopspring/decimal" +) + +type Transaction struct { + Tr Trns `type:"TRNS"` + Splits []Spl `type:"SPL"` +} + +type Trns struct { + TransactionType string `iif:"TRNSTYPE"` + Date time.Time `iif:"DATE"` + Account string `iif:"ACCNT"` + Name string `iif:"NAME"` + Class string `iif:"CLASS"` + Amount decimal.Decimal `iif:"AMOUNT"` + Memo string `iif:"MEMO"` +} + +type Spl struct { + TransactionType string `iif:"TRNSTYPE"` + Date time.Time `iif:"DATE"` + Account string `iif:"ACCNT"` + Name string `iif:"NAME"` + Class string `iif:"CLASS"` + Amount decimal.Decimal `iif:"AMOUNT"` + Memo string `iif:"MEMO"` +} + +func DeserializeTransactions(b Block) ([]Transaction, error) { + var out []Transaction + + for _, recGroup := range b.Records { + if len(recGroup) == 0 { + continue + } + + var tx Transaction + if err := DeserializeRecordGroup(&tx, recGroup); err != nil { + return nil, err + } + out = append(out, tx) + } + + return out, nil +} + +func DeserializeRecordGroup(tx any, recs []Record) error { + for _, r := range recs { + if err := applyRecord(tx, r); err != nil { + return err + } + } + return nil +} + +func applyRecord(tx any, r Record) error { + txVal := reflect.ValueOf(tx).Elem() + txType := txVal.Type() + + for i := 0; i < txType.NumField(); i++ { + field := txType.Field(i) + tag := field.Tag.Get("type") + if tag == "" || string(r.Type) != tag { + continue + } + + fv := txVal.Field(i) + + if fv.Kind() == reflect.Slice { + elemType := fv.Type().Elem() + elemPtr := reflect.New(elemType).Elem() + + if err := populateStructFromRecord(elemPtr, r); err != nil { + return err + } + + fv.Set(reflect.Append(fv, elemPtr)) + return nil + } + if fv.Kind() == reflect.Struct { + if err := populateStructFromRecord(fv, r); err != nil { + return err + } + return nil + } + } + return nil +} + +func populateStructFromRecord(v reflect.Value, r Record) error { + if v.Kind() != reflect.Struct { + return fmt.Errorf("populateStructFromRecord: expected struct, got %s", v.Kind()) + } + + t := v.Type() + for i := 0; i < t.NumField(); i++ { + sf := t.Field(i) + tag := sf.Tag.Get("iif") + if tag == "" { + continue + } + + raw, ok := r.Fields[tag] + if !ok { + continue + } + + fv := v.Field(i) + if !fv.CanSet() { + continue + } + + if err := setFieldValueFromString(fv, raw); err != nil { + return fmt.Errorf("field %s: %w", sf.Name, err) + } + } + + return nil +} + +// setFieldValueFromString converts the string representation from a Record +// into the appropriate Go type and assigns it to fv. +func setFieldValueFromString(fv reflect.Value, s string) error { + switch fv.Kind() { + case reflect.String: + fv.SetString(s) + return nil + case reflect.Struct: + // Handle known struct types (time.Time, decimal.Decimal, etc.) + switch fv.Type() { + case reflect.TypeOf(time.Time{}): + // IIF date formats can vary; here we assume the standard + // QuickBooks IIF date "MM/DD/YYYY". Adjust if needed. + if s == "" { + return nil + } + t, err := time.Parse("1/2/2006", s) + if err != nil { + return err + } + fv.Set(reflect.ValueOf(t)) + return nil + case reflect.TypeOf(decimal.Decimal{}): + if s == "" { + fv.Set(reflect.ValueOf(decimal.Zero)) + return nil + } + d, err := decimal.NewFromString(s) + if err != nil { + return err + } + fv.Set(reflect.ValueOf(d)) + return nil + default: + return fmt.Errorf("unsupported struct type %s", fv.Type()) + } + default: + return fmt.Errorf("unsupported kind %s", fv.Kind()) + } +} diff --git a/ledger/cmd/internal/import/iif/iif_trns_test.go b/ledger/cmd/internal/import/iif/iif_trns_test.go new file mode 100644 index 0000000..aeee2c8 --- /dev/null +++ b/ledger/cmd/internal/import/iif/iif_trns_test.go @@ -0,0 +1,115 @@ +package iif_test + +import ( + "reflect" + "testing" + "time" + + "github.com/howeyc/ledger/ledger/cmd/internal/import/iif" + "github.com/shopspring/decimal" +) + +func TestDeserializeTransactions(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + b iif.Block + want []iif.Transaction + wantErr bool + }{ + { + name: "empty", + b: iif.Block{ + Headers: []iif.Header{ + {Type: iif.RecordType("ACCNT"), Fields: []string{"NAME", "ACCNTTYPE", "DESC", "ACCNUM", "EXTRA"}}, + }, + }, + want: nil, + }, + { + name: "simple", + b: iif.Block{ + Headers: []iif.Header{ + {Type: iif.RecordType("TRNS"), Fields: []string{"TRNSID", "TRNSTYPE", "DATE", "ACCNT", "NAME", "CLASS", "AMOUNT", "DOCNUM", "MEMO", "CLEAR"}}, + {Type: iif.RecordType("SPL"), Fields: []string{"SPLID", "TRNSTYPE", "DATE", "ACCNT", "NAME", "CLASS", "AMOUNT", "DOCNUM", "MEMO", "CLEAR"}}, + {Type: iif.RecordType("ENDTRNS"), Fields: []string{}}, + }, + Records: [][]iif.Record{ + { + { + Type: iif.RecordType("TRNS"), + Fields: map[string]string{ + "TRNSID": " ", + "TRNSTYPE": "DEPOSIT", + "DATE": "7/1/1998", + "ACCNT": "Checking", + "NAME": "", + "CLASS": "", + "AMOUNT": "10000", + "DOCNUM": "", + "MEMO": "Hello", + "CLEAR": "N", + }, + }, + { + Type: iif.RecordType("SPL"), + Fields: map[string]string{ + "SPLID": "", + "TRNSTYPE": "DEPOSIT", + "DATE": "7/1/1998", + "ACCNT": "Income", + "NAME": "Customer", + "CLASS": "", + "AMOUNT": "-10000", + "DOCNUM": "", + "MEMO": "", + "CLEAR": "N", + }, + }, + { + Type: iif.RecordType("ENDTRNS"), + Fields: map[string]string{}, + }, + }, + }, + }, + want: []iif.Transaction{ + { + Tr: iif.Trns{ + TransactionType: "DEPOSIT", + Date: time.Date(1998, 7, 1, 0, 0, 0, 0, time.UTC), + Account: "Checking", + Amount: decimal.NewFromInt(10000), + Memo: "Hello", + }, + Splits: []iif.Spl{ + { + TransactionType: "DEPOSIT", + Date: time.Date(1998, 7, 1, 0, 0, 0, 0, time.UTC), + Account: "Income", + Name: "Customer", + Amount: decimal.NewFromInt(-10000), + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := iif.DeserializeTransactions(tt.b) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("DeserializeTransactions() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("DeserializeTransactions() succeeded unexpectedly") + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DeserializeTransactions() = %+v, want %+v", got, tt.want) + } + }) + } +} From febcb048a27d1280cc3909ce05883a67221f7bd2 Mon Sep 17 00:00:00 2001 From: Aidan <6690599+aodhan-domhnaill@users.noreply.github.com> Date: Sun, 22 Feb 2026 16:59:05 +0200 Subject: [PATCH 13/14] Replace unsupported deps (#10) * replacing godate * replacing github.com/alfredxing/calc/compute with github.com/expr-lang/expr * fixing minor bug * adding more linting * fixing linting bugs * linting fixes from go-critic * fixing go version errors in ci * fixing go version errors in ci * migrate ci config * removing golangci * fix go mod * deterministic implicit conversion --------- Co-authored-by: Aidan Macdonald --- .github/workflows/go.yml | 2 +- .gitignore | 1 + go.mod | 10 +++--- go.sum | 24 +++++++++---- ledger/cmd/import.go | 13 +++---- ledger/cmd/internal/httpcompress/compress.go | 4 +-- ledger/cmd/print.go | 6 ++-- parse.go | 37 ++++++++++++++------ parseFuzz_test.go | 7 ++-- parse_test.go | 2 +- transaction.go | 22 ++++++++---- 11 files changed, 85 insertions(+), 43 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 8d6687d..2721fb7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '>=1.20.0' + go-version: '>=1.26' check-latest: true - name: Build diff --git a/.gitignore b/.gitignore index 18ef242..ff8971e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ pkg dist html-book src-book +.* diff --git a/go.mod b/go.mod index 26b485f..19289cd 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,22 @@ module github.com/howeyc/ledger -go 1.24 +go 1.26 + +toolchain go1.26.0 require ( - github.com/alfredxing/calc v0.0.0-20180827002445-77daf576f976 github.com/andybalholm/brotli v1.0.6 + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de + github.com/expr-lang/expr v1.17.8 github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b github.com/ivanpirog/coloredcobra v1.0.1 github.com/jbrukh/bayesian v0.0.0-20200318221351-d726b684ca4a - github.com/joyt/godate v0.0.0-20150226210126-7151572574a7 github.com/juztin/numeronym v0.0.0-20160223091026-859fcc2918e2 github.com/lucasb-eyer/go-colorful v1.2.0 github.com/mattn/go-isatty v0.0.20 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pelletier/go-toml v1.9.5 - github.com/shopspring/decimal v1.3.1 + github.com/shopspring/decimal v1.4.0 github.com/spf13/cobra v1.7.0 golang.org/x/term v0.13.0 golang.org/x/time v0.3.0 diff --git a/go.sum b/go.sum index 3527d84..21c645c 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ -github.com/alfredxing/calc v0.0.0-20180827002445-77daf576f976 h1:+jyVKPjl5Y39thM0ZlVrRqKjSO/Upr5tP9ZQGELv8gw= -github.com/alfredxing/calc v0.0.0-20180827002445-77daf576f976/go.mod h1:/HQknSiD7YKT15DoHXuiXezQfNPBUm8PeqFaTxeA3HU= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM= +github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= @@ -16,8 +20,6 @@ github.com/ivanpirog/coloredcobra v1.0.1 h1:aURSdEmlR90/tSiWS0dMjdwOvCVUeYLfltLf github.com/ivanpirog/coloredcobra v1.0.1/go.mod h1:iho4nEKcnwZFiniGSdcgdvRgZNjxm+h20acv8vqmN6Q= github.com/jbrukh/bayesian v0.0.0-20200318221351-d726b684ca4a h1:gbdjhSslIoRRiSSLCP3kKuLmqAJGmhnPVhIyf6Dbw34= github.com/jbrukh/bayesian v0.0.0-20200318221351-d726b684ca4a/go.mod h1:SELxwZQq/mPnfPCR2mchLmT4TQaPJvYtLcCtDWSM7vM= -github.com/joyt/godate v0.0.0-20150226210126-7151572574a7 h1:2wH5antjhmU3EuWyidm0lJ4B9hGMpl5lNRo+M9uGJ5A= -github.com/joyt/godate v0.0.0-20150226210126-7151572574a7/go.mod h1:R+UgFL3iylLhx9N4w35zZ2HdhDlgorRDx4SxbchWuN0= github.com/juztin/numeronym v0.0.0-20160223091026-859fcc2918e2 h1:jrs0oyU9XY7MlTHbNxecqFgY+fgEENZdP4Z8FZln/pw= github.com/juztin/numeronym v0.0.0-20160223091026-859fcc2918e2/go.mod h1:uVDl4OnjvPk07IzoXF/dFM7nBYqAKdJsz4e9xjjWo7Q= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -30,18 +32,26 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -55,4 +65,6 @@ golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ledger/cmd/import.go b/ledger/cmd/import.go index 3b69595..bc5b86e 100644 --- a/ledger/cmd/import.go +++ b/ledger/cmd/import.go @@ -489,15 +489,16 @@ var importCmd = &cobra.Command{ defer imp.Close() lower := strings.ToLower(fileName) - if strings.HasSuffix(lower, ".xml") { + switch { + case strings.HasSuffix(lower, ".xml"): imp.importCamt() - } else if strings.HasSuffix(lower, ".qfx") || strings.HasSuffix(lower, ".ofx") { + case strings.HasSuffix(lower, ".qfx") || strings.HasSuffix(lower, ".ofx"): imp.importQFX() - } else if strings.HasSuffix(lower, ".qif") { + case strings.HasSuffix(lower, ".qif"): imp.importQIF() - } else if strings.HasSuffix(lower, ".iif") { + case strings.HasSuffix(lower, ".iif"): imp.importIIF() - } else { + default: imp.importCSV() } @@ -517,7 +518,7 @@ func init() { func (imp *Importer) existingTransaction(transDate time.Time, payee string) bool { for _, trans := range imp.generalLedger { - if trans.Date == transDate && strings.TrimSpace(trans.Payee) == strings.TrimSpace(payee) { + if trans.Date.Equal(transDate) && strings.TrimSpace(trans.Payee) == strings.TrimSpace(payee) { return true } } diff --git a/ledger/cmd/internal/httpcompress/compress.go b/ledger/cmd/internal/httpcompress/compress.go index 8ef43d9..a129d2f 100644 --- a/ledger/cmd/internal/httpcompress/compress.go +++ b/ledger/cmd/internal/httpcompress/compress.go @@ -16,9 +16,9 @@ type CompressResponseWriter struct { } func (res CompressResponseWriter) Write(b []byte) (int, error) { - if "" == res.Header().Get("Content-Type") { + if "" == res.ResponseWriter.Header().Get("Content-Type") { // If no content type, apply sniffing algorithm to un-gzipped body. - res.Header().Set("Content-Type", http.DetectContentType(b)) + res.ResponseWriter.Header().Set("Content-Type", http.DetectContentType(b)) } return res.Writer.Write(b) } diff --git a/ledger/cmd/print.go b/ledger/cmd/print.go index fb2f714..e9263e0 100644 --- a/ledger/cmd/print.go +++ b/ledger/cmd/print.go @@ -13,9 +13,9 @@ import ( "time" "unicode/utf8" + "github.com/araddon/dateparse" "github.com/howeyc/ledger" "github.com/howeyc/ledger/ledger/cmd/internal/fastcolor" - date "github.com/joyt/godate" "github.com/shopspring/decimal" "github.com/spf13/cobra" "golang.org/x/term" @@ -46,8 +46,8 @@ func cliTransactions() ([]*ledger.Transaction, error) { } } - parsedStartDate, tstartErr := date.Parse(startString) - parsedEndDate, tendErr := date.Parse(endString) + parsedStartDate, tstartErr := dateparse.ParseAny(startString) + parsedEndDate, tendErr := dateparse.ParseAny(endString) if tstartErr != nil || tendErr != nil { return nil, errors.New("unable to parse start or end date string argument") diff --git a/parse.go b/parse.go index 0e64411..91463a8 100644 --- a/parse.go +++ b/parse.go @@ -11,8 +11,8 @@ import ( "sync" "time" - "github.com/alfredxing/calc/compute" - date "github.com/joyt/godate" + "github.com/expr-lang/expr" + "github.com/araddon/dateparse" "github.com/shopspring/decimal" ) @@ -203,14 +203,10 @@ func (lp *parser) parseDate(dateString string) (transDate time.Time, err error) return lp.prevDate, lp.prevDateErr } - // try current date layout - transDate, err = time.Parse(lp.dateLayout, dateString) + // Use dateparse to handle flexible date formats + transDate, err = dateparse.ParseAny(dateString) if err != nil { - // try to find new date layout - transDate, lp.dateLayout, err = date.ParseAndGetLayout(dateString) - if err != nil { - err = fmt.Errorf("unable to parse date(%s): %w", dateString, err) - } + err = fmt.Errorf("unable to parse date(%s): %w", dateString, err) } // maybe next date is same @@ -249,11 +245,30 @@ func (a *Account) parsePosting(trimmedLine string, comment string) (err error) { a.Comment = comment if m[3] != "" { - bal, err := compute.Evaluate(m[3]) + program, err := expr.Compile(m[3]) + if err != nil { + return err + } + out, err := expr.Run(program, nil) if err != nil { return err } - a.Balance = decimal.NewFromFloat(bal) + + var f float64 + switch v := out.(type) { + case int: + f = float64(v) + case int64: + f = float64(v) + case float32: + f = float64(v) + case float64: + f = v + default: + return fmt.Errorf("expression did not evaluate to a number: %T", out) + } + + a.Balance = decimal.NewFromFloat(f) } // @@ explicit converted amount diff --git a/parseFuzz_test.go b/parseFuzz_test.go index 40116c3..d4c8048 100644 --- a/parseFuzz_test.go +++ b/parseFuzz_test.go @@ -21,13 +21,14 @@ func FuzzParseLedger(f *testing.F) { overall := decimal.Zero for _, t := range trans { for _, p := range t.AccountChanges { - if p.Converted != nil { + switch { + case p.Converted != nil: overall = overall.Add(p.Converted.Neg()) - } else if p.ConversionFactor != nil { + case p.ConversionFactor != nil: overall = overall.Add(p.Balance.Mul( *p.ConversionFactor, )) - } else { + default: overall = overall.Add(p.Balance) } } diff --git a/parse_test.go b/parse_test.go index ab2cb36..49e5108 100644 --- a/parse_test.go +++ b/parse_test.go @@ -59,7 +59,7 @@ var testCases = []testCase{ Assets 123 `, nil, - errors.New(`:1: unable to parse transaction: unable to parse date(1970/02/31): parsing time "1970/02/31": extra text: "1970/02/31"`), + errors.New(`:1: unable to parse transaction: unable to parse date(1970/02/31): parsing time "1970/02/31": day out of range`), }, { "unbalanced error", diff --git a/transaction.go b/transaction.go index 4e89a67..717f3c7 100644 --- a/transaction.go +++ b/transaction.go @@ -31,11 +31,12 @@ func (t *Transaction) IsBalanced() error { emptyAccIndex = i } - if acc.Converted != nil { + switch { + case acc.Converted != nil: transBal = transBal.Add(acc.Converted.Neg()) - } else if acc.ConversionFactor != nil { + case acc.ConversionFactor != nil: transBal = transBal.Add(acc.Balance.Mul(*acc.ConversionFactor)) - } else { + default: transBal = transBal.Add(acc.Balance) } } @@ -92,8 +93,10 @@ func (t *Transaction) inferConversionFactorForTwoCurrencyTx() error { var ( curKeys [2]string groups [2]*currencyGroup - i int ) + + // Collect keys to deterministically choose base/other currency + i := 0 for k, g := range currencyMap { if i >= 2 { break @@ -103,6 +106,12 @@ func (t *Transaction) inferConversionFactorForTwoCurrencyTx() error { i++ } + // Assign base currency as the one with the lower sort order + if curKeys[1] < curKeys[0] { + curKeys[0], curKeys[1] = curKeys[1], curKeys[0] + groups[0], groups[1] = groups[1], groups[0] + } + var baseCurIdx, otherCurIdx int hasConv0 := false for _, idx := range groups[0].indices { @@ -150,9 +159,10 @@ func (t *Transaction) inferConversionFactorForTwoCurrencyTx() error { for _, idx := range groups[otherCurIdx].indices { acc := &t.AccountChanges[idx] if acc.Converted != nil || acc.ConversionFactor != nil { - if acc.Converted != nil { + switch { + case acc.Converted != nil: sumOtherRaw = sumOtherRaw.Add(acc.Converted.Neg()) - } else if acc.ConversionFactor != nil { + case acc.ConversionFactor != nil: sumOtherRaw = sumOtherRaw.Add(acc.Balance.Mul(*acc.ConversionFactor)) } } else { From 7034adef30e1f2e109a9fc12587277a69eab8cb7 Mon Sep 17 00:00:00 2001 From: Aidan <6690599+aodhan-domhnaill@users.noreply.github.com> Date: Sat, 21 Mar 2026 07:56:23 +0000 Subject: [PATCH 14/14] No callback (#12) * removed callback flow and async, because it was overcomplicated * flattened parsing code --------- Co-authored-by: Aidan Macdonald --- balances_test.go | 4 +- go.mod | 2 + go.sum | 4 + ledger/cmd/print.go | 2 +- ledger/cmd/webHandlerAccounts.go | 2 +- linescanner.go | 27 +----- parse.go | 141 ++++++++----------------------- parseFuzz_test.go | 2 +- parse_test.go | 49 +---------- 9 files changed, 51 insertions(+), 182 deletions(-) diff --git a/balances_test.go b/balances_test.go index af70a46..14fe68c 100644 --- a/balances_test.go +++ b/balances_test.go @@ -122,7 +122,7 @@ var testBalCases = []testBalCase{ func TestBalanceLedger(t *testing.T) { for _, tc := range testBalCases { b := bytes.NewBufferString(tc.data) - transactions, err := ParseLedger(b) + transactions, err := ParseLedger("", b) bals := GetBalances(transactions, []string{}) if (err != nil && tc.err == nil) || (err != nil && tc.err != nil && err.Error() != tc.err.Error()) { t.Errorf("Error: expected `%s`, got `%s`", tc.err, err) @@ -189,7 +189,7 @@ func TestBalancesByPeriod(t *testing.T) { `) - trans, _ := ParseLedger(b) + trans, _ := ParseLedger("", b) partitionRb := BalancesByPeriod(trans, PeriodQuarter, RangePartition) snapshotRb := BalancesByPeriod(trans, PeriodQuarter, RangeSnapshot) diff --git a/go.mod b/go.mod index 19289cd..2142c90 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pelletier/go-toml v1.9.5 + github.com/samber/lo v1.53.0 github.com/shopspring/decimal v1.4.0 github.com/spf13/cobra v1.7.0 golang.org/x/term v0.13.0 @@ -28,4 +29,5 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 21c645c..b1f66b3 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM= +github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= @@ -61,6 +63,8 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/ledger/cmd/print.go b/ledger/cmd/print.go index e9263e0..35e08fe 100644 --- a/ledger/cmd/print.go +++ b/ledger/cmd/print.go @@ -59,7 +59,7 @@ func cliTransactions() ([]*ledger.Transaction, error) { var generalLedger []*ledger.Transaction var parseError error if ledgerFilePath == "-" { - generalLedger, parseError = ledger.ParseLedger(os.Stdin) + generalLedger, parseError = ledger.ParseLedger("", os.Stdin) } else { generalLedger, parseError = ledger.ParseLedgerFile(ledgerFilePath) } diff --git a/ledger/cmd/webHandlerAccounts.go b/ledger/cmd/webHandlerAccounts.go index 6118a89..92fb6df 100644 --- a/ledger/cmd/webHandlerAccounts.go +++ b/ledger/cmd/webHandlerAccounts.go @@ -75,7 +75,7 @@ func addTransactionPostHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(&tbuf, "") /* Check valid transaction is created */ - trans, perr := ledger.ParseLedger(&tbuf) + trans, perr := ledger.ParseLedger("", &tbuf) if perr != nil { http.Error(w, perr.Error(), 500) return diff --git a/linescanner.go b/linescanner.go index 191aa5f..f7edd85 100644 --- a/linescanner.go +++ b/linescanner.go @@ -3,14 +3,10 @@ package ledger import ( "bufio" "io" - "os" - "unsafe" ) type linescanner struct { - scanner *bufio.Scanner - unsafe bool - + scanner *bufio.Scanner filename string lineCount int } @@ -20,17 +16,6 @@ type linescanner struct { func newLineScanner(filename string, r io.Reader) *linescanner { lp := &linescanner{} lp.scanner = bufio.NewScanner(r) - if fs, fserr := os.Stat(filename); fserr == nil { - size := int(fs.Size()) - // only pre-alloc for large files - if size > bufio.MaxScanTokenSize { - // allocate large enough such that scanner in bufio doesn't need to - // move anything during scanning - size *= 2 - lp.scanner.Buffer(make([]byte, size), size) - lp.unsafe = true - } - } lp.filename = filename return lp @@ -42,15 +27,7 @@ func (lp *linescanner) Scan() bool { func (lp *linescanner) Text() string { var line string - if lp.unsafe { - if lbytes := lp.scanner.Bytes(); len(lbytes) > 0 { - line = unsafe.String(unsafe.SliceData(lbytes), len(lbytes)) - } else { - line = "" - } - } else { - line = lp.scanner.Text() - } + line = lp.scanner.Text() lp.lineCount++ return line } diff --git a/parse.go b/parse.go index 91463a8..d7e8139 100644 --- a/parse.go +++ b/parse.go @@ -8,11 +8,11 @@ import ( "path/filepath" "regexp" "strings" - "sync" "time" - "github.com/expr-lang/expr" "github.com/araddon/dateparse" + "github.com/expr-lang/expr" + "github.com/samber/lo" "github.com/shopspring/decimal" ) @@ -23,61 +23,23 @@ func ParseLedgerFile(filename string) (generalLedger []*Transaction, err error) return nil, ierr } defer ifile.Close() - var mu sync.Mutex - parseLedger(filename, ifile, func(t []*Transaction, e error) (stop bool) { - if e != nil { - err = e - stop = true - return - } - - mu.Lock() - generalLedger = append(generalLedger, t...) - mu.Unlock() - return - }) - - return + return ParseLedger(filename, ifile) } // ParseLedger parses a ledger file and returns a list of Transactions. -func ParseLedger(ledgerReader io.Reader) (generalLedger []*Transaction, err error) { - parseLedger("", ledgerReader, func(t []*Transaction, e error) (stop bool) { - if e != nil { - err = e - stop = true - return - } +func ParseLedger(name string, ledgerReader io.Reader) (generalLedger []*Transaction, err error) { + blocks, err := parseBlocks(name, ledgerReader) + if err != nil { + return nil, err + } - generalLedger = append(generalLedger, t...) - return + return lo.MapErr(blocks, func(b block, _ int) (*Transaction, error) { + trans, transErr := b.parseTransaction() + if transErr != nil { + return nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", b.filename, b.lineNum, transErr) + } + return trans, nil }) - - return -} - -// ParseLedgerAsync parses a ledger file and returns a Transaction and error channels . -func ParseLedgerAsync(ledgerReader io.Reader) (c chan *Transaction, e chan error) { - c = make(chan *Transaction) - e = make(chan error) - - go func() { - parseLedger("", ledgerReader, func(tlist []*Transaction, err error) (stop bool) { - if err != nil { - e <- err - } else { - for _, t := range tlist { - c <- t - } - } - return - }) - - e <- nil - close(c) - close(e) - }() - return c, e } type parser struct { @@ -91,12 +53,10 @@ type parser struct { prevDate time.Time } -func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Transaction, err error) (stop bool)) (stop bool) { +func parseBlocks(filename string, ledgerReader io.Reader) ([]block, error) { var lp parser lp.scanner = newLineScanner(filename, ledgerReader) - var tlist []*Transaction - blocks := []block{} comments := []string{} for lp.scanner.Scan() { @@ -121,30 +81,36 @@ func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Tra before, after, split := strings.Cut(trimmedLine, " ") if !split { - if callback(nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", lp.scanner.Name(), lp.scanner.LineNumber(), - fmt.Errorf("unable to parse payee line: %s", trimmedLine))) { - return true - } - if len(currentComment) > 0 { - comments = append(comments, currentComment) - } - continue + return nil, fmt.Errorf( + "%s:%d: unable to parse transaction: %w", + lp.scanner.Name(), + lp.scanner.LineNumber(), + fmt.Errorf("unable to parse payee line: %s", trimmedLine), + ) } switch before { case "account": lp.skipAccount() case "include": - stop := lp.include(after, callback) - if stop { - return stop + paths, _ := filepath.Glob(filepath.Join(filepath.Dir(lp.scanner.Name()), after)) + if len(paths) < 1 { + return nil, fmt.Errorf( + "%s:%d: unable to include file(%s): %w", lp.scanner.Name(), lp.scanner.LineNumber(), after, errors.New("not found")) } + + b, err := lo.FlatMapErr(paths, func(path string, _ int) ([]block, error) { + f, _ := os.Open(path) + defer f.Close() + return parseBlocks(path, f) + }) + if err != nil { + return nil, err + } + blocks = append(blocks, b...) default: transDate, derr := lp.parseDate(before) if derr != nil { - if callback(nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", lp.scanner.Name(), lp.scanner.LineNumber(), derr)) { - return true - } - continue + return nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", lp.scanner.Name(), lp.scanner.LineNumber(), derr) } blocks = append(blocks, lp.parseBlock(transDate, after, currentComment, comments)) @@ -152,18 +118,7 @@ func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Tra } } - for _, block := range blocks { - trans, transErr := block.parseTransaction() - if transErr != nil { - if callback(nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", block.filename, block.lineNum, transErr)) { - return true - } - continue - } - tlist = append(tlist, trans) - } - callback(tlist, nil) - return false + return blocks, nil } func (lp *parser) skipAccount() { @@ -175,28 +130,6 @@ func (lp *parser) skipAccount() { } } -func (lp *parser) include(after string, callback func(t []*Transaction, err error) (stop bool)) (stop bool) { - paths, _ := filepath.Glob(filepath.Join(filepath.Dir(lp.scanner.Name()), after)) - if len(paths) < 1 { - callback(nil, fmt.Errorf("%s:%d: unable to include file(%s): %w", lp.scanner.Name(), lp.scanner.LineNumber(), after, errors.New("not found"))) - return true - } - var wg sync.WaitGroup - for _, incpath := range paths { - wg.Add(1) - go func(ipath string) { - ifile, _ := os.Open(ipath) - defer ifile.Close() - if parseLedger(ipath, ifile, callback) { - stop = true - } - wg.Done() - }(incpath) - } - wg.Wait() - return -} - func (lp *parser) parseDate(dateString string) (transDate time.Time, err error) { // seen before, skip parse if lp.strPrevDate == dateString { diff --git a/parseFuzz_test.go b/parseFuzz_test.go index d4c8048..95e6dd8 100644 --- a/parseFuzz_test.go +++ b/parseFuzz_test.go @@ -17,7 +17,7 @@ func FuzzParseLedger(f *testing.F) { } f.Fuzz(func(t *testing.T, s string) { b := bytes.NewBufferString(s) - trans, _ := ParseLedger(b) + trans, _ := ParseLedger("", b) overall := decimal.Zero for _, t := range trans { for _, p := range t.AccountChanges { diff --git a/parse_test.go b/parse_test.go index 49e5108..eab903d 100644 --- a/parse_test.go +++ b/parse_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "errors" - "sync" "testing" "time" @@ -650,7 +649,7 @@ func p(d decimal.Decimal) *decimal.Decimal { func TestParseLedger(t *testing.T) { for _, tc := range testCases { b := bytes.NewBufferString(tc.data) - transactions, err := ParseLedger(b) + transactions, err := ParseLedger("", b) if (err != nil && tc.err == nil) || (err != nil && tc.err != nil && err.Error() != tc.err.Error()) { t.Errorf("Error: expected `%s`, got `%s`", tc.err, err) } @@ -662,52 +661,6 @@ func TestParseLedger(t *testing.T) { } } -func TestParseLedgerAsync(t *testing.T) { - buf := bytes.NewBufferString(`; test -account bam:bam - subacc line ; sub comment - another subacc line - -1970/01/01 Payee - Assets 50 - Expenses - -1970/02/30 Error ; oops - Assets 30 - Expenses - -1970/01/01bbafafdaf;bad comment - Assets 20 - Expenses - -account endofledger`) - - tc, ec := ParseLedgerAsync(buf) - - var trans []*Transaction - var errors []error - - var wg sync.WaitGroup - wg.Add(2) - go func() { - for t := range tc { - trans = append(trans, t) - } - wg.Done() - }() - go func() { - for e := range ec { - errors = append(errors, e) - } - wg.Done() - }() - wg.Wait() - - if len(trans) < 1 || len(errors) < 1 { - t.Error("async parse failed") - } -} - func BenchmarkParseLedger(b *testing.B) { for b.Loop() { _, _ = ParseLedgerFile("testdata/ledgerBench.dat")