From 016c0ed6536211734c4cec93ece5939aac317b24 Mon Sep 17 00:00:00 2001 From: osamingo <1390409+osamingo@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:08:34 +0900 Subject: [PATCH 1/4] feat(cmd/csvpp/internal/tui): add interactive filter to view command - Add "/" key to enter filter mode with real-time row filtering - Support "column:value" syntax for column-specific search - Add case-insensitive substring matching across all columns - Preserve row selection state using original record indices - Extract color palette to package-level variables for readability - Add FilterPrompt and FilterActive styles - Add textinput dependency for filter input widget - Add TestParseFilterQuery and TestMatchesFilter test cases Co-Authored-By: Claude Opus 4.6 --- cmd/csvpp/internal/tui/export_test.go | 24 ++ cmd/csvpp/internal/tui/model.go | 321 +++++++++++++++++++++++--- cmd/csvpp/internal/tui/model_test.go | 173 ++++++++++++++ cmd/csvpp/internal/tui/styles.go | 32 ++- go.mod | 1 + go.sum | 2 + 6 files changed, 506 insertions(+), 47 deletions(-) create mode 100644 cmd/csvpp/internal/tui/export_test.go diff --git a/cmd/csvpp/internal/tui/export_test.go b/cmd/csvpp/internal/tui/export_test.go new file mode 100644 index 0000000..0e639c2 --- /dev/null +++ b/cmd/csvpp/internal/tui/export_test.go @@ -0,0 +1,24 @@ +package tui + +import ( + "github.com/charmbracelet/bubbles/table" + + "github.com/osamingo/go-csvpp" +) + +// FilterQuery is an exported wrapper of filterQuery for testing. +type FilterQuery struct { + Column string + Value string +} + +// ParseFilterQuery exports parseFilterQuery for testing. +func ParseFilterQuery(s string) FilterQuery { + q := parseFilterQuery(s) + return FilterQuery{Column: q.column, Value: q.value} +} + +// MatchesFilter exports matchesFilter for testing. +func MatchesFilter(query FilterQuery, headers []*csvpp.ColumnHeader, row table.Row) bool { + return matchesFilter(filterQuery{column: query.Column, value: query.Value}, headers, row) +} diff --git a/cmd/csvpp/internal/tui/model.go b/cmd/csvpp/internal/tui/model.go index 2a694f9..adc0bb1 100644 --- a/cmd/csvpp/internal/tui/model.go +++ b/cmd/csvpp/internal/tui/model.go @@ -6,12 +6,78 @@ import ( "strings" "github.com/charmbracelet/bubbles/table" + "github.com/charmbracelet/bubbles/textinput" tea "github.com/charmbracelet/bubbletea" "golang.design/x/clipboard" "github.com/osamingo/go-csvpp" ) +// filterQuery represents a parsed filter query. +type filterQuery struct { + column string // empty means search all columns + value string // search string (lowercase) +} + +// parseFilterQuery parses a filter query string. +// Supports "column:value" syntax for column-specific filtering. +// If the part before ":" is empty, it searches all columns. +func parseFilterQuery(s string) filterQuery { + s = strings.TrimSpace(s) + if s == "" { + return filterQuery{} + } + + if idx := strings.Index(s, ":"); idx >= 0 { + col := strings.TrimSpace(s[:idx]) + val := strings.TrimSpace(s[idx+1:]) + if col != "" { + return filterQuery{ + column: strings.ToLower(col), + value: strings.ToLower(val), + } + } + // empty column part means search all columns + return filterQuery{ + value: strings.ToLower(val), + } + } + + return filterQuery{ + value: strings.ToLower(s), + } +} + +// matchesFilter checks if a row matches the given filter query. +func matchesFilter(query filterQuery, headers []*csvpp.ColumnHeader, row table.Row) bool { + if query.value == "" { + return true + } + + if query.column != "" { + // Search specific column + for i, h := range headers { + if strings.ToLower(formatHeaderTitle(h)) == query.column { + // row[0] is the selection marker, data starts at index 1 + if i+1 < len(row) && strings.Contains(strings.ToLower(row[i+1]), query.value) { + return true + } + return false + } + } + // Column not found + return false + } + + // Search all data columns (skip index 0 which is selection marker) + for i := 1; i < len(row); i++ { + if strings.Contains(strings.ToLower(row[i]), query.value) { + return true + } + } + return false +} + // Model represents the TUI model for viewing CSV++ data. type Model struct { table table.Model @@ -21,8 +87,15 @@ type Model struct { width int height int err error - selected map[int]bool + selected map[int]bool // keyed by original record index copied bool + + // Filter fields + filtering bool // true when filter input is active + filterInput textinput.Model // text input widget + filterText string // committed filter text + filteredIdx []int // display position -> original record index + allRows []table.Row // cache of all rows } // NewModel creates a new TUI model with the given data. @@ -75,12 +148,34 @@ func NewModel(headers []*csvpp.ColumnHeader, records [][]*csvpp.Field) Model { s.Cell = styles.Cell t.SetStyles(s) + // Initialize filter input + fi := textinput.New() + fi.Placeholder = "type to filter..." + fi.CharLimit = 256 + + // Build allRows cache + allRows := make([]table.Row, len(rows)) + for i, row := range rows { + r := make(table.Row, len(row)) + copy(r, row) + allRows[i] = r + } + + // Build initial filteredIdx (1:1 mapping) + filteredIdx := make([]int, len(rows)) + for i := range filteredIdx { + filteredIdx[i] = i + } + return Model{ - table: t, - headers: headers, - records: records, - styles: styles, - selected: make(map[int]bool), + table: t, + headers: headers, + records: records, + styles: styles, + selected: make(map[int]bool), + filterInput: fi, + filteredIdx: filteredIdx, + allRows: allRows, } } @@ -95,33 +190,10 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { //nostyle:recvtype switch msg := msg.(type) { case tea.KeyMsg: - switch msg.String() { - case "q", "ctrl+c": - return m, tea.Quit - case "esc": - // Clear selection - m.selected = make(map[int]bool) - m.copied = false - m.updateRowMarkers() - return m, nil - case " ": - // Toggle selection - cursor := m.table.Cursor() - if m.selected[cursor] { - delete(m.selected, cursor) - } else { - m.selected[cursor] = true - } - m.copied = false - m.updateRowMarkers() - return m, nil - case "y", "c": - // Copy selected rows - if len(m.selected) > 0 { - m.copyToClipboard() - } - return m, nil + if m.filtering { + return m.updateFilterMode(msg) } + return m.updateNormalMode(msg) case tea.WindowSizeMsg: m.width = msg.Width m.height = msg.Height @@ -133,11 +205,104 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { //nostyle:recvtype return m, cmd } -// updateRowMarkers updates the selection markers in the table rows. -func (m *Model) updateRowMarkers() { +// updateFilterMode handles key events when filter input is active. +func (m Model) updateFilterMode(msg tea.KeyMsg) (tea.Model, tea.Cmd) { //nostyle:recvtype + switch msg.Type { + case tea.KeyEnter: + // Commit filter + m.filterText = m.filterInput.Value() + m.filtering = false + m.filterInput.Blur() + m.table.Focus() + m.applyFilter() + return m, nil + case tea.KeyEsc: + // Cancel filter + m.filtering = false + m.filterInput.SetValue("") + m.filterInput.Blur() + m.table.Focus() + m.clearFilter() + return m, nil + } + + // Forward key to textinput + var cmd tea.Cmd + m.filterInput, cmd = m.filterInput.Update(msg) + + // Real-time filtering + m.applyFilter() + + return m, cmd +} + +// updateNormalMode handles key events in normal navigation mode. +func (m Model) updateNormalMode(msg tea.KeyMsg) (tea.Model, tea.Cmd) { //nostyle:recvtype + switch msg.String() { + case "q", "ctrl+c": + return m, tea.Quit + case "/": + // Enter filter mode + m.filtering = true + m.filterInput.SetValue("") + m.table.Blur() + return m, m.filterInput.Focus() + case "esc": + if m.filterText != "" { + // Clear active filter + m.clearFilter() + } else { + // Clear selection + m.selected = make(map[int]bool) + m.copied = false + m.rebuildRowMarkers() + } + return m, nil + case " ": + // Toggle selection using original index + origIdx := m.originalIndex() + if origIdx < 0 { + return m, nil + } + if m.selected[origIdx] { + delete(m.selected, origIdx) + } else { + m.selected[origIdx] = true + } + m.copied = false + m.rebuildRowMarkers() + return m, nil + case "y", "c": + // Copy selected rows + if len(m.selected) > 0 { + m.copyToClipboard() + } + return m, nil + } + + var cmd tea.Cmd + m.table, cmd = m.table.Update(msg) + return m, cmd +} + +// originalIndex returns the original record index for the current cursor position. +// Returns -1 if the cursor position is out of range. +func (m *Model) originalIndex() int { + cursor := m.table.Cursor() + if cursor < 0 || cursor >= len(m.filteredIdx) { + return -1 + } + return m.filteredIdx[cursor] +} + +// rebuildRowMarkers updates selection markers in the currently displayed rows. +func (m *Model) rebuildRowMarkers() { rows := m.table.Rows() - for i := range rows { - if m.selected[i] { + for i, origIdx := range m.filteredIdx { + if i >= len(rows) { + break + } + if m.selected[origIdx] { rows[i][0] = "✓" } else { rows[i][0] = " " @@ -146,6 +311,68 @@ func (m *Model) updateRowMarkers() { m.table.SetRows(rows) } +// applyFilter filters rows based on the current filter input value. +func (m *Model) applyFilter() { + query := parseFilterQuery(m.filterInput.Value()) + + if query.value == "" { + // Show all rows without clearing committed filter state + m.restoreAllRows() + return + } + + var filtered []table.Row + var idx []int + + for i, row := range m.allRows { + if matchesFilter(query, m.headers, row) { + r := make(table.Row, len(row)) + copy(r, row) + // Set selection marker + if m.selected[i] { + r[0] = "✓" + } else { + r[0] = " " + } + filtered = append(filtered, r) + idx = append(idx, i) + } + } + + m.filteredIdx = idx + m.table.SetRows(filtered) + m.table.GotoTop() +} + +// clearFilter resets the filter state and restores all rows. +func (m *Model) clearFilter() { + m.filterText = "" + m.restoreAllRows() +} + +// restoreAllRows restores all rows to the table without modifying filter state. +func (m *Model) restoreAllRows() { + rows := make([]table.Row, len(m.allRows)) + for i, row := range m.allRows { + r := make(table.Row, len(row)) + copy(r, row) + if m.selected[i] { + r[0] = "✓" + } else { + r[0] = " " + } + rows[i] = r + } + + filteredIdx := make([]int, len(m.allRows)) + for i := range filteredIdx { + filteredIdx[i] = i + } + + m.filteredIdx = filteredIdx + m.table.SetRows(rows) +} + // copyToClipboard copies selected rows to clipboard in CSVPP format. func (m *Model) copyToClipboard() { if err := clipboard.Init(); err != nil { @@ -190,8 +417,21 @@ func (m Model) View() string { //nostyle:recvtype b.WriteString(m.table.View()) b.WriteString("\n\n") + // Filter line + if m.filtering { + b.WriteString(m.styles.FilterPrompt.Render("/")) + b.WriteString(m.filterInput.View()) + b.WriteString("\n") + } else if m.filterText != "" { + b.WriteString(m.styles.FilterActive.Render(fmt.Sprintf("Filter: %s", m.filterText))) + b.WriteString("\n") + } + // Status line status := fmt.Sprintf("%d columns, %d records", len(m.headers), len(m.records)) + if m.filterText != "" || m.filtering { + status += fmt.Sprintf(" (%d shown)", len(m.filteredIdx)) + } if len(m.selected) > 0 { status += fmt.Sprintf(" | %d selected", len(m.selected)) } @@ -202,7 +442,14 @@ func (m Model) View() string { //nostyle:recvtype b.WriteString("\n") // Help - help := "↑/↓: navigate • Space: select • y/c: copy • Esc: clear • q: quit" + var help string + if m.filtering { + help = "Enter: apply filter • Esc: cancel • type to filter" + } else if m.filterText != "" { + help = "↑/↓: navigate • Space: select • y/c: copy • /: filter • Esc: clear filter • q: quit" + } else { + help = "↑/↓: navigate • Space: select • y/c: copy • /: filter • Esc: clear • q: quit" + } b.WriteString(m.styles.Help.Render(help)) return b.String() diff --git a/cmd/csvpp/internal/tui/model_test.go b/cmd/csvpp/internal/tui/model_test.go index c7cdaa9..96be3ac 100644 --- a/cmd/csvpp/internal/tui/model_test.go +++ b/cmd/csvpp/internal/tui/model_test.go @@ -4,6 +4,9 @@ import ( "strings" "testing" + "github.com/charmbracelet/bubbles/table" + "github.com/google/go-cmp/cmp" + "github.com/osamingo/go-csvpp" "github.com/osamingo/go-csvpp/cmd/csvpp/internal/tui" ) @@ -127,3 +130,173 @@ func TestNewModel(t *testing.T) { t.Error("expected non-empty view") } } + +func TestParseFilterQuery(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want tui.FilterQuery + }{ + { + name: "success: empty string", + in: "", + want: tui.FilterQuery{}, + }, + { + name: "success: whitespace only", + in: " ", + want: tui.FilterQuery{}, + }, + { + name: "success: simple text searches all columns", + in: "alice", + want: tui.FilterQuery{Column: "", Value: "alice"}, + }, + { + name: "success: preserves lowercase conversion", + in: "ALICE", + want: tui.FilterQuery{Column: "", Value: "alice"}, + }, + { + name: "success: column specific search", + in: "name:alice", + want: tui.FilterQuery{Column: "name", Value: "alice"}, + }, + { + name: "success: column specific with spaces trimmed", + in: " name : alice ", + want: tui.FilterQuery{Column: "name", Value: "alice"}, + }, + { + name: "success: column name is lowercased", + in: "Name:alice", + want: tui.FilterQuery{Column: "name", Value: "alice"}, + }, + { + name: "success: empty column part searches all columns", + in: ":alice", + want: tui.FilterQuery{Column: "", Value: "alice"}, + }, + { + name: "success: column with empty value", + in: "name:", + want: tui.FilterQuery{Column: "name", Value: ""}, + }, + { + name: "success: value with colon inside", + in: "name:a:b", + want: tui.FilterQuery{Column: "name", Value: "a:b"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := tui.ParseFilterQuery(tt.in) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("ParseFilterQuery(%q) mismatch (-want +got):\n%s", tt.in, diff) + } + }) + } +} + +func TestMatchesFilter(t *testing.T) { + t.Parallel() + + headers := []*csvpp.ColumnHeader{ + {Name: "name", Kind: csvpp.SimpleField}, + {Name: "age", Kind: csvpp.SimpleField}, + {Name: "city", Kind: csvpp.SimpleField}, + } + + // row[0] = selection marker, row[1..] = data columns + row := table.Row{" ", "Alice", "30", "Tokyo"} + + tests := []struct { + name string + query tui.FilterQuery + row table.Row + want bool + }{ + { + name: "success: empty query matches all", + query: tui.FilterQuery{Value: ""}, + row: row, + want: true, + }, + { + name: "success: all columns match by name", + query: tui.FilterQuery{Value: "alice"}, + row: row, + want: true, + }, + { + name: "success: all columns match by city", + query: tui.FilterQuery{Value: "tokyo"}, + row: row, + want: true, + }, + { + name: "success: case insensitive match on row value", + query: tui.FilterQuery{Value: "alice"}, + row: table.Row{" ", "ALICE", "30", "Tokyo"}, + want: true, + }, + { + name: "success: partial match", + query: tui.FilterQuery{Value: "ali"}, + row: row, + want: true, + }, + { + name: "error: no match in any column", + query: tui.FilterQuery{Value: "xyz"}, + row: row, + want: false, + }, + { + name: "success: column specific match", + query: tui.FilterQuery{Column: "name", Value: "alice"}, + row: row, + want: true, + }, + { + name: "error: column specific no match", + query: tui.FilterQuery{Column: "name", Value: "tokyo"}, + row: row, + want: false, + }, + { + name: "error: nonexistent column", + query: tui.FilterQuery{Column: "email", Value: "alice"}, + row: row, + want: false, + }, + { + name: "success: column specific match by age", + query: tui.FilterQuery{Column: "age", Value: "30"}, + row: row, + want: true, + }, + { + name: "error: selection marker not searched", + query: tui.FilterQuery{Value: "✓"}, + row: table.Row{"✓", "Alice", "30", "Tokyo"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := tui.MatchesFilter(tt.query, headers, tt.row) + if got != tt.want { + t.Errorf("MatchesFilter() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/csvpp/internal/tui/styles.go b/cmd/csvpp/internal/tui/styles.go index b8ddf67..e5bbaee 100644 --- a/cmd/csvpp/internal/tui/styles.go +++ b/cmd/csvpp/internal/tui/styles.go @@ -5,22 +5,34 @@ import ( "github.com/charmbracelet/lipgloss" ) +// Color palette for the TUI theme. +var ( + colorAccent = lipgloss.Color("229") // yellow – header/selected foreground, filter prompt + colorPrimary = lipgloss.Color("57") // purple – header/selected background + colorMuted = lipgloss.Color("241") // gray – help/status text + colorFilter = lipgloss.Color("86") // green – active filter indicator +) + // Styles holds the styles for the TUI components. type Styles struct { - Header lipgloss.Style - Cell lipgloss.Style - Selected lipgloss.Style - Help lipgloss.Style - Status lipgloss.Style + Header lipgloss.Style + Cell lipgloss.Style + Selected lipgloss.Style + Help lipgloss.Style + Status lipgloss.Style + FilterPrompt lipgloss.Style + FilterActive lipgloss.Style } // DefaultStyles returns the default styles for the TUI. func DefaultStyles() Styles { return Styles{ - Header: lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("229")).Background(lipgloss.Color("57")).Padding(0, 1), - Cell: lipgloss.NewStyle().Padding(0, 1), - Selected: lipgloss.NewStyle().Foreground(lipgloss.Color("229")).Background(lipgloss.Color("57")).Padding(0, 1), - Help: lipgloss.NewStyle().Foreground(lipgloss.Color("241")), - Status: lipgloss.NewStyle().Foreground(lipgloss.Color("241")).Padding(0, 1), + Header: lipgloss.NewStyle().Bold(true).Foreground(colorAccent).Background(colorPrimary).Padding(0, 1), + Cell: lipgloss.NewStyle().Padding(0, 1), + Selected: lipgloss.NewStyle().Foreground(colorAccent).Background(colorPrimary).Padding(0, 1), + Help: lipgloss.NewStyle().Foreground(colorMuted), + Status: lipgloss.NewStyle().Foreground(colorMuted).Padding(0, 1), + FilterPrompt: lipgloss.NewStyle().Bold(true).Foreground(colorAccent), + FilterActive: lipgloss.NewStyle().Foreground(colorFilter), } } diff --git a/go.mod b/go.mod index 255a31f..2327b06 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( ) require ( + github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bmatcuk/doublestar/v4 v4.9.2 // indirect github.com/charmbracelet/colorprofile v0.4.1 // indirect diff --git a/go.sum b/go.sum index 6f76478..4f50e50 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= From 1467017d4bb6fb4f291b2f8cc8ef1977982a6f6c Mon Sep 17 00:00:00 2001 From: osamingo <1390409+osamingo@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:08:56 +0900 Subject: [PATCH 2/4] fix(cmd/csvpp): require --from flag when converting stdin to csvpp - Remove silent default to JSON when input format is ambiguous - Return explicit error message guiding user to specify --from flag - Keep CSVPP as default input format when converting to JSON/YAML Co-Authored-By: Claude Opus 4.6 --- cmd/csvpp/convert.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/csvpp/convert.go b/cmd/csvpp/convert.go index fdeb75c..f7e73e0 100644 --- a/cmd/csvpp/convert.go +++ b/cmd/csvpp/convert.go @@ -94,13 +94,10 @@ func runConvert(cmd *cobra.Command, _ []string) (retErr error) { // Infer input format from output format for stdin if inputFormat == "" && inputFile == "" { - // If output is CSVPP, input must be JSON or YAML (default to JSON) - // Otherwise, input is CSVPP if outFormat == FormatCSVPP { - inputFormat = FormatJSON // Default to JSON when converting to CSVPP from stdin - } else { - inputFormat = FormatCSVPP + return fmt.Errorf("--from flag is required when reading from stdin and converting to csvpp (specify json or yaml)") } + inputFormat = FormatCSVPP } // Open input From 62a4a090668466f241a5896f506179900f6641bb Mon Sep 17 00:00:00 2001 From: osamingo <1390409+osamingo@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:09:10 +0900 Subject: [PATCH 3/4] fix(cmd/csvpp/internal/converter): preserve key order from JSON/YAML parsing - Add keyOrderInfo type for tracking ordered keys with nested structure - Extract JSON key order using token-based decoder on first record - Extract YAML key order using yaml.MapSlice for order-preserving decode - Add reorderComponents to fix structured field component ordering - Remove non-deterministic collectKeyOrder function - Strengthen tests with cmp.Diff for exact header and record comparison Co-Authored-By: Claude Opus 4.6 --- cmd/csvpp/internal/converter/decode.go | 298 ++++++++++++++++++-- cmd/csvpp/internal/converter/decode_test.go | 122 ++++---- 2 files changed, 321 insertions(+), 99 deletions(-) diff --git a/cmd/csvpp/internal/converter/decode.go b/cmd/csvpp/internal/converter/decode.go index bb0e6b8..bc781e7 100644 --- a/cmd/csvpp/internal/converter/decode.go +++ b/cmd/csvpp/internal/converter/decode.go @@ -2,6 +2,7 @@ package converter import ( + "bytes" "encoding/json" "fmt" "io" @@ -11,55 +12,285 @@ import ( "github.com/osamingo/go-csvpp" ) +// keyOrderInfo holds the ordered keys extracted from the first record. +type keyOrderInfo struct { + keys []string + nested map[string]*keyOrderInfo +} + // FromJSON reads JSON array and converts to CSVPP headers and records. // The JSON must be an array of objects with consistent keys. func FromJSON(r io.Reader) ([]*csvpp.ColumnHeader, [][]*csvpp.Field, error) { - var data []map[string]any - if err := json.NewDecoder(r).Decode(&data); err != nil { + data, err := io.ReadAll(r) + if err != nil { + return nil, nil, fmt.Errorf("failed to read input: %w", err) + } + + var records []map[string]any + if err := json.Unmarshal(data, &records); err != nil { return nil, nil, fmt.Errorf("failed to decode JSON: %w", err) } - if len(data) == 0 { + if len(records) == 0 { return nil, nil, nil } - headers := inferHeaders(data) - records := convertRecords(headers, data) + order, err := extractJSONKeyOrder(data) + if err != nil { + return nil, nil, fmt.Errorf("failed to extract JSON key order: %w", err) + } - return headers, records, nil + headers := inferHeaders(records, order) + fields := convertRecords(headers, records) + + return headers, fields, nil } // FromYAML reads YAML array and converts to CSVPP headers and records. // The YAML must be an array of objects with consistent keys. func FromYAML(r io.Reader) ([]*csvpp.ColumnHeader, [][]*csvpp.Field, error) { - var data []map[string]any - if err := yaml.NewDecoder(r).Decode(&data); err != nil { + data, err := io.ReadAll(r) + if err != nil { + return nil, nil, fmt.Errorf("failed to read input: %w", err) + } + + var records []map[string]any + if err := yaml.Unmarshal(data, &records); err != nil { return nil, nil, fmt.Errorf("failed to decode YAML: %w", err) } - if len(data) == 0 { + if len(records) == 0 { return nil, nil, nil } - headers := inferHeaders(data) - records := convertRecords(headers, data) + order, err := extractYAMLKeyOrder(data) + if err != nil { + return nil, nil, fmt.Errorf("failed to extract YAML key order: %w", err) + } + + headers := inferHeaders(records, order) + fields := convertRecords(headers, records) + + return headers, fields, nil +} + +// extractJSONKeyOrder extracts key order from the first record in a JSON array. +func extractJSONKeyOrder(data []byte) (*keyOrderInfo, error) { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + if len(raw) == 0 { + return &keyOrderInfo{nested: make(map[string]*keyOrderInfo)}, nil + } + return readJSONObjectOrder(json.NewDecoder(bytes.NewReader(raw[0]))) +} + +// readJSONObjectOrder reads one JSON object from a decoder and extracts ordered keys. +func readJSONObjectOrder(dec *json.Decoder) (*keyOrderInfo, error) { + t, err := dec.Token() + if err != nil { + return nil, err + } + if d, ok := t.(json.Delim); !ok || d != '{' { + return nil, fmt.Errorf("expected '{', got %v", t) + } + + info := &keyOrderInfo{nested: make(map[string]*keyOrderInfo)} + + for dec.More() { + t, err = dec.Token() + if err != nil { + return nil, err + } + key, ok := t.(string) + if !ok { + return nil, fmt.Errorf("expected string key, got %T", t) + } + info.keys = append(info.keys, key) + + nested, err := readJSONValueOrder(dec) + if err != nil { + return nil, err + } + if nested != nil { + info.nested[key] = nested + } + } + + // consume closing '}' + _, err = dec.Token() + return info, err +} + +// readJSONValueOrder reads one JSON value, extracting key order if it's an object or array of objects. +func readJSONValueOrder(dec *json.Decoder) (*keyOrderInfo, error) { + t, err := dec.Token() + if err != nil { + return nil, err + } + + d, ok := t.(json.Delim) + if !ok { + return nil, nil // scalar value + } + + switch d { + case '{': + // Object value - extract component keys + info := &keyOrderInfo{nested: make(map[string]*keyOrderInfo)} + for dec.More() { + t, err = dec.Token() + if err != nil { + return nil, err + } + key, ok := t.(string) + if !ok { + return nil, fmt.Errorf("expected string key, got %T", t) + } + info.keys = append(info.keys, key) + if err := skipJSONValue(dec); err != nil { + return nil, err + } + } + _, err = dec.Token() // '}' + return info, err + + case '[': + // Array value - check if first element is an object + if !dec.More() { + _, err = dec.Token() // ']' + return nil, err + } + + t, err = dec.Token() + if err != nil { + return nil, err + } + + if d2, ok := t.(json.Delim); ok && d2 == '{' { + // Array of objects - extract keys from the first object + info := &keyOrderInfo{nested: make(map[string]*keyOrderInfo)} + for dec.More() { + t, err = dec.Token() + if err != nil { + return nil, err + } + key, ok := t.(string) + if !ok { + return nil, fmt.Errorf("expected string key, got %T", t) + } + info.keys = append(info.keys, key) + if err := skipJSONValue(dec); err != nil { + return nil, err + } + } + _, err = dec.Token() // closing '}' of first object + if err != nil { + return nil, err + } + // Skip remaining array elements + for dec.More() { + if err := skipJSONValue(dec); err != nil { + return nil, err + } + } + _, err = dec.Token() // ']' + return info, err + } + + // First element is not an object - skip the rest + if d2, ok := t.(json.Delim); ok { + if err := skipJSONDelimContent(dec, d2); err != nil { + return nil, err + } + } + for dec.More() { + if err := skipJSONValue(dec); err != nil { + return nil, err + } + } + _, err = dec.Token() // ']' + return nil, err + } + + return nil, nil +} + +// skipJSONValue reads and discards one complete JSON value. +func skipJSONValue(dec *json.Decoder) error { + t, err := dec.Token() + if err != nil { + return err + } + d, ok := t.(json.Delim) + if !ok { + return nil // scalar + } + return skipJSONDelimContent(dec, d) +} + +// skipJSONDelimContent skips content after an opening delimiter ('{' or '[') was consumed. +func skipJSONDelimContent(dec *json.Decoder, open json.Delim) error { + for dec.More() { + if open == '{' { + if _, err := dec.Token(); err != nil { // skip key + return err + } + } + if err := skipJSONValue(dec); err != nil { + return err + } + } + _, err := dec.Token() // closing delimiter + return err +} + +// extractYAMLKeyOrder extracts key order from the first record in a YAML sequence. +func extractYAMLKeyOrder(data []byte) (*keyOrderInfo, error) { + var records []yaml.MapSlice + if err := yaml.Unmarshal(data, &records); err != nil { + return nil, err + } + if len(records) == 0 { + return &keyOrderInfo{nested: make(map[string]*keyOrderInfo)}, nil + } + return buildYAMLKeyOrder(records[0]), nil +} + +// buildYAMLKeyOrder builds keyOrderInfo from a yaml.MapSlice. +func buildYAMLKeyOrder(ms yaml.MapSlice) *keyOrderInfo { + info := &keyOrderInfo{nested: make(map[string]*keyOrderInfo)} + for _, item := range ms { + key := fmt.Sprintf("%v", item.Key) + info.keys = append(info.keys, key) - return headers, records, nil + switch v := item.Value.(type) { + case yaml.MapSlice: + info.nested[key] = buildYAMLKeyOrder(v) + case []any: + if len(v) > 0 { + if ms2, ok := v[0].(yaml.MapSlice); ok { + info.nested[key] = buildYAMLKeyOrder(ms2) + } + } + } + } + return info } -// inferHeaders infers CSVPP headers from JSON/YAML data structure. +// inferHeaders infers CSVPP headers from data using the provided key order. +// // Header inference rules: // - string → SimpleField // - []string → ArrayField // - map[string]any → StructuredField // - []map[string]any → ArrayStructuredField -func inferHeaders(data []map[string]any) []*csvpp.ColumnHeader { +func inferHeaders(data []map[string]any, order *keyOrderInfo) []*csvpp.ColumnHeader { if len(data) == 0 { return nil } - // Collect all unique keys from all records (first record defines order) - keyOrder := collectKeyOrder(data[0]) keyTypes := make(map[string]csvpp.FieldKind) keyComponents := make(map[string][]*csvpp.ColumnHeader) @@ -80,15 +311,20 @@ func inferHeaders(data []map[string]any) []*csvpp.ColumnHeader { } } - // Build headers maintaining key order - headers := make([]*csvpp.ColumnHeader, 0, len(keyOrder)) - for _, key := range keyOrder { + // Build headers using preserved key order + headers := make([]*csvpp.ColumnHeader, 0, len(order.keys)) + for _, key := range order.keys { + components := keyComponents[key] + if nestedOrder, ok := order.nested[key]; ok && len(components) > 0 { + components = reorderComponents(components, nestedOrder) + } + header := &csvpp.ColumnHeader{ Name: key, Kind: keyTypes[key], ArrayDelimiter: csvpp.DefaultArrayDelimiter, ComponentDelimiter: csvpp.DefaultComponentDelimiter, - Components: keyComponents[key], + Components: components, } headers = append(headers, header) } @@ -96,14 +332,20 @@ func inferHeaders(data []map[string]any) []*csvpp.ColumnHeader { return headers } -// collectKeyOrder returns keys in iteration order (Go 1.12+ maps have random order). -// Uses first record to determine key order. -func collectKeyOrder(record map[string]any) []string { - keys := make([]string, 0, len(record)) - for key := range record { - keys = append(keys, key) +// reorderComponents reorders component headers according to the key order info. +func reorderComponents(components []*csvpp.ColumnHeader, order *keyOrderInfo) []*csvpp.ColumnHeader { + compMap := make(map[string]*csvpp.ColumnHeader, len(components)) + for _, c := range components { + compMap[c.Name] = c + } + + ordered := make([]*csvpp.ColumnHeader, 0, len(order.keys)) + for _, key := range order.keys { + if c, ok := compMap[key]; ok { + ordered = append(ordered, c) + } } - return keys + return ordered } // inferFieldKind determines the FieldKind from a value. @@ -134,6 +376,8 @@ func inferFieldKind(value any) (csvpp.FieldKind, []*csvpp.ColumnHeader) { } // inferComponentHeaders creates headers for structured field components. +// Note: The returned order may be non-deterministic (map iteration). +// Callers should use reorderComponents to fix the order. func inferComponentHeaders(m map[string]any) []*csvpp.ColumnHeader { headers := make([]*csvpp.ColumnHeader, 0, len(m)) for key, value := range m { diff --git a/cmd/csvpp/internal/converter/decode_test.go b/cmd/csvpp/internal/converter/decode_test.go index c2f22b7..9a9638c 100644 --- a/cmd/csvpp/internal/converter/decode_test.go +++ b/cmd/csvpp/internal/converter/decode_test.go @@ -21,7 +21,7 @@ func TestFromJSON(t *testing.T) { wantErr bool }{ { - name: "success: simple fields", + name: "success: simple fields with preserved order", input: `[{"name":"Alice","age":"30"},{"name":"Bob","age":"25"}]`, wantHeaders: []*csvpp.ColumnHeader{ {Name: "name", Kind: csvpp.SimpleField, ArrayDelimiter: '~', ComponentDelimiter: '^'}, @@ -31,10 +31,9 @@ func TestFromJSON(t *testing.T) { {{Value: "Alice"}, {Value: "30"}}, {{Value: "Bob"}, {Value: "25"}}, }, - wantErr: false, }, { - name: "success: array field", + name: "success: array field with preserved order", input: `[{"name":"Alice","phones":["111","222"]},{"name":"Bob","phones":["333"]}]`, wantHeaders: []*csvpp.ColumnHeader{ {Name: "name", Kind: csvpp.SimpleField, ArrayDelimiter: '~', ComponentDelimiter: '^'}, @@ -44,14 +43,12 @@ func TestFromJSON(t *testing.T) { {{Value: "Alice"}, {Values: []string{"111", "222"}}}, {{Value: "Bob"}, {Values: []string{"333"}}}, }, - wantErr: false, }, { name: "success: empty array", input: `[]`, wantHeaders: nil, wantRecords: nil, - wantErr: false, }, { name: "error: invalid json", @@ -78,14 +75,12 @@ func TestFromJSON(t *testing.T) { return } - // Compare headers (ignore order since JSON doesn't preserve it) - if len(headers) != len(tt.wantHeaders) { - t.Errorf("headers count mismatch: want %d, got %d", len(tt.wantHeaders), len(headers)) + if diff := cmp.Diff(tt.wantHeaders, headers); diff != "" { + t.Errorf("headers mismatch (-want +got):\n%s", diff) } - // Compare records count - if len(records) != len(tt.wantRecords) { - t.Errorf("records count mismatch: want %d, got %d", len(tt.wantRecords), len(records)) + if diff := cmp.Diff(tt.wantRecords, records); diff != "" { + t.Errorf("records mismatch (-want +got):\n%s", diff) } }) } @@ -102,7 +97,7 @@ func TestFromYAML(t *testing.T) { wantErr bool }{ { - name: "success: simple fields", + name: "success: simple fields with preserved order", input: `- name: Alice age: "30" - name: Bob @@ -116,14 +111,12 @@ func TestFromYAML(t *testing.T) { {{Value: "Alice"}, {Value: "30"}}, {{Value: "Bob"}, {Value: "25"}}, }, - wantErr: false, }, { name: "success: empty array", input: `[]`, wantHeaders: nil, wantRecords: nil, - wantErr: false, }, { name: "error: invalid yaml", @@ -150,32 +143,12 @@ func TestFromYAML(t *testing.T) { return } - // Compare headers count - if len(headers) != len(tt.wantHeaders) { - t.Errorf("headers count mismatch: want %d, got %d", len(tt.wantHeaders), len(headers)) + if diff := cmp.Diff(tt.wantHeaders, headers); diff != "" { + t.Errorf("headers mismatch (-want +got):\n%s", diff) } - // Compare headers by name (order not guaranteed due to map iteration) - if len(tt.wantHeaders) > 0 { - headerMap := make(map[string]*csvpp.ColumnHeader) - for _, h := range headers { - headerMap[h.Name] = h - } - for _, want := range tt.wantHeaders { - got, ok := headerMap[want.Name] - if !ok { - t.Errorf("header %q not found", want.Name) - continue - } - if got.Kind != want.Kind { - t.Errorf("header %q kind mismatch: want %v, got %v", want.Name, want.Kind, got.Kind) - } - } - } - - // Compare records count - if len(records) != len(tt.wantRecords) { - t.Errorf("records count mismatch: want %d, got %d", len(tt.wantRecords), len(records)) + if diff := cmp.Diff(tt.wantRecords, records); diff != "" { + t.Errorf("records mismatch (-want +got):\n%s", diff) } }) } @@ -191,29 +164,30 @@ func TestFromJSONStructuredField(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - // Find geo header - var geoHeader *csvpp.ColumnHeader - for _, h := range headers { - if h.Name == "geo" { - geoHeader = h - break - } - } - - if geoHeader == nil { - t.Fatal("geo header not found") + wantHeaders := []*csvpp.ColumnHeader{ + {Name: "name", Kind: csvpp.SimpleField, ArrayDelimiter: '~', ComponentDelimiter: '^'}, + { + Name: "geo", Kind: csvpp.StructuredField, ArrayDelimiter: '~', ComponentDelimiter: '^', + Components: []*csvpp.ColumnHeader{ + {Name: "lat", Kind: csvpp.SimpleField, ArrayDelimiter: '~', ComponentDelimiter: '^'}, + {Name: "lon", Kind: csvpp.SimpleField, ArrayDelimiter: '~', ComponentDelimiter: '^'}, + }, + }, } - if geoHeader.Kind != csvpp.StructuredField { - t.Errorf("geo kind mismatch: want StructuredField, got %v", geoHeader.Kind) + if diff := cmp.Diff(wantHeaders, headers); diff != "" { + t.Errorf("headers mismatch (-want +got):\n%s", diff) } - if len(geoHeader.Components) != 2 { - t.Errorf("geo components count mismatch: want 2, got %d", len(geoHeader.Components)) + wantRecords := [][]*csvpp.Field{ + { + {Value: "Alice"}, + {Components: []*csvpp.Field{{Value: "34.05"}, {Value: "-118.24"}}}, + }, } - if len(records) != 1 { - t.Errorf("records count mismatch: want 1, got %d", len(records)) + if diff := cmp.Diff(wantRecords, records); diff != "" { + t.Errorf("records mismatch (-want +got):\n%s", diff) } } @@ -227,29 +201,33 @@ func TestFromJSONArrayStructuredField(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - // Find addresses header - var addrHeader *csvpp.ColumnHeader - for _, h := range headers { - if h.Name == "addresses" { - addrHeader = h - break - } - } - - if addrHeader == nil { - t.Fatal("addresses header not found") + wantHeaders := []*csvpp.ColumnHeader{ + {Name: "name", Kind: csvpp.SimpleField, ArrayDelimiter: '~', ComponentDelimiter: '^'}, + { + Name: "addresses", Kind: csvpp.ArrayStructuredField, ArrayDelimiter: '~', ComponentDelimiter: '^', + Components: []*csvpp.ColumnHeader{ + {Name: "street", Kind: csvpp.SimpleField, ArrayDelimiter: '~', ComponentDelimiter: '^'}, + {Name: "city", Kind: csvpp.SimpleField, ArrayDelimiter: '~', ComponentDelimiter: '^'}, + }, + }, } - if addrHeader.Kind != csvpp.ArrayStructuredField { - t.Errorf("addresses kind mismatch: want ArrayStructuredField, got %v", addrHeader.Kind) + if diff := cmp.Diff(wantHeaders, headers); diff != "" { + t.Errorf("headers mismatch (-want +got):\n%s", diff) } - if len(addrHeader.Components) != 2 { - t.Errorf("addresses components count mismatch: want 2, got %d", len(addrHeader.Components)) + wantRecords := [][]*csvpp.Field{ + { + {Value: "Alice"}, + {Components: []*csvpp.Field{ + {Components: []*csvpp.Field{{Value: "123 Main"}, {Value: "LA"}}}, + {Components: []*csvpp.Field{{Value: "456 Oak"}, {Value: "NY"}}}, + }}, + }, } - if len(records) != 1 { - t.Errorf("records count mismatch: want 1, got %d", len(records)) + if diff := cmp.Diff(wantRecords, records); diff != "" { + t.Errorf("records mismatch (-want +got):\n%s", diff) } } From f42bf94890b6d2405a13f76192d44db710968771 Mon Sep 17 00:00:00 2001 From: osamingo <1390409+osamingo@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:34:15 +0900 Subject: [PATCH 4/4] refactor(cmd/csvppvet): remove shadow analyzer and fix idiomatic short variable declarations - Remove shadow.Analyzer from csvppvet as err shadowing is idiomatic Go - Replace `if err = ` with `if err := ` in golden_test.go for consistency Co-Authored-By: Claude Opus 4.6 --- cmd/csvppvet/main.go | 3 +-- golden_test.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/csvppvet/main.go b/cmd/csvppvet/main.go index f628e7a..48ed02b 100644 --- a/cmd/csvppvet/main.go +++ b/cmd/csvppvet/main.go @@ -4,10 +4,9 @@ import ( gostyle "github.com/k1LoW/gostyle/analyzer" "golang.org/x/tools/go/analysis/multichecker" "golang.org/x/tools/go/analysis/passes/nilness" - "golang.org/x/tools/go/analysis/passes/shadow" "golang.org/x/tools/go/analysis/passes/unusedwrite" ) func main() { - multichecker.Main(append(gostyle.Analyzers, nilness.Analyzer, shadow.Analyzer, unusedwrite.Analyzer)...) + multichecker.Main(append(gostyle.Analyzers, nilness.Analyzer, unusedwrite.Analyzer)...) } diff --git a/golden_test.go b/golden_test.go index b42964a..d2e0398 100644 --- a/golden_test.go +++ b/golden_test.go @@ -102,7 +102,7 @@ func runGoldenReadTest(t *testing.T, file string) { } var tc goldenReadTest - if err = json.Unmarshal(data, &tc); err != nil { + if err := json.Unmarshal(data, &tc); err != nil { t.Fatalf("failed to parse test file: %v", err) } @@ -155,7 +155,7 @@ func runGoldenErrorTest(t *testing.T, file string) { } var tc goldenReadTest - if err = json.Unmarshal(data, &tc); err != nil { + if err := json.Unmarshal(data, &tc); err != nil { t.Fatalf("failed to parse test file: %v", err) }