diff --git a/cmd/multivendor.go b/cmd/multivendor.go index b96f926..dacb067 100644 --- a/cmd/multivendor.go +++ b/cmd/multivendor.go @@ -76,6 +76,7 @@ package cmd import ( "fmt" + "time" "github.com/spf13/viper" @@ -148,6 +149,10 @@ func InitializeMultiVendor() error { logging.Debugf("Multi-vendor cache directory: %s", cacheDir) cacheManager = vendors.NewCacheManager(cacheDir, apiRegistry) + cacheManager.SetRefreshTuning( + viper.GetInt("api.refresh_concurrency"), + time.Duration(viper.GetInt("api.refresh_timeout"))*time.Second, + ) if err := cacheManager.Initialize(); err != nil { return fmt.Errorf("failed to initialize cache manager: %w", err) diff --git a/cmd/refresh.go b/cmd/refresh.go index abde118..d9917c6 100644 --- a/cmd/refresh.go +++ b/cmd/refresh.go @@ -17,6 +17,7 @@ import ( "github.com/ravinald/wifimgr/internal/cmdutils" "github.com/ravinald/wifimgr/internal/config" "github.com/ravinald/wifimgr/internal/logging" + "github.com/ravinald/wifimgr/internal/refreshui" "github.com/ravinald/wifimgr/internal/vendors" ) @@ -148,12 +149,27 @@ func runRefreshAllSites(scope string) error { fmt.Printf("Successfully refreshed %s\n", apiFlag) } else { fmt.Printf("Refreshing cache for %d APIs...\n", len(targetAPIs)) + + // On a terminal, drive the live status board and hold log output (incl. + // the index-rebuild MAC-collision warnings) until the board tears down so + // it doesn't paint over the render. Piped/redirected output keeps the + // linear text and logs unbuffered. + interactive := refreshui.Interactive() + reporter, stopBoard := refreshui.New(targetAPIs, interactive) + release := func() {} + if interactive { + release = logging.PauseOutput() + } + var errs map[string]error if managed == nil { - errs = cacheMgr.RefreshAllAPIs(ctx) + errs = cacheMgr.RefreshAllAPIs(ctx, reporter) } else { - errs = cacheMgr.RefreshAllAPIsManaged(ctx, managed) + errs = cacheMgr.RefreshAllAPIsManaged(ctx, managed, reporter) } + stopBoard() + release() + successCount := len(targetAPIs) - len(errs) fmt.Printf("\nRefreshed %d/%d APIs successfully\n", successCount, len(targetAPIs)) if len(errs) > 0 { diff --git a/docs/config-minimal-sample.json b/docs/config-minimal-sample.json index cc4313a..6ecbecc 100644 --- a/docs/config-minimal-sample.json +++ b/docs/config-minimal-sample.json @@ -13,7 +13,8 @@ "url": "https://api.mist.com/api/v1", "credentials": { "org_id": "your-org-uuid-here" - } + }, + "sync_type": ["ap"] } } } diff --git a/docs/configuration.md b/docs/configuration.md index d676162..d50e9f1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -456,7 +456,8 @@ The main configuration file (`~/.config/wifimgr/wifimgr-config.json`) supports t "rate_limit": 5000, "results_limit": 100, "cache_ttl": 86400, - "connection_timeout": 5 + "connection_timeout": 5, + "sync_type": ["ap", "switch", "gateway"] } }, "display": { @@ -502,6 +503,24 @@ instead of hanging until the request timeout. A dead host now surfaces as `unhealthy` / `connection failure` in `show api status` within `connection_timeout` seconds rather than ~30s. +### Sync Type + +`sync_type` is a per-API list declaring which device types a refresh collects: +any of `ap`, `switch`, `gateway`. Site attributes (name, address, timezone) +always sync. + +- **Omitted or `[]`:** site attributes only — no device inventory, configs, + statuses, or BSSIDs. +- **`["ap"]`:** APs only, with their statuses and BSSIDs. +- **`["ap", "switch", "gateway"]`:** all device types. + +Statuses are fetched only when at least one device type is listed; BSSIDs only +when `ap` is listed. Org-level templates, profiles, and WLANs always sync. + +> **Upgrade note:** earlier versions collected all three device types +> unconditionally. An API without `sync_type` now syncs site attributes only — +> add `sync_type` to keep collecting devices. + ### Accessing Configuration Values **Direct Viper Access:** diff --git a/docs/multi-vendor/configuration.md b/docs/multi-vendor/configuration.md index f0ac6a9..dd96fe3 100644 --- a/docs/multi-vendor/configuration.md +++ b/docs/multi-vendor/configuration.md @@ -22,7 +22,8 @@ API connections are defined with user-chosen labels in the main config file: }, "rate_limit": 5000, "results_limit": 100, - "cache_ttl": 86400 + "cache_ttl": 86400, + "sync_type": ["ap", "switch", "gateway"] }, "mist-lab": { "vendor": "mist", @@ -31,7 +32,8 @@ API connections are defined with user-chosen labels in the main config file: "org_id": "xyz-789-uvw", "api_key": "..." }, - "cache_ttl": 0 + "cache_ttl": 0, + "sync_type": ["ap"] }, "meraki-corp": { "vendor": "meraki", @@ -75,6 +77,27 @@ API connections are defined with user-chosen labels in the main config file: | `rate_limit` | No | Requests per minute (vendor default if omitted) | | `results_limit` | No | Max results per API call | | `cache_ttl` | No | Cache TTL in seconds (see below) | +| `sync_type` | No | Device types to collect: any of `ap`, `switch`, `gateway` (see below) | + +### Sync Type Configuration + +`sync_type` declares which device types a refresh collects for this API. It's a +list drawn from `ap`, `switch`, and `gateway`. Site attributes (name, address, +timezone) always sync regardless. + +| Value | Behavior | +|-------|----------| +| Omitted | **Site attributes only — no devices.** | +| `[]` | Same as omitted (sites only) | +| `["ap"]` | Collects APs only (plus AP statuses and BSSIDs) | +| `["ap", "switch", "gateway"]` | Collects all device types | + +Device statuses are fetched only when at least one device type is listed; BSSIDs +only when `ap` is listed. Org-level templates, profiles, and WLANs always sync. + +> **Upgrade note:** earlier versions collected all three device types +> unconditionally. An API without `sync_type` now syncs **site attributes only**. +> Add `sync_type` to keep collecting devices. ### Cache TTL Configuration diff --git a/go.mod b/go.mod index 1d4486f..036d74d 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/colorprofile v0.4.1 // indirect + github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/charmbracelet/x/ansi v0.11.6 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect diff --git a/go.sum b/go.sum index fe6feb2..6573f1f 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlv github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk= github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8= diff --git a/internal/config/api_config.go b/internal/config/api_config.go index 8c3d4c2..57ca3ea 100644 --- a/internal/config/api_config.go +++ b/internal/config/api_config.go @@ -103,6 +103,9 @@ func BuildAPIConfigsFromViper() (map[string]*vendors.APIConfig, []ValidationWarn } } + syncTypes, syncWarnings := parseSyncTypes(label, nested) + warnings = append(warnings, syncWarnings...) + config := &vendors.APIConfig{ Label: label, Vendor: vendor, @@ -112,6 +115,7 @@ func BuildAPIConfigsFromViper() (map[string]*vendors.APIConfig, []ValidationWarn ResultsLimit: getIntFromMap(nested, "results_limit"), CacheTTL: getCacheTTLFromMap(nested), ConnectTimeout: resolveConnectTimeout(nested), + SyncTypes: syncTypes, } // Apply vendor-specific defaults @@ -353,6 +357,62 @@ func getStringFromMap(m map[string]interface{}, key string) string { return "" } +// validSyncTypes is the set of device types an API may declare under sync_type. +var validSyncTypes = map[string]bool{"ap": true, "switch": true, "gateway": true} + +// parseSyncTypes reads the optional sync_type list for an API, normalizing each +// entry to lowercase, dropping blanks and duplicates. Unknown entries are +// skipped with a warning rather than failing the load. A missing key returns a +// nil slice, which the refresh path treats as "sites only". +func parseSyncTypes(label string, nested map[string]interface{}) ([]string, []ValidationWarning) { + raw, ok := nested["sync_type"] + if !ok { + return nil, nil + } + + // JSON/YAML decode lists as []interface{}; tolerate []string too. + var items []interface{} + switch v := raw.(type) { + case []interface{}: + items = v + case []string: + for _, s := range v { + items = append(items, s) + } + default: + return nil, []ValidationWarning{{ + Level: "api", + API: label, + Message: fmt.Sprintf("API %q has invalid 'sync_type' (expected a list of 'ap', 'switch', 'gateway')", label), + }} + } + + var result []string + var warnings []ValidationWarning + seen := make(map[string]bool) + for _, item := range items { + s, ok := item.(string) + if !ok { + continue + } + t := strings.ToLower(strings.TrimSpace(s)) + if t == "" || seen[t] { + continue + } + if !validSyncTypes[t] { + warnings = append(warnings, ValidationWarning{ + Level: "api", + API: label, + Message: fmt.Sprintf("API %q sync_type has unknown device type %q (allowed: ap, switch, gateway)", label, s), + }) + continue + } + seen[t] = true + result = append(result, t) + } + return result, warnings +} + // getIntFromMap safely extracts an int value from a map[string]interface{} func getIntFromMap(m map[string]interface{}, key string) int { if v, ok := m[key]; ok { diff --git a/internal/config/sync_type_test.go b/internal/config/sync_type_test.go new file mode 100644 index 0000000..8930e49 --- /dev/null +++ b/internal/config/sync_type_test.go @@ -0,0 +1,70 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestParseSyncTypes(t *testing.T) { + cases := []struct { + name string + nested map[string]interface{} + want []string + wantWarn bool + }{ + { + name: "absent yields nil", + nested: map[string]interface{}{}, + want: nil, + }, + { + name: "empty list yields empty", + nested: map[string]interface{}{"sync_type": []interface{}{}}, + want: nil, + }, + { + name: "single type", + nested: map[string]interface{}{"sync_type": []interface{}{"ap"}}, + want: []string{"ap"}, + }, + { + name: "normalizes case and whitespace", + nested: map[string]interface{}{"sync_type": []interface{}{"AP", " switch "}}, + want: []string{"ap", "switch"}, + }, + { + name: "dedupes", + nested: map[string]interface{}{"sync_type": []interface{}{"ap", "ap", "gateway"}}, + want: []string{"ap", "gateway"}, + }, + { + name: "unknown type warns and is skipped", + nested: map[string]interface{}{"sync_type": []interface{}{"bogus", "ap"}}, + want: []string{"ap"}, + wantWarn: true, + }, + { + name: "wrong type warns", + nested: map[string]interface{}{"sync_type": "ap"}, + want: nil, + wantWarn: true, + }, + { + name: "tolerates []string", + nested: map[string]interface{}{"sync_type": []string{"gateway"}}, + want: []string{"gateway"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, warnings := parseSyncTypes("test-api", c.nested) + if !reflect.DeepEqual(got, c.want) { + t.Errorf("got %v, want %v", got, c.want) + } + if (len(warnings) > 0) != c.wantWarn { + t.Errorf("warnings = %v, wantWarn = %v", warnings, c.wantWarn) + } + }) + } +} diff --git a/internal/logging/logging.go b/internal/logging/logging.go index 5eaaf90..f7368e3 100644 --- a/internal/logging/logging.go +++ b/internal/logging/logging.go @@ -1,6 +1,7 @@ package logging import ( + "bytes" "fmt" "io" "os" @@ -271,6 +272,27 @@ func Errorf(format string, args ...interface{}) { defaultLogger.Errorf(format, args...) } +// PauseOutput redirects the logger to an in-memory buffer and returns a release +// func. While paused, log lines accumulate instead of printing; release restores +// the prior destination and flushes everything buffered to it. A full-screen +// renderer (the refresh board) calls this so concurrent warnings — like MAC +// collisions during the index rebuild — don't corrupt the live display, then +// surface as a clean trailing block once the board tears down. +// +// Not reentrant: a second PauseOutput before release loses the first buffer's +// destination. The refresh path is the only caller and pauses once. +func PauseOutput() (release func()) { + prev := defaultLogger.Out + buf := &bytes.Buffer{} + defaultLogger.SetOutput(buf) + return func() { + defaultLogger.SetOutput(prev) + if buf.Len() > 0 { + _, _ = prev.Write(buf.Bytes()) + } + } +} + // ConfigureLogger configures the logger with a simplified interface // This is an alias for ConfigureLogging to maintain backward compatibility with existing code func ConfigureLogger(level string, toStdout bool) error { diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go index 9a94499..df8f833 100644 --- a/internal/logging/logging_test.go +++ b/internal/logging/logging_test.go @@ -268,3 +268,24 @@ func TestNameFormatting(t *testing.T) { t.Errorf("FormatOrgID(%s) = %s, want %s", unknownOrgID, formattedOrg, unknownOrgID) } } + +func TestPauseOutputBuffersThenFlushes(t *testing.T) { + originalLogger := defaultLogger + defer func() { defaultLogger = originalLogger }() + + defaultLogger = logrus.New() + var sink bytes.Buffer + defaultLogger.SetOutput(&sink) + defaultLogger.SetLevel(logrus.InfoLevel) + + release := PauseOutput() + Warnf("collision while paused") + if sink.Len() != 0 { + t.Fatalf("paused log leaked to sink: %q", sink.String()) + } + + release() + if !strings.Contains(sink.String(), "collision while paused") { + t.Fatalf("release did not flush buffered log; sink=%q", sink.String()) + } +} diff --git a/internal/refreshui/board.go b/internal/refreshui/board.go new file mode 100644 index 0000000..699960d --- /dev/null +++ b/internal/refreshui/board.go @@ -0,0 +1,236 @@ +package refreshui + +import ( + "fmt" + "os" + "sync" + "time" + + "github.com/charmbracelet/bubbles/progress" + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +// barWidth is the cell width of the determinate progress bar and the dotted +// placeholder shown before a counted stage begins, so a row's geometry doesn't +// jump when it switches between them. +const barWidth = 18 + +type rowState int + +const ( + rowActive rowState = iota + rowDone + rowFailed +) + +// row is the live state of one API's refresh, mutated by board messages. +type row struct { + label string + state rowState + stage string + done int + total int + dur time.Duration + failErr error +} + +// boardModel is the bubbletea model: a fixed, ordered set of rows repainted in +// place. The row set never grows after construction, so the rendered line count +// is stable and bubbletea can repaint without scrolling. +type boardModel struct { + order []string + rows map[string]*row + spin spinner.Model + bar progress.Model + labelW int +} + +func newBoardModel(labels []string) *boardModel { + rows := make(map[string]*row, len(labels)) + labelW := 0 + for _, l := range labels { + rows[l] = &row{label: l, stage: "waiting"} + if len(l) > labelW { + labelW = len(l) + } + } + sp := spinner.New(spinner.WithSpinner(spinner.MiniDot)) + return &boardModel{ + order: labels, + rows: rows, + spin: sp, + bar: progress.New(progress.WithWidth(barWidth), progress.WithoutPercentage()), + labelW: labelW, + } +} + +func (m *boardModel) Init() tea.Cmd { return m.spin.Tick } + +func (m *boardModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + // Let the operator escape a wedged render; the background refresh keeps + // running and its summary still prints once teardown returns. + if msg.Type == tea.KeyCtrlC { + return m, tea.Quit + } + case spinner.TickMsg: + var cmd tea.Cmd + m.spin, cmd = m.spin.Update(msg) + return m, cmd + case startMsg: + if r := m.rows[msg.label]; r != nil { + r.state = rowActive + r.stage = "starting" + } + case stageMsg: + if r := m.rows[msg.label]; r != nil { + r.stage = msg.stage + r.done, r.total = 0, 0 + } + case progressMsg: + if r := m.rows[msg.label]; r != nil { + r.done, r.total = msg.done, msg.total + } + case doneMsg: + if r := m.rows[msg.label]; r != nil { + r.state = rowDone + r.dur = msg.dur + } + case errMsg: + if r := m.rows[msg.label]; r != nil { + r.state = rowFailed + r.failErr = msg.err + } + } + return m, nil +} + +var ( + doneStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#00FF00")).Bold(true) + failStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#FF0000")).Bold(true) + labelStyle = lipgloss.NewStyle().Bold(true) + dimStyle = lipgloss.NewStyle().Faint(true) +) + +func (m *boardModel) View() string { + var b []byte + for _, label := range m.order { + r := m.rows[label] + name := labelStyle.Render(fmt.Sprintf("%-*s", m.labelW, label)) + b = append(b, '[') + b = append(b, name...) + b = append(b, ']', ' ') + switch r.state { + case rowDone: + b = append(b, doneStyle.Render("✔")...) + b = append(b, ' ', ' ') + b = append(b, m.bar.ViewAs(1)...) + b = append(b, fmt.Sprintf(" Started %dms", r.dur.Milliseconds())...) + case rowFailed: + b = append(b, failStyle.Render("✖")...) + b = append(b, ' ', ' ') + b = append(b, dimStyle.Render(dots(barWidth))...) + b = append(b, fmt.Sprintf(" Failed: %v", r.failErr)...) + default: // rowActive + b = append(b, m.spin.View()...) + b = append(b, ' ') + if r.total > 0 { + b = append(b, m.bar.ViewAs(float64(r.done)/float64(r.total))...) + b = append(b, fmt.Sprintf(" %s %d/%d", r.stage, r.done, r.total)...) + } else { + b = append(b, dimStyle.Render(dots(barWidth))...) + b = append(b, ' ', ' ') + b = append(b, r.stage...) + } + } + b = append(b, '\n') + } + return string(b) +} + +// dots returns n dot runes — the indeterminate stand-in for the bar before a +// counted stage supplies a fraction. +func dots(n int) string { + out := make([]byte, n) + for i := range out { + out[i] = '.' + } + return string(out) +} + +// board owns the bubbletea program lifecycle and hands out a Reporter that feeds +// it. start launches the render loop; stop quits it and waits for the final +// frame to settle. +type board struct { + prog *tea.Program + done chan struct{} + once sync.Once +} + +func newBoard(labels []string) *board { + m := newBoardModel(labels) + p := tea.NewProgram(m, tea.WithOutput(os.Stdout)) + return &board{prog: p, done: make(chan struct{})} +} + +func (b *board) start() { + go func() { + _, _ = b.prog.Run() + close(b.done) + }() +} + +func (b *board) reporter() Reporter { return &boardReporter{prog: b.prog} } + +func (b *board) stop() { + b.once.Do(func() { + b.prog.Quit() + <-b.done + }) +} + +// message types carry reporter events to the model over the program's queue. +type ( + startMsg struct{ label, vendor, site string } + stageMsg struct{ label, stage string } + progressMsg struct { + label string + done, total int + } + doneMsg struct { + label string + dur time.Duration + } + errMsg struct { + label string + err error + } +) + +// boardReporter translates Reporter calls into program messages. Send is +// goroutine-safe, so the concurrent per-API refreshes share one reporter. +type boardReporter struct{ prog *tea.Program } + +func (b *boardReporter) APIStart(label, vendor, site string) { + b.prog.Send(startMsg{label: label, vendor: vendor, site: site}) +} +func (b *boardReporter) Stage(label, stage string) { + b.prog.Send(stageMsg{label: label, stage: stage}) +} + +// StageResult is unused by the board — the next Stage or the determinate count +// supersedes it — but the linear reporter needs it, so the interface keeps it. +func (b *boardReporter) StageResult(string, string) {} + +func (b *boardReporter) Progress(label string, done, total int) { + b.prog.Send(progressMsg{label: label, done: done, total: total}) +} +func (b *boardReporter) APIDone(label string, dur time.Duration) { + b.prog.Send(doneMsg{label: label, dur: dur}) +} +func (b *boardReporter) APIError(label string, err error) { + b.prog.Send(errMsg{label: label, err: err}) +} diff --git a/internal/refreshui/board_test.go b/internal/refreshui/board_test.go new file mode 100644 index 0000000..14bc8d6 --- /dev/null +++ b/internal/refreshui/board_test.go @@ -0,0 +1,85 @@ +package refreshui + +import ( + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/charmbracelet/lipgloss" + "github.com/muesli/termenv" +) + +// TestMain strips ANSI styling so View assertions match on plain text. +func TestMain(m *testing.M) { + lipgloss.SetColorProfile(termenv.Ascii) + os.Exit(m.Run()) +} + +func TestBoardModelActiveCountedStage(t *testing.T) { + m := newBoardModel([]string{"mist", "meraki"}) + + if m.labelW != len("meraki") { + t.Fatalf("labelW = %d, want %d", m.labelW, len("meraki")) + } + + m.Update(startMsg{label: "mist", vendor: "mist"}) + m.Update(stageMsg{label: "mist", stage: "AP configs"}) + m.Update(progressMsg{label: "mist", done: 143, total: 231}) + + r := m.rows["mist"] + if r.state != rowActive || r.done != 143 || r.total != 231 { + t.Fatalf("row = %+v, want active 143/231", r) + } + if v := m.View(); !strings.Contains(v, "AP configs 143/231") { + t.Fatalf("view missing counted stage:\n%s", v) + } +} + +func TestBoardModelStageResetsCount(t *testing.T) { + m := newBoardModel([]string{"mist"}) + m.Update(progressMsg{label: "mist", done: 5, total: 10}) + m.Update(stageMsg{label: "mist", stage: "Fetching WLANs"}) + + if r := m.rows["mist"]; r.done != 0 || r.total != 0 { + t.Fatalf("count not reset on new stage: %+v", r) + } +} + +func TestBoardModelDoneAndError(t *testing.T) { + m := newBoardModel([]string{"mist", "meraki"}) + + m.Update(doneMsg{label: "mist", dur: 952 * time.Millisecond}) + m.Update(errMsg{label: "meraki", err: errors.New("login timeout")}) + + if r := m.rows["mist"]; r.state != rowDone || r.dur != 952*time.Millisecond { + t.Fatalf("mist row = %+v, want done 952ms", r) + } + if r := m.rows["meraki"]; r.state != rowFailed { + t.Fatalf("meraki row = %+v, want failed", r) + } + + v := m.View() + if !strings.Contains(v, "Started 952ms") { + t.Fatalf("view missing done summary:\n%s", v) + } + if !strings.Contains(v, "Failed: login timeout") { + t.Fatalf("view missing error summary:\n%s", v) + } +} + +func TestBoardModelLineCountStable(t *testing.T) { + // The rendered line count must equal the API count regardless of state, so + // bubbletea repaints in place instead of scrolling. + m := newBoardModel([]string{"a", "b", "c"}) + want := 3 + if got := strings.Count(m.View(), "\n"); got != want { + t.Fatalf("initial line count = %d, want %d", got, want) + } + m.Update(doneMsg{label: "a", dur: time.Second}) + m.Update(errMsg{label: "b", err: errors.New("x")}) + if got := strings.Count(m.View(), "\n"); got != want { + t.Fatalf("post-update line count = %d, want %d", got, want) + } +} diff --git a/internal/refreshui/reporter.go b/internal/refreshui/reporter.go new file mode 100644 index 0000000..0e54f83 --- /dev/null +++ b/internal/refreshui/reporter.go @@ -0,0 +1,129 @@ +// Package refreshui renders the progress of a multi-API cache refresh. It offers +// two reporters behind one interface: a linear reporter that prints one whole +// line per stage (safe to pipe, the default), and a live "status board" that +// repaints a row per API in place — the docker-compose look — when stdout is an +// interactive terminal. +package refreshui + +import ( + "fmt" + "io" + "os" + "sync" + "time" + + "golang.org/x/term" +) + +// Reporter receives progress events for a cache refresh. doRefreshAPI drives the +// per-stage calls; the orchestrator marks terminal success/failure. A single +// Reporter is shared across the concurrent per-API goroutines, so every +// implementation must be safe for concurrent use. +type Reporter interface { + APIStart(label, vendor, siteID string) // a per-API refresh began + Stage(label, stage string) // a named step started, e.g. "Fetching sites" + StageResult(label, summary string) // the step closed with a short summary, e.g. "5 sites" + Progress(label string, done, total int) // determinate count within the active step (config fetch) + APIDone(label string, dur time.Duration) + APIError(label string, err error) +} + +// New returns a Reporter and a teardown func. When interactive, it starts the +// live board (one repainting row per label) and the teardown paints the final +// frame and releases the terminal; otherwise it returns the linear reporter and +// a no-op teardown. Always call the teardown — defer it. +func New(labels []string, interactive bool) (Reporter, func()) { + if !interactive || len(labels) == 0 { + return NewLinear(), func() {} + } + b := newBoard(labels) + b.start() + return b.reporter(), b.stop +} + +// Interactive reports whether stdout can host the live board: a real terminal +// that isn't the dumb fallback. A pipe, redirect, or TERM=dumb falls back to +// linear text so captured output stays free of cursor-control escapes. +func Interactive() bool { + if os.Getenv("TERM") == "dumb" { + return false + } + return term.IsTerminal(int(os.Stdout.Fd())) // #nosec G115 -- fds are small non-negative ints +} + +// sharedLinear backs Resolve so callers that pass a nil Reporter (single-API +// refreshes) still get linear output without each allocating one. It is +// stateless beyond a per-label pending map guarded by its own mutex. +var sharedLinear = NewLinear() + +// Resolve returns r, or the shared linear reporter when r is nil. doRefreshAPI +// uses it so a refresh invoked without a reporter keeps the original behavior. +func Resolve(r Reporter) Reporter { + if r == nil { + return sharedLinear + } + return r +} + +// linearReporter prints one whole line per stage. It buffers the in-progress +// stage per label and emits the line atomically on StageResult, so concurrent +// per-API goroutines can't splice their output mid-line the way bare fmt.Printf +// did. Progress is a no-op — counts surface only in the final stage summary. +type linearReporter struct { + mu sync.Mutex + w io.Writer + pending map[string]string // label -> active stage text awaiting its summary +} + +// NewLinear returns a linear reporter writing to stdout. +func NewLinear() Reporter { return NewLinearWriter(os.Stdout) } + +// NewLinearWriter returns a linear reporter writing to w. Used in tests to +// capture output. +func NewLinearWriter(w io.Writer) Reporter { + return &linearReporter{w: w, pending: make(map[string]string)} +} + +func (l *linearReporter) APIStart(label, vendor, siteID string) { + l.mu.Lock() + defer l.mu.Unlock() + if siteID != "" { + _, _ = fmt.Fprintf(l.w, " [%s] Refreshing %s API (site %s)...\n", label, vendor, siteID) + return + } + _, _ = fmt.Fprintf(l.w, " [%s] Refreshing %s API...\n", label, vendor) +} + +func (l *linearReporter) Stage(label, stage string) { + l.mu.Lock() + defer l.mu.Unlock() + l.pending[label] = stage +} + +func (l *linearReporter) StageResult(label, summary string) { + l.mu.Lock() + defer l.mu.Unlock() + stage := l.pending[label] + delete(l.pending, label) + if summary == "" { + _, _ = fmt.Fprintf(l.w, " %s...\n", stage) + return + } + _, _ = fmt.Fprintf(l.w, " %s... %s\n", stage, summary) +} + +func (l *linearReporter) Progress(string, int, int) {} + +func (l *linearReporter) APIDone(label string, dur time.Duration) { + l.mu.Lock() + defer l.mu.Unlock() + _, _ = fmt.Fprintf(l.w, " [%s] Complete in %dms\n", label, dur.Milliseconds()) +} + +func (l *linearReporter) APIError(label string, err error) { + l.mu.Lock() + defer l.mu.Unlock() + // The command layer prints the aggregated error block; keep the inline trace + // terse so a piped log still shows where a refresh died. + _, _ = fmt.Fprintf(l.w, " [%s] Failed: %v\n", label, err) +} diff --git a/internal/refreshui/reporter_test.go b/internal/refreshui/reporter_test.go new file mode 100644 index 0000000..10db672 --- /dev/null +++ b/internal/refreshui/reporter_test.go @@ -0,0 +1,53 @@ +package refreshui + +import ( + "bytes" + "errors" + "testing" + "time" +) + +func TestLinearReporterStageLineIsAtomic(t *testing.T) { + var buf bytes.Buffer + r := NewLinearWriter(&buf) + + r.APIStart("mist", "mist", "") + r.Stage("mist", "Fetching sites") + r.Progress("mist", 1, 1) // no-op for linear; must not split the line + r.StageResult("mist", "5 sites") + r.APIDone("mist", 952*time.Millisecond) + + want := " [mist] Refreshing mist API...\n" + + " Fetching sites... 5 sites\n" + + " [mist] Complete in 952ms\n" + if got := buf.String(); got != want { + t.Fatalf("linear output mismatch:\n got: %q\nwant: %q", got, want) + } +} + +func TestLinearReporterSiteAndEmptyResult(t *testing.T) { + var buf bytes.Buffer + r := NewLinearWriter(&buf) + + r.APIStart("meraki", "meraki", "US-LAB-01") + r.Stage("meraki", "Skipping device configs (use 'refresh cache' to fetch)") + r.StageResult("meraki", "") + r.APIError("meraki", errors.New("login timeout")) + + want := " [meraki] Refreshing meraki API (site US-LAB-01)...\n" + + " Skipping device configs (use 'refresh cache' to fetch)...\n" + + " [meraki] Failed: login timeout\n" + if got := buf.String(); got != want { + t.Fatalf("linear output mismatch:\n got: %q\nwant: %q", got, want) + } +} + +func TestResolveNilFallsBackToLinear(t *testing.T) { + if Resolve(nil) == nil { + t.Fatal("Resolve(nil) returned nil; want shared linear reporter") + } + r := NewLinearWriter(&bytes.Buffer{}) + if Resolve(r) != r { + t.Fatal("Resolve(r) should return r unchanged") + } +} diff --git a/internal/vendors/cache_manager.go b/internal/vendors/cache_manager.go index d125d2b..6b11b24 100644 --- a/internal/vendors/cache_manager.go +++ b/internal/vendors/cache_manager.go @@ -1,15 +1,18 @@ package vendors import ( + "context" "encoding/json" "fmt" "os" "path/filepath" "sync" + "time" "github.com/ravinald/wifimgr/internal/encryption" "github.com/ravinald/wifimgr/internal/helpers" "github.com/ravinald/wifimgr/internal/logging" + "github.com/ravinald/wifimgr/internal/refreshui" ) // CacheManager manages per-API cache files and the cross-API index. @@ -31,6 +34,50 @@ type CacheManager struct { secretPwOnce sync.Once secretPw string secretPwErr error + + // refresh-all tuning, applied by SetRefreshTuning. Zero values mean + // defaults: a bounded fan-out and no per-API deadline. + refreshConcurrency int + refreshTimeout time.Duration +} + +// defaultRefreshConcurrency caps how many APIs refresh at once when no explicit +// limit is configured. Refresh-all fans out one goroutine per API; without a cap +// a large config would open that many concurrent vendor sessions at once. +const defaultRefreshConcurrency = 8 + +// SetRefreshTuning sets the refresh-all fan-out cap and the per-API timeout. +// concurrency <= 0 keeps the default cap; timeout <= 0 leaves per-API refreshes +// unbounded (so a legitimately large org isn't cut off mid-fetch). The command +// layer wires these from config. +func (c *CacheManager) SetRefreshTuning(concurrency int, timeout time.Duration) { + c.refreshConcurrency = concurrency + c.refreshTimeout = timeout +} + +// refreshLimit resolves the fan-out cap for n APIs: the configured value, else +// the default, clamped to [1, n]. +func (c *CacheManager) refreshLimit(n int) int { + limit := c.refreshConcurrency + if limit <= 0 { + limit = defaultRefreshConcurrency + } + if limit > n { + limit = n + } + if limit < 1 { + limit = 1 + } + return limit +} + +// refreshCtx derives the per-API context: a timeout-bounded child when a refresh +// timeout is configured, else the parent with a no-op cancel. +func (c *CacheManager) refreshCtx(parent context.Context) (context.Context, context.CancelFunc) { + if c.refreshTimeout <= 0 { + return parent, func() {} + } + return context.WithTimeout(parent, c.refreshTimeout) } // secretPassword resolves the password used to encrypt WLAN secrets in the @@ -64,6 +111,16 @@ type RefreshOptions struct { // managed refresh stays cheap on Meraki without discarding data a full // pass collected. nil means no managed filter: fetch every device in scope. ManagedMACs map[string]bool + + // Reporter receives progress events. nil falls back to linear stdout output + // (refreshui.Resolve), preserving the original single-API behavior. + Reporter refreshui.Reporter + + // SkipIndexRebuild suppresses the per-API cross-API index rebuild at the end + // of doRefreshAPI. The refresh-all orchestrator sets it so the index is built + // once after the batch instead of once per API — which otherwise re-scans + // every cache file N times and re-emits each MAC collision N times. + SkipIndexRebuild bool } // NewCacheManager creates a new cache manager. diff --git a/internal/vendors/cache_manager_index.go b/internal/vendors/cache_manager_index.go index f164a53..59c7b7c 100644 --- a/internal/vendors/cache_manager_index.go +++ b/internal/vendors/cache_manager_index.go @@ -30,6 +30,11 @@ func (c *CacheManager) RebuildIndex() error { return fmt.Errorf("failed to read apis directory: %w", err) } + // collisions records, per normalized MAC, the API that keeps it and the one + // that lost — deduped so a MAC owned by two APIs is reported once per rebuild, + // not once per device that re-hits it. + collisions := make(map[string]macCollision) + for _, entry := range entries { if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { continue @@ -44,13 +49,13 @@ func (c *CacheManager) RebuildIndex() error { // Index MACs for mac := range cache.Inventory.AP { - c.indexMAC(index, mac, apiLabel) + c.indexMAC(index, collisions, mac, apiLabel) } for mac := range cache.Inventory.Switch { - c.indexMAC(index, mac, apiLabel) + c.indexMAC(index, collisions, mac, apiLabel) } for mac := range cache.Inventory.Gateway { - c.indexMAC(index, mac, apiLabel) + c.indexMAC(index, collisions, mac, apiLabel) } // Index site names @@ -59,19 +64,31 @@ func (c *CacheManager) RebuildIndex() error { } } + for mac, col := range collisions { + logging.Warnf("MAC collision: %s exists in both %q and %q - keeping %q", mac, col.kept, col.dropped, col.kept) + } + c.index = index return c.saveIndex() } -// indexMAC adds a MAC to the index with collision detection. -func (c *CacheManager) indexMAC(index *CrossAPIIndex, mac, apiLabel string) { +// macCollision is one MAC claimed by two APIs: the first indexed wins (kept). +type macCollision struct { + kept string + dropped string +} + +// indexMAC adds a MAC to the index, keeping the first API to claim it. A +// conflicting claim is recorded in collisions (deduped) for the caller to log +// once, rather than printed per occurrence. +func (c *CacheManager) indexMAC(index *CrossAPIIndex, collisions map[string]macCollision, mac, apiLabel string) { normalizedMAC := NormalizeMAC(mac) if existingAPI, found := index.MACToAPI[normalizedMAC]; found { if existingAPI != apiLabel { - // MAC collision - log error but keep existing mapping - _, _ = fmt.Fprintf(os.Stderr, "ERROR MAC collision: %s exists in both %q and %q - keeping %q\n", - normalizedMAC, existingAPI, apiLabel, existingAPI) + if _, seen := collisions[normalizedMAC]; !seen { + collisions[normalizedMAC] = macCollision{kept: existingAPI, dropped: apiLabel} + } return } } diff --git a/internal/vendors/cache_manager_refresh.go b/internal/vendors/cache_manager_refresh.go index e4b290e..e392f00 100644 --- a/internal/vendors/cache_manager_refresh.go +++ b/internal/vendors/cache_manager_refresh.go @@ -9,6 +9,7 @@ import ( "github.com/ravinald/wifimgr/internal/encryption" "github.com/ravinald/wifimgr/internal/logging" + "github.com/ravinald/wifimgr/internal/refreshui" ) // wlanHasPlaintextSecret reports whether a freshly fetched WLAN carries a PSK or @@ -116,6 +117,7 @@ func (c *CacheManager) RefreshAPIWithOptions(ctx context.Context, apiLabel strin // doRefreshAPI performs the refresh. The caller holds the per-label lock. func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts RefreshOptions) error { + report := refreshui.Resolve(opts.Reporter) logging.Debugf("[cache] Starting refresh for API %s (fetchConfigs=%v)", apiLabel, opts.FetchDeviceConfigs) client, err := c.registry.GetClient(apiLabel) @@ -146,12 +148,7 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R logging.Debugf("[cache] Refreshing %s (vendor=%s, org=%s, fetchConfigs=%v, siteID=%q)", apiLabel, config.Vendor, config.Credentials["org_id"], shouldFetchConfigs, opts.SiteID) - // Progress message: Starting API refresh - if opts.SiteID != "" { - fmt.Printf(" [%s] Refreshing %s API (site %s)...\n", apiLabel, config.Vendor, opts.SiteID) - } else { - fmt.Printf(" [%s] Refreshing %s API...\n", apiLabel, config.Vendor) - } + report.APIStart(apiLabel, config.Vendor, opts.SiteID) startTime := time.Now() @@ -180,96 +177,107 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R cache.Meta.LastFailure = time.Time{} // Fetch sites - fmt.Printf(" Fetching sites...") + report.Stage(apiLabel, "Fetching sites") logging.Debugf("[cache] Fetching sites for %s", apiLabel) if sitesSvc := client.Sites(); sitesSvc != nil { sites, err := sitesSvc.List(ctx) if err != nil { - fmt.Printf(" error\n") + report.StageResult(apiLabel, "error") logging.Debugf("[cache] Failed to fetch sites for %s: %v", apiLabel, err) return fmt.Errorf("failed to fetch sites: %w", err) } - fmt.Printf(" %d sites\n", len(sites)) + report.StageResult(apiLabel, fmt.Sprintf("%d sites", len(sites))) logging.Debugf("[cache] Fetched %d sites for %s", len(sites), apiLabel) for _, site := range sites { cache.Sites.Info = append(cache.Sites.Info, *site) } } else { - fmt.Printf(" not supported\n") + report.StageResult(apiLabel, "not supported") } - // Fetch inventory + // Fetch inventory. Each device type is gated on the API's sync_type — an API + // that doesn't list a type leaves its inventory map empty (and the per-device + // config, status, and BSSID fetches below short-circuit off that). logging.Debugf("[cache] Fetching inventory for %s", apiLabel) if invSvc := client.Inventory(); invSvc != nil { // APs - fmt.Printf(" Fetching APs...") - aps, err := invSvc.List(ctx, "ap") - if err == nil { - for _, item := range aps { - if item.MAC != "" { - cache.Inventory.AP[NormalizeMAC(item.MAC)] = item + if config.ShouldSync("ap") { + report.Stage(apiLabel, "Fetching APs") + aps, err := invSvc.List(ctx, "ap") + if err == nil { + for _, item := range aps { + if item.MAC != "" { + cache.Inventory.AP[NormalizeMAC(item.MAC)] = item + } } + report.StageResult(apiLabel, fmt.Sprintf("%d devices", len(aps))) + logging.Debugf("[cache] Fetched %d APs for %s", len(aps), apiLabel) + } else { + report.StageResult(apiLabel, "error") + logging.Debugf("[cache] Failed to fetch APs for %s: %v", apiLabel, err) } - fmt.Printf(" %d devices\n", len(aps)) - logging.Debugf("[cache] Fetched %d APs for %s", len(aps), apiLabel) - } else { - fmt.Printf(" error\n") - logging.Debugf("[cache] Failed to fetch APs for %s: %v", apiLabel, err) } // Switches - fmt.Printf(" Fetching switches...") - switches, err := invSvc.List(ctx, "switch") - if err == nil { - for _, item := range switches { - if item.MAC != "" { - cache.Inventory.Switch[NormalizeMAC(item.MAC)] = item + if config.ShouldSync("switch") { + report.Stage(apiLabel, "Fetching switches") + switches, err := invSvc.List(ctx, "switch") + if err == nil { + for _, item := range switches { + if item.MAC != "" { + cache.Inventory.Switch[NormalizeMAC(item.MAC)] = item + } } + report.StageResult(apiLabel, fmt.Sprintf("%d devices", len(switches))) + logging.Debugf("[cache] Fetched %d switches for %s", len(switches), apiLabel) + } else { + report.StageResult(apiLabel, "error") + logging.Debugf("[cache] Failed to fetch switches for %s: %v", apiLabel, err) } - fmt.Printf(" %d devices\n", len(switches)) - logging.Debugf("[cache] Fetched %d switches for %s", len(switches), apiLabel) - } else { - fmt.Printf(" error\n") - logging.Debugf("[cache] Failed to fetch switches for %s: %v", apiLabel, err) } // Gateways - fmt.Printf(" Fetching gateways...") - gateways, err := invSvc.List(ctx, "gateway") - if err == nil { - for _, item := range gateways { - if item.MAC != "" { - cache.Inventory.Gateway[NormalizeMAC(item.MAC)] = item + if config.ShouldSync("gateway") { + report.Stage(apiLabel, "Fetching gateways") + gateways, err := invSvc.List(ctx, "gateway") + if err == nil { + for _, item := range gateways { + if item.MAC != "" { + cache.Inventory.Gateway[NormalizeMAC(item.MAC)] = item + } } + report.StageResult(apiLabel, fmt.Sprintf("%d devices", len(gateways))) + logging.Debugf("[cache] Fetched %d gateways for %s", len(gateways), apiLabel) + } else { + report.StageResult(apiLabel, "error") + logging.Debugf("[cache] Failed to fetch gateways for %s: %v", apiLabel, err) } - fmt.Printf(" %d devices\n", len(gateways)) - logging.Debugf("[cache] Fetched %d gateways for %s", len(gateways), apiLabel) - } else { - fmt.Printf(" error\n") - logging.Debugf("[cache] Failed to fetch gateways for %s: %v", apiLabel, err) } } - // Fetch device statuses - fmt.Printf(" Fetching device statuses...") - logging.Debugf("[cache] Fetching device statuses for %s", apiLabel) - if statusSvc := client.Statuses(); statusSvc != nil { - statuses, err := statusSvc.GetAll(ctx) - if err == nil { - cache.DeviceStatus = statuses - fmt.Printf(" %d statuses\n", len(statuses)) - logging.Debugf("[cache] Fetched status for %d devices for %s", len(statuses), apiLabel) + // Fetch device statuses. Skipped entirely for a site-only sync — statuses + // describe devices we aren't collecting. + if config.SyncsAnyDevice() { + report.Stage(apiLabel, "Fetching device statuses") + logging.Debugf("[cache] Fetching device statuses for %s", apiLabel) + if statusSvc := client.Statuses(); statusSvc != nil { + statuses, err := statusSvc.GetAll(ctx) + if err == nil { + cache.DeviceStatus = statuses + report.StageResult(apiLabel, fmt.Sprintf("%d statuses", len(statuses))) + logging.Debugf("[cache] Fetched status for %d devices for %s", len(statuses), apiLabel) + } else { + report.StageResult(apiLabel, "error") + logging.Debugf("[cache] Failed to fetch device statuses for %s: %v", apiLabel, err) + } } else { - fmt.Printf(" error\n") - logging.Debugf("[cache] Failed to fetch device statuses for %s: %v", apiLabel, err) + report.StageResult(apiLabel, "not supported") } - } else { - fmt.Printf(" not supported\n") } - // Fetch BSSIDs (if supported) - if bssidSvc := client.BSSIDs(); bssidSvc != nil { - fmt.Printf(" Fetching BSSIDs...") + // Fetch BSSIDs (if supported). BSSIDs are AP-scoped, so skip when APs aren't synced. + if bssidSvc := client.BSSIDs(); bssidSvc != nil && config.ShouldSync("ap") { + report.Stage(apiLabel, "Fetching BSSIDs") logging.Debugf("[cache] Fetching BSSIDs for %s", apiLabel) entries, err := bssidSvc.List(ctx) if err == nil { @@ -292,17 +300,17 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R } cache.BSSIDs[NormalizeMAC(entry.BSSID)] = entry } - fmt.Printf(" %d BSSIDs\n", len(entries)) + report.StageResult(apiLabel, fmt.Sprintf("%d BSSIDs", len(entries))) logging.Debugf("[cache] Fetched %d BSSIDs for %s", len(entries), apiLabel) } else { - fmt.Printf(" error\n") + report.StageResult(apiLabel, "error") logging.Warnf("[cache] Failed to fetch BSSIDs for %s: %v", apiLabel, err) } } // Fetch templates (if supported) if tmplSvc := client.Templates(); tmplSvc != nil { - fmt.Printf(" Fetching templates...") + report.Stage(apiLabel, "Fetching templates") rfCount, gwCount, wlanCount := 0, 0, 0 if rf, err := tmplSvc.ListRF(ctx); err == nil { for _, t := range rf { @@ -328,25 +336,25 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R } else { logging.Warnf("[cache] Failed to fetch WLAN templates: %v", err) } - fmt.Printf(" %d RF, %d GW, %d WLAN\n", rfCount, gwCount, wlanCount) + report.StageResult(apiLabel, fmt.Sprintf("%d RF, %d GW, %d WLAN", rfCount, gwCount, wlanCount)) } // Fetch profiles (if supported) if profSvc := client.Profiles(); profSvc != nil { - fmt.Printf(" Fetching device profiles...") + report.Stage(apiLabel, "Fetching device profiles") if profiles, err := profSvc.List(ctx, ""); err == nil { for _, p := range profiles { cache.Profiles.Devices = append(cache.Profiles.Devices, *p) } - fmt.Printf(" %d profiles\n", len(profiles)) + report.StageResult(apiLabel, fmt.Sprintf("%d profiles", len(profiles))) } else { - fmt.Printf(" error\n") + report.StageResult(apiLabel, "error") } } // Fetch WLANs (if supported) if wlanSvc := client.WLANs(); wlanSvc != nil { - fmt.Printf(" Fetching WLANs...") + report.Stage(apiLabel, "Fetching WLANs") if wlans, err := wlanSvc.List(ctx); err == nil { // Initialize map if needed if cache.WLANs == nil { @@ -373,9 +381,9 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R } cache.WLANs[w.ID] = w } - fmt.Printf(" %d WLANs\n", len(wlans)) + report.StageResult(apiLabel, fmt.Sprintf("%d WLANs", len(wlans))) } else { - fmt.Printf(" error: %v\n", err) + report.StageResult(apiLabel, fmt.Sprintf("error: %v", err)) logging.Warnf("[cache] Failed to fetch WLANs: %v", err) } } @@ -386,14 +394,17 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R logging.Debugf("[cache] Fetching device configs for %s", apiLabel) // Fetch AP configs - if len(cache.Inventory.AP) > 0 { + if config.ShouldSync("ap") && len(cache.Inventory.AP) > 0 { if opts.SiteID != "" { - fmt.Printf(" Fetching AP configs (site %s)...", opts.SiteID) + report.Stage(apiLabel, fmt.Sprintf("AP configs (site %s)", opts.SiteID)) } else { - fmt.Printf(" Fetching AP configs...") + report.Stage(apiLabel, "AP configs") } apConfigCount, apCarriedCount := 0, 0 + apTotal, apDone := len(cache.Inventory.AP), 0 for mac, item := range cache.Inventory.AP { + apDone++ + report.Progress(apiLabel, apDone, apTotal) if item.ID == "" || item.SiteID == "" { continue } @@ -413,22 +424,25 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R } } if apCarriedCount > 0 { - fmt.Printf(" %d fetched, %d preserved\n", apConfigCount, apCarriedCount) + report.StageResult(apiLabel, fmt.Sprintf("%d fetched, %d preserved", apConfigCount, apCarriedCount)) } else { - fmt.Printf(" %d configs\n", apConfigCount) + report.StageResult(apiLabel, fmt.Sprintf("%d configs", apConfigCount)) } logging.Debugf("[cache] Fetched %d AP configs (preserved %d) for %s", apConfigCount, apCarriedCount, apiLabel) } // Fetch Switch configs - if len(cache.Inventory.Switch) > 0 { + if config.ShouldSync("switch") && len(cache.Inventory.Switch) > 0 { if opts.SiteID != "" { - fmt.Printf(" Fetching switch configs (site %s)...", opts.SiteID) + report.Stage(apiLabel, fmt.Sprintf("switch configs (site %s)", opts.SiteID)) } else { - fmt.Printf(" Fetching switch configs...") + report.Stage(apiLabel, "switch configs") } switchConfigCount, switchCarriedCount := 0, 0 + swTotal, swDone := len(cache.Inventory.Switch), 0 for mac, item := range cache.Inventory.Switch { + swDone++ + report.Progress(apiLabel, swDone, swTotal) if item.ID == "" || item.SiteID == "" { continue } @@ -448,22 +462,25 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R } } if switchCarriedCount > 0 { - fmt.Printf(" %d fetched, %d preserved\n", switchConfigCount, switchCarriedCount) + report.StageResult(apiLabel, fmt.Sprintf("%d fetched, %d preserved", switchConfigCount, switchCarriedCount)) } else { - fmt.Printf(" %d configs\n", switchConfigCount) + report.StageResult(apiLabel, fmt.Sprintf("%d configs", switchConfigCount)) } logging.Debugf("[cache] Fetched %d switch configs (preserved %d) for %s", switchConfigCount, switchCarriedCount, apiLabel) } // Fetch Gateway configs - if len(cache.Inventory.Gateway) > 0 { + if config.ShouldSync("gateway") && len(cache.Inventory.Gateway) > 0 { if opts.SiteID != "" { - fmt.Printf(" Fetching gateway configs (site %s)...", opts.SiteID) + report.Stage(apiLabel, fmt.Sprintf("gateway configs (site %s)", opts.SiteID)) } else { - fmt.Printf(" Fetching gateway configs...") + report.Stage(apiLabel, "gateway configs") } gatewayConfigCount, gatewayCarriedCount := 0, 0 + gwTotal, gwDone := len(cache.Inventory.Gateway), 0 for mac, item := range cache.Inventory.Gateway { + gwDone++ + report.Progress(apiLabel, gwDone, gwTotal) if item.ID == "" || item.SiteID == "" { continue } @@ -483,15 +500,16 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R } } if gatewayCarriedCount > 0 { - fmt.Printf(" %d fetched, %d preserved\n", gatewayConfigCount, gatewayCarriedCount) + report.StageResult(apiLabel, fmt.Sprintf("%d fetched, %d preserved", gatewayConfigCount, gatewayCarriedCount)) } else { - fmt.Printf(" %d configs\n", gatewayConfigCount) + report.StageResult(apiLabel, fmt.Sprintf("%d configs", gatewayConfigCount)) } logging.Debugf("[cache] Fetched %d gateway configs (preserved %d) for %s", gatewayConfigCount, gatewayCarriedCount, apiLabel) } } } else { - fmt.Printf(" Skipping device configs (use 'refresh cache' to fetch)\n") + report.Stage(apiLabel, "Skipping device configs (use 'refresh cache' to fetch)") + report.StageResult(apiLabel, "") logging.Debugf("[cache] Skipping device config fetch for %s (Meraki optimization)", apiLabel) } @@ -501,7 +519,7 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R // carried forward from the prior cache keep their original (older) timestamp. cache.StampFreshObjects(startTime) - fmt.Printf(" [%s] Complete in %dms\n", apiLabel, cache.Meta.RefreshDurationMs) + report.APIDone(apiLabel, time.Since(startTime)) logging.Debugf("[cache] Refresh complete for %s in %dms", apiLabel, cache.Meta.RefreshDurationMs) // Save cache. Use the locked variant because this function already holds @@ -513,13 +531,19 @@ func (c *CacheManager) doRefreshAPI(ctx context.Context, apiLabel string, opts R logging.Debugf("[cache] Saved cache for %s", apiLabel) - // Rebuild cross-API index + // Rebuild the cross-API index — unless the caller batches it. Refresh-all + // sets SkipIndexRebuild and rebuilds once after every API saves, instead of + // rebuilding (and re-reporting every MAC collision) once per API. + if opts.SkipIndexRebuild { + return nil + } return c.RebuildIndex() } -// RefreshAllAPIs refreshes all API caches in parallel. -func (c *CacheManager) RefreshAllAPIs(ctx context.Context) map[string]error { - return c.refreshAllAPIs(ctx, func(string) RefreshOptions { +// RefreshAllAPIs refreshes all API caches in parallel, reporting progress to +// report (nil falls back to linear stdout output). +func (c *CacheManager) RefreshAllAPIs(ctx context.Context, report refreshui.Reporter) map[string]error { + return c.refreshAllAPIs(ctx, report, func(string) RefreshOptions { return RefreshOptions{FetchDeviceConfigs: true} }) } @@ -527,24 +551,40 @@ func (c *CacheManager) RefreshAllAPIs(ctx context.Context) map[string]error { // RefreshAllAPIsManaged refreshes every API in parallel, limiting per-device // config fetches to the armed (managed) MACs. The same set is applied to each // API; MACs not present in a given API's inventory simply never match. -func (c *CacheManager) RefreshAllAPIsManaged(ctx context.Context, managed map[string]bool) map[string]error { - return c.refreshAllAPIs(ctx, func(string) RefreshOptions { +func (c *CacheManager) RefreshAllAPIsManaged(ctx context.Context, managed map[string]bool, report refreshui.Reporter) map[string]error { + return c.refreshAllAPIs(ctx, report, func(string) RefreshOptions { return RefreshOptions{FetchDeviceConfigs: true, ManagedMACs: managed} }) } -func (c *CacheManager) refreshAllAPIs(ctx context.Context, optsFor func(apiLabel string) RefreshOptions) map[string]error { +func (c *CacheManager) refreshAllAPIs(ctx context.Context, report refreshui.Reporter, optsFor func(apiLabel string) RefreshOptions) map[string]error { labels := c.registry.GetAllLabels() + report = refreshui.Resolve(report) - var wg sync.WaitGroup errors := make(map[string]error) var mu sync.Mutex + // Bound the fan-out: refresh-all otherwise launches one goroutine per API at + // once. A buffered channel caps how many run concurrently; the per-label + // mutex inside RefreshAPIWithOptions still serializes same-API refreshes. + var wg sync.WaitGroup + sem := make(chan struct{}, c.refreshLimit(len(labels))) + for _, label := range labels { wg.Add(1) + sem <- struct{}{} go func(apiLabel string) { defer wg.Done() - if err := c.RefreshAPIWithOptions(ctx, apiLabel, optsFor(apiLabel)); err != nil { + defer func() { <-sem }() + + apiCtx, cancel := c.refreshCtx(ctx) + defer cancel() + + opts := optsFor(apiLabel) + opts.Reporter = report + opts.SkipIndexRebuild = true // batch the rebuild once, below + if err := c.RefreshAPIWithOptions(apiCtx, apiLabel, opts); err != nil { + report.APIError(apiLabel, err) mu.Lock() errors[apiLabel] = err mu.Unlock() @@ -554,7 +594,7 @@ func (c *CacheManager) refreshAllAPIs(ctx context.Context, optsFor func(apiLabel wg.Wait() - // Rebuild index even if some APIs failed + // Rebuild index once, even if some APIs failed. if err := c.RebuildIndex(); err != nil { errors["_index"] = err } diff --git a/internal/vendors/cache_manager_refresh_site_test.go b/internal/vendors/cache_manager_refresh_site_test.go index b493564..ef5b548 100644 --- a/internal/vendors/cache_manager_refresh_site_test.go +++ b/internal/vendors/cache_manager_refresh_site_test.go @@ -45,6 +45,7 @@ func TestRefreshAPISite(t *testing.T) { Label: "test-api", Vendor: "mock", Credentials: map[string]string{"org_id": "org-123"}, + SyncTypes: []string{"ap"}, }, } registry.InitializeClients(configs) diff --git a/internal/vendors/cache_manager_test.go b/internal/vendors/cache_manager_test.go index 2f8a8f9..9c88113 100644 --- a/internal/vendors/cache_manager_test.go +++ b/internal/vendors/cache_manager_test.go @@ -552,7 +552,7 @@ func TestCacheManager_RefreshAllAPIs(t *testing.T) { } ctx := context.Background() - errs := cm.RefreshAllAPIs(ctx) + errs := cm.RefreshAllAPIs(ctx, nil) if len(errs) > 0 { t.Errorf("unexpected errors: %v", errs) diff --git a/internal/vendors/registry.go b/internal/vendors/registry.go index 4d960e9..6ce582c 100644 --- a/internal/vendors/registry.go +++ b/internal/vendors/registry.go @@ -2,6 +2,7 @@ package vendors import ( "fmt" + "slices" "sort" "sync" "time" @@ -40,8 +41,22 @@ type APIConfig struct { // not the overall request — so a dead host fails fast without capping slow // but working responses. Vendor clients apply it to their transport. ConnectTimeout time.Duration + // SyncTypes lists the device types this API collects: any of "ap", "switch", + // "gateway". Empty means site attributes only — no device inventory, configs, + // statuses, or BSSIDs are fetched. Normalized lowercase and deduped at load. + SyncTypes []string } +// ShouldSync reports whether deviceType ("ap"/"switch"/"gateway") is collected +// for this API. +func (c *APIConfig) ShouldSync(deviceType string) bool { + return slices.Contains(c.SyncTypes, deviceType) +} + +// SyncsAnyDevice reports whether any device type is collected. False means a +// site-attributes-only sync. +func (c *APIConfig) SyncsAnyDevice() bool { return len(c.SyncTypes) > 0 } + // APIStatus represents the status of an API connection. type APIStatus struct { Label string diff --git a/internal/vendors/registry_test.go b/internal/vendors/registry_test.go index 45c244d..90efed8 100644 --- a/internal/vendors/registry_test.go +++ b/internal/vendors/registry_test.go @@ -651,3 +651,29 @@ func TestMockSearchService(t *testing.T) { t.Error("expected NeedsConfirmation=false for mock") } } + +func TestAPIConfigShouldSync(t *testing.T) { + cfg := &APIConfig{SyncTypes: []string{"ap", "switch"}} + if !cfg.ShouldSync("ap") { + t.Error("expected ShouldSync(ap)=true") + } + if !cfg.ShouldSync("switch") { + t.Error("expected ShouldSync(switch)=true") + } + if cfg.ShouldSync("gateway") { + t.Error("expected ShouldSync(gateway)=false") + } + if !cfg.SyncsAnyDevice() { + t.Error("expected SyncsAnyDevice=true") + } +} + +func TestAPIConfigSyncsAnyDeviceEmpty(t *testing.T) { + cfg := &APIConfig{} + if cfg.SyncsAnyDevice() { + t.Error("expected SyncsAnyDevice=false for empty SyncTypes") + } + if cfg.ShouldSync("ap") { + t.Error("expected ShouldSync(ap)=false for empty SyncTypes") + } +}