diff --git a/docs/api-reference.md b/docs/api-reference.md index 5c269278..eaa0d2a6 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -263,6 +263,39 @@ Get the VNC console URL for the instance. } ``` +### GET /instances/:id/stats +Get real-time resource usage stats for an instance (CPU, memory, network I/O, disk I/O). + +**Response:** +```json +{ + "cpu_percentage": 25.5, + "memory_usage_bytes": 524288000, + "memory_limit_bytes": 1073741824, + "memory_percentage": 48.8, + "network_rx_bytes": 1024000, + "network_tx_bytes": 512000, + "disk_read_bytes": 10240000, + "disk_write_bytes": 5120000, + "cpu_time_nanoseconds": 5000000000 +} +``` + +**Fields:** +- `cpu_percentage` (float): CPU usage as percentage of available +- `memory_usage_bytes` (float): Current memory usage in bytes +- `memory_limit_bytes` (float): Memory limit in bytes +- `memory_percentage` (float): Memory usage as percentage of limit +- `network_rx_bytes` (uint64): Total network bytes received +- `network_tx_bytes` (uint64): Total network bytes transmitted +- `disk_read_bytes` (uint64): Total disk bytes read +- `disk_write_bytes` (uint64): Total disk bytes written +- `cpu_time_nanoseconds` (uint64, optional): Cumulative CPU time in nanoseconds (Libvirt backend) + +**Error Responses:** +- `404` — Instance not found +- `503` — Backend stats unavailable + ### POST /instances/:id/pause Pause a running instance (freezes CPU, retains memory/network). diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 7502320e..08ad937f 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -9056,6 +9056,16 @@ const docTemplate = `{ "cpu_percentage": { "type": "number" }, + "cpu_time_nanoseconds": { + "description": "only populated by Libvirt backend; Docker uses delta-based percentage instead", + "type": "integer" + }, + "disk_read_bytes": { + "type": "integer" + }, + "disk_write_bytes": { + "type": "integer" + }, "memory_limit_bytes": { "type": "number" }, @@ -9064,6 +9074,12 @@ const docTemplate = `{ }, "memory_usage_bytes": { "type": "number" + }, + "network_rx_bytes": { + "type": "integer" + }, + "network_tx_bytes": { + "type": "integer" } } }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index d9ff7211..25d8f105 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -9048,6 +9048,16 @@ "cpu_percentage": { "type": "number" }, + "cpu_time_nanoseconds": { + "description": "only populated by Libvirt backend; Docker uses delta-based percentage instead", + "type": "integer" + }, + "disk_read_bytes": { + "type": "integer" + }, + "disk_write_bytes": { + "type": "integer" + }, "memory_limit_bytes": { "type": "number" }, @@ -9056,6 +9066,12 @@ }, "memory_usage_bytes": { "type": "number" + }, + "network_rx_bytes": { + "type": "integer" + }, + "network_tx_bytes": { + "type": "integer" } } }, diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index c9b5f750..1eced7fb 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -663,12 +663,24 @@ definitions: properties: cpu_percentage: type: number + cpu_time_nanoseconds: + description: only populated by Libvirt backend; Docker uses delta-based percentage + instead + type: integer + disk_read_bytes: + type: integer + disk_write_bytes: + type: integer memory_limit_bytes: type: number memory_percentage: type: number memory_usage_bytes: type: number + network_rx_bytes: + type: integer + network_tx_bytes: + type: integer type: object domain.InstanceStatus: enum: diff --git a/internal/core/domain/instance.go b/internal/core/domain/instance.go index e3f30bb1..d19c6e58 100644 --- a/internal/core/domain/instance.go +++ b/internal/core/domain/instance.go @@ -91,10 +91,15 @@ type Instance struct { // InstanceStats contains real-time resource usage metrics. // Values are instantaneous snapshots from the compute backend. type InstanceStats struct { - CPUPercentage float64 `json:"cpu_percentage"` - MemoryUsageBytes float64 `json:"memory_usage_bytes"` - MemoryLimitBytes float64 `json:"memory_limit_bytes"` - MemoryPercentage float64 `json:"memory_percentage"` + CPUPercentage float64 `json:"cpu_percentage"` + MemoryUsageBytes float64 `json:"memory_usage_bytes"` + MemoryLimitBytes float64 `json:"memory_limit_bytes"` + MemoryPercentage float64 `json:"memory_percentage"` + NetworkRxBytes uint64 `json:"network_rx_bytes"` + NetworkTxBytes uint64 `json:"network_tx_bytes"` + DiskReadBytes uint64 `json:"disk_read_bytes"` + DiskWriteBytes uint64 `json:"disk_write_bytes"` + CPUTimeNanoseconds uint64 `json:"cpu_time_nanoseconds,omitempty"` // only populated by Libvirt backend; Docker uses delta-based percentage instead } // RawDockerStats mirrors Docker's stats payload for CPU/memory calculations. @@ -104,6 +109,7 @@ type RawDockerStats struct { TotalUsage uint64 `json:"total_usage"` } `json:"cpu_usage"` SystemCPUUsage uint64 `json:"system_cpu_usage"` + CPUTime uint64 `json:"cpu_time"` // libvirt: cumulative CPU time in nanoseconds } `json:"cpu_stats"` PreCPUStats struct { CPUUsage struct { @@ -115,4 +121,18 @@ type RawDockerStats struct { Usage uint64 `json:"usage"` Limit uint64 `json:"limit"` } `json:"memory_stats"` + NetworkStats map[string]struct { + RxBytes uint64 `json:"rx_bytes"` + TxBytes uint64 `json:"tx_bytes"` + } `json:"network_stats"` + BlkioStats struct { + IoServiceBytes []BlkioStatEntry `json:"ioservice_bytes"` + } `json:"blkio_stats"` +} + +// BlkioStatEntry represents a single block I/O stat entry. +type BlkioStatEntry struct { + Op string `json:"op"` + Device string `json:"device"` + Value uint64 `json:"value"` } diff --git a/internal/core/services/instance.go b/internal/core/services/instance.go index 338003c3..4155f088 100644 --- a/internal/core/services/instance.go +++ b/internal/core/services/instance.go @@ -1290,11 +1290,34 @@ func (s *InstanceService) calculateInstanceStats(stats *domain.RawDockerStats) * memPercent = (memUsage / memLimit) * 100.0 } + // Sum network rx/tx across all interfaces + var rxBytes, txBytes uint64 + for _, net := range stats.NetworkStats { + rxBytes += net.RxBytes + txBytes += net.TxBytes + } + + // Sum block read/write bytes + var readBytes, writeBytes uint64 + for _, entry := range stats.BlkioStats.IoServiceBytes { + switch entry.Op { + case "read", "Read": + readBytes += entry.Value + case "write", "Write": + writeBytes += entry.Value + } + } + return &domain.InstanceStats{ - CPUPercentage: cpuPercent, - MemoryUsageBytes: memUsage, - MemoryLimitBytes: memLimit, - MemoryPercentage: memPercent, + CPUPercentage: cpuPercent, + MemoryUsageBytes: memUsage, + MemoryLimitBytes: memLimit, + MemoryPercentage: memPercent, + NetworkRxBytes: rxBytes, + NetworkTxBytes: txBytes, + DiskReadBytes: readBytes, + DiskWriteBytes: writeBytes, + CPUTimeNanoseconds: stats.CPUStats.CPUTime, } } diff --git a/internal/core/services/instance_internal_test.go b/internal/core/services/instance_internal_test.go index f1369816..2af3a983 100644 --- a/internal/core/services/instance_internal_test.go +++ b/internal/core/services/instance_internal_test.go @@ -98,20 +98,90 @@ func TestInstanceServiceInternalUpdateVolumesAfterLaunch(t *testing.T) { func TestInstanceService_CalculateInstanceStats(t *testing.T) { svc := &InstanceService{} - stats := &domain.RawDockerStats{} - stats.CPUStats.CPUUsage.TotalUsage = 1000 - stats.CPUStats.SystemCPUUsage = 10000 + t.Run("Basic CPU and Memory", func(t *testing.T) { + stats := &domain.RawDockerStats{} + stats.CPUStats.CPUUsage.TotalUsage = 1000 + stats.CPUStats.SystemCPUUsage = 10000 + stats.PreCPUStats.CPUUsage.TotalUsage = 500 + stats.PreCPUStats.SystemCPUUsage = 5000 + stats.MemoryStats.Usage = 1024 + stats.MemoryStats.Limit = 2048 + + res := svc.calculateInstanceStats(stats) + assert.InDelta(t, 10.0, res.CPUPercentage, 0.01) // (1000-500)/(10000-5000) * 100 = 10% + assert.InDelta(t, 50.0, res.MemoryPercentage, 0.01) + assert.Equal(t, uint64(0), res.NetworkRxBytes) + assert.Equal(t, uint64(0), res.NetworkTxBytes) + assert.Equal(t, uint64(0), res.DiskReadBytes) + assert.Equal(t, uint64(0), res.DiskWriteBytes) + }) + + t.Run("Network I/O multiple interfaces", func(t *testing.T) { + stats := &domain.RawDockerStats{} + stats.NetworkStats = map[string]struct { + RxBytes uint64 `json:"rx_bytes"` + TxBytes uint64 `json:"tx_bytes"` + }{ + "eth0": {RxBytes: 1000, TxBytes: 500}, + "eth1": {RxBytes: 2000, TxBytes: 1500}, + } + + res := svc.calculateInstanceStats(stats) + assert.Equal(t, uint64(3000), res.NetworkRxBytes) // 1000 + 2000 + assert.Equal(t, uint64(2000), res.NetworkTxBytes) // 500 + 1500 + }) - stats.PreCPUStats.CPUUsage.TotalUsage = 500 - stats.PreCPUStats.SystemCPUUsage = 5000 + t.Run("Block I/O read and write", func(t *testing.T) { + stats := &domain.RawDockerStats{} + stats.BlkioStats.IoServiceBytes = []domain.BlkioStatEntry{ + {Op: "read", Value: 5000}, + {Op: "write", Value: 3000}, + {Op: "Read", Value: 1000}, // uppercase variant + {Op: "Write", Value: 2000}, // uppercase variant + } + + res := svc.calculateInstanceStats(stats) + assert.Equal(t, uint64(6000), res.DiskReadBytes) // 5000 + 1000 + assert.Equal(t, uint64(5000), res.DiskWriteBytes) // 3000 + 2000 + }) - stats.MemoryStats.Usage = 1024 - stats.MemoryStats.Limit = 2048 + t.Run("CPU time nanoseconds", func(t *testing.T) { + stats := &domain.RawDockerStats{} + stats.CPUStats.CPUTime = 5000000000 // 5 nanoseconds - res := svc.calculateInstanceStats(stats) - assert.InDelta(t, 10.0, res.CPUPercentage, 0.01) // (1000-500)/(10000-5000) * 100 = 10% - assert.InDelta(t, 50.0, res.MemoryPercentage, 0.01) + res := svc.calculateInstanceStats(stats) + assert.Equal(t, uint64(5000000000), res.CPUTimeNanoseconds) + }) + + t.Run("Combined all fields", func(t *testing.T) { + stats := &domain.RawDockerStats{} + stats.CPUStats.CPUUsage.TotalUsage = 800 + stats.CPUStats.SystemCPUUsage = 8000 + stats.PreCPUStats.CPUUsage.TotalUsage = 400 + stats.PreCPUStats.SystemCPUUsage = 4000 + stats.MemoryStats.Usage = 512 + stats.MemoryStats.Limit = 1024 + stats.CPUStats.CPUTime = 3000000000 + stats.NetworkStats = map[string]struct { + RxBytes uint64 `json:"rx_bytes"` + TxBytes uint64 `json:"tx_bytes"` + }{ + "eth0": {RxBytes: 500, TxBytes: 250}, + } + stats.BlkioStats.IoServiceBytes = []domain.BlkioStatEntry{ + {Op: "read", Value: 2048}, + } + + res := svc.calculateInstanceStats(stats) + assert.InDelta(t, 10.0, res.CPUPercentage, 0.01) + assert.InDelta(t, 50.0, res.MemoryPercentage, 0.01) + assert.Equal(t, uint64(500), res.NetworkRxBytes) + assert.Equal(t, uint64(250), res.NetworkTxBytes) + assert.Equal(t, uint64(2048), res.DiskReadBytes) + assert.Equal(t, uint64(0), res.DiskWriteBytes) + assert.Equal(t, uint64(3000000000), res.CPUTimeNanoseconds) + }) } func TestInstanceService_FormatContainerName(t *testing.T) { diff --git a/internal/core/services/instance_unit_test.go b/internal/core/services/instance_unit_test.go index 3de1da86..46ac06a9 100644 --- a/internal/core/services/instance_unit_test.go +++ b/internal/core/services/instance_unit_test.go @@ -1578,6 +1578,15 @@ func testInstanceServiceUnitRepoErrors(t *testing.T) { require.Error(t, err) }) + t.Run("GetInstanceStats_ComputeError", func(t *testing.T) { + repo.On("GetByName", mock.Anything, "test-inst").Return(inst, nil).Once() + compute.On("GetInstanceStats", mock.Anything, "cid-1").Return(nil, fmt.Errorf("stats unavailable")).Once() + + _, err := svc.GetInstanceStats(ctx, "test-inst") + require.Error(t, err) + assert.Contains(t, err.Error(), "stats unavailable") + }) + t.Run("Exec_NotFound", func(t *testing.T) { repo.On("GetByName", mock.Anything, mock.Anything).Return(nil, svcerrors.New(svcerrors.NotFound, "not found")).Once() repo.On("GetByID", mock.Anything, mock.Anything).Return(nil, svcerrors.New(svcerrors.NotFound, "not found")).Once() diff --git a/internal/handlers/instance_handler_test.go b/internal/handlers/instance_handler_test.go index c51522ad..4a0a6620 100644 --- a/internal/handlers/instance_handler_test.go +++ b/internal/handlers/instance_handler_test.go @@ -460,7 +460,17 @@ func TestInstanceHandlerGetStats(t *testing.T) { r.GET(instancesPath+"/:id/stats", handler.GetStats) id := uuid.New().String() - stats := &domain.InstanceStats{CPUPercentage: 10.5, MemoryUsageBytes: 128} + stats := &domain.InstanceStats{ + CPUPercentage: 10.5, + MemoryUsageBytes: 128, + MemoryLimitBytes: 256, + MemoryPercentage: 50.0, + NetworkRxBytes: 1024, + NetworkTxBytes: 512, + DiskReadBytes: 4096, + DiskWriteBytes: 2048, + CPUTimeNanoseconds: 3000000000, + } mockSvc.On("GetInstanceStats", mock.Anything, id).Return(stats, nil) req := httptest.NewRequest(http.MethodGet, instancesPath+"/"+id+"/stats", nil) @@ -469,6 +479,24 @@ func TestInstanceHandlerGetStats(t *testing.T) { r.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) + + // Verify all new stats fields are present in the JSON response + // httputil.Success wraps data in {"data": {...}} + var wrapper struct { + Data map[string]interface{} `json:"data"` + } + err := json.Unmarshal(w.Body.Bytes(), &wrapper) + require.NoError(t, err) + + assert.InDelta(t, 10.5, wrapper.Data["cpu_percentage"], 0.01) + assert.InDelta(t, 128, wrapper.Data["memory_usage_bytes"], 0.01) + assert.InDelta(t, 256, wrapper.Data["memory_limit_bytes"], 0.01) + assert.InDelta(t, 50.0, wrapper.Data["memory_percentage"], 0.01) + assert.Equal(t, uint64(1024), uint64(wrapper.Data["network_rx_bytes"].(float64))) + assert.Equal(t, uint64(512), uint64(wrapper.Data["network_tx_bytes"].(float64))) + assert.Equal(t, uint64(4096), uint64(wrapper.Data["disk_read_bytes"].(float64))) + assert.Equal(t, uint64(2048), uint64(wrapper.Data["disk_write_bytes"].(float64))) + assert.Equal(t, uint64(3000000000), uint64(wrapper.Data["cpu_time_nanoseconds"].(float64))) } func TestInstanceHandlerLaunchWithVolumesAndVPC(t *testing.T) { diff --git a/internal/repositories/libvirt/adapter.go b/internal/repositories/libvirt/adapter.go index 99b0f780..496ba1c6 100644 --- a/internal/repositories/libvirt/adapter.go +++ b/internal/repositories/libvirt/adapter.go @@ -46,6 +46,10 @@ const ( // Memory stat tags memStatTagActual = 5 memStatTagRSS = 6 + + // typedParamULLONG is the discriminator value for uint64 in libvirt.TypedParamValue. + // The go-libvirt library uses a struct{D uint32; I interface{}} where D==4 means ULLONG. + typedParamULLONG = 4 ) // domainStateName returns a human-readable name for a libvirt domain state. @@ -602,11 +606,28 @@ func (a *LibvirtAdapter) GetInstanceStats(ctx context.Context, id string) (io.Re } } + // Get CPU stats via DomainGetCPUStats + cpuParams, _, err := a.client.DomainGetCPUStats(ctx, dom, 0, 0, 0, 0) + var cpuTime uint64 + if err == nil { + for _, p := range cpuParams { + if p.Field == "cpu_time" && p.Value.D == typedParamULLONG { + if v, ok := p.Value.I.(uint64); ok { + cpuTime = v + } + } + } + } + // Stats are best-effort; CPU time is omitted on error (graceful degradation) + statJSON, _ := json.Marshal(map[string]interface{}{ "memory_stats": map[string]uint64{ "usage": usage, "limit": limit, }, + "cpu_stats": map[string]uint64{ + "cpu_time": cpuTime, + }, }) return io.NopCloser(bytes.NewReader(statJSON)), nil } diff --git a/internal/repositories/libvirt/adapter_unit_test.go b/internal/repositories/libvirt/adapter_unit_test.go index 21633983..0e494665 100644 --- a/internal/repositories/libvirt/adapter_unit_test.go +++ b/internal/repositories/libvirt/adapter_unit_test.go @@ -742,6 +742,7 @@ func TestLibvirtAdapter_StatsAndConsole(t *testing.T) { t.Run("GetInstanceStats", func(t *testing.T) { m.On("DomainLookupByName", mock.Anything, id).Return(dom, nil).Once() m.On("DomainMemoryStats", mock.Anything, dom, uint32(10), uint32(0)).Return([]libvirt.DomainMemoryStat{}, nil).Once() + m.On("DomainGetCPUStats", mock.Anything, dom, uint32(0), int32(0), uint32(0), libvirt.TypedParameterFlags(0)).Return([]libvirt.TypedParam{}, int32(0), nil).Once() m.On("DomainGetState", mock.Anything, dom, uint32(0)).Return(int32(1), int32(0), nil).Once() stats, err := a.GetInstanceStats(ctx, id) @@ -749,6 +750,25 @@ func TestLibvirtAdapter_StatsAndConsole(t *testing.T) { assert.NotNil(t, stats) }) + t.Run("GetInstanceStats_WithCPUStats", func(t *testing.T) { + cpuParams := []libvirt.TypedParam{ + {Field: "cpu_time", Value: libvirt.TypedParamValue{D: typedParamULLONG, I: uint64(5000000000)}}, + } + m.On("DomainLookupByName", mock.Anything, id).Return(dom, nil).Once() + m.On("DomainMemoryStats", mock.Anything, dom, uint32(10), uint32(0)).Return([]libvirt.DomainMemoryStat{}, nil).Once() + m.On("DomainGetCPUStats", mock.Anything, dom, uint32(0), int32(0), uint32(0), libvirt.TypedParameterFlags(0)).Return(cpuParams, int32(0), nil).Once() + m.On("DomainGetState", mock.Anything, dom, uint32(0)).Return(int32(1), int32(0), nil).Once() + + stats, err := a.GetInstanceStats(ctx, id) + require.NoError(t, err) + assert.NotNil(t, stats) + + // Read and verify the stats JSON contains the CPU time + statsBytes, err := io.ReadAll(stats) + require.NoError(t, err) + assert.Contains(t, string(statsBytes), `"cpu_time":5000000000`) + }) + t.Run("GetConsoleURL", func(t *testing.T) { xml := "" m.On("DomainLookupByName", mock.Anything, id).Return(dom, nil).Once() diff --git a/internal/repositories/libvirt/libvirt_client.go b/internal/repositories/libvirt/libvirt_client.go index 4ff26d7a..cbd27e67 100644 --- a/internal/repositories/libvirt/libvirt_client.go +++ b/internal/repositories/libvirt/libvirt_client.go @@ -27,6 +27,7 @@ type LibvirtClient interface { DomainAttachDevice(ctx context.Context, dom libvirt.Domain, xml string) error DomainDetachDevice(ctx context.Context, dom libvirt.Domain, xml string) error DomainMemoryStats(ctx context.Context, dom libvirt.Domain, maxStats uint32, flags uint32) ([]libvirt.DomainMemoryStat, error) + DomainGetCPUStats(ctx context.Context, dom libvirt.Domain, nparams uint32, startCPU int32, ncpus uint32, flags libvirt.TypedParameterFlags) ([]libvirt.TypedParam, int32, error) // Network NetworkLookupByName(ctx context.Context, name string) (libvirt.Network, error) diff --git a/internal/repositories/libvirt/mock_client_test.go b/internal/repositories/libvirt/mock_client_test.go index 947e53ea..e775e944 100644 --- a/internal/repositories/libvirt/mock_client_test.go +++ b/internal/repositories/libvirt/mock_client_test.go @@ -83,6 +83,13 @@ func (m *MockLibvirtClient) DomainMemoryStats(ctx context.Context, dom libvirt.D return r0, args.Error(1) } +func (m *MockLibvirtClient) DomainGetCPUStats(ctx context.Context, dom libvirt.Domain, nparams uint32, startCPU int32, ncpus uint32, flags libvirt.TypedParameterFlags) ([]libvirt.TypedParam, int32, error) { + args := m.Called(ctx, dom, nparams, startCPU, ncpus, flags) + r0, _ := args.Get(0).([]libvirt.TypedParam) + r1, _ := args.Get(1).(int32) + return r0, r1, args.Error(2) +} + // Network func (m *MockLibvirtClient) NetworkLookupByName(ctx context.Context, name string) (libvirt.Network, error) { diff --git a/internal/repositories/libvirt/real_client.go b/internal/repositories/libvirt/real_client.go index 41a517a5..4b89bf06 100644 --- a/internal/repositories/libvirt/real_client.go +++ b/internal/repositories/libvirt/real_client.go @@ -175,6 +175,15 @@ func (r *RealLibvirtClient) DomainMemoryStats(ctx context.Context, dom libvirt.D return r.conn.DomainMemoryStats(dom, maxStats, flags) } +func (r *RealLibvirtClient) DomainGetCPUStats(ctx context.Context, dom libvirt.Domain, nparams uint32, startCPU int32, ncpus uint32, flags libvirt.TypedParameterFlags) ([]libvirt.TypedParam, int32, error) { + select { + case <-ctx.Done(): + return nil, 0, ctx.Err() + default: + } + return r.conn.DomainGetCPUStats(dom, nparams, startCPU, ncpus, flags) +} + // Network func (r *RealLibvirtClient) NetworkLookupByName(ctx context.Context, name string) (libvirt.Network, error) {