diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 5e2fffe..c26d2bb 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -21,9 +21,10 @@ "Bash(mkdir:*)", "Bash(make lint:*)", "Bash(make:*)", - "Bash(./ddtest:*)" + "Bash(./ddtest:*)", + "Bash(git diff:*)" ], "deny": [], "defaultMode": "acceptEdits" } -} \ No newline at end of file +} diff --git a/internal/runner/dd_test_optimization.go b/internal/runner/dd_test_optimization.go index 557ba61..d6d5550 100644 --- a/internal/runner/dd_test_optimization.go +++ b/internal/runner/dd_test_optimization.go @@ -5,8 +5,13 @@ import ( "fmt" "log/slog" "maps" + "os" + "regexp" + "strings" "time" + "github.com/DataDog/ddtest/civisibility/constants" + "github.com/DataDog/ddtest/civisibility/utils" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization" "golang.org/x/sync/errgroup" @@ -55,6 +60,7 @@ func (tr *TestRunner) PrepareTestOptimization(ctx context.Context) error { var fullDiscoverySucceeded bool var fullDiscoveryErr error var fastDiscoveryErr error + tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) g, _ := errgroup.WithContext(ctx) @@ -77,6 +83,8 @@ func (tr *TestRunner) PrepareTestOptimization(ctx context.Context) error { } } + tr.fetchAndStoreTestSuiteDurations() + startTime := time.Now() slog.Info("Fetching skippable tests from Datadog...") skippableTests = tr.optimizationClient.GetSkippableTests() @@ -178,3 +186,68 @@ func (tr *TestRunner) PrepareTestOptimization(ctx context.Context) error { return nil } + +func initializeDurationsFetchInputs() (string, string, error) { + ciTags := utils.GetCITags() + repositoryURL := ciTags[constants.GitRepositoryURL] + if repositoryURL == "" { + return "", "", fmt.Errorf("repository URL is required") + } + + service := os.Getenv("DD_SERVICE") + if service == "" { + repoRegex := regexp.MustCompile(`(?m)/([a-zA-Z0-9\-_.]*)$`) + matches := repoRegex.FindStringSubmatch(repositoryURL) + if len(matches) > 1 { + repositoryURL = strings.TrimSuffix(matches[1], ".git") + } + service = repositoryURL + } + + return ciTags[constants.GitRepositoryURL], service, nil +} + +func (tr *TestRunner) fetchAndStoreTestSuiteDurations() { + repositoryURL, service, err := initializeDurationsFetchInputs() + if err != nil { + logDurationsAPIError(err) + tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) + return + } + + durations, err := tr.durationsClient.GetTestSuiteDurations(repositoryURL, service) + if err != nil { + logDurationsAPIError(err) + tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) + return + } + + tr.storeTestSuiteDurations(repositoryURL, service, durations) +} + +func (tr *TestRunner) storeTestSuiteDurations( + repositoryURL, service string, + durations map[string]map[string]testoptimization.TestSuiteDurationInfo, +) { + totalSuites := countTestSuites(durations) + if totalSuites == 0 { + slog.Warn("Test durations API returned no test suites", "service", service, "repositoryURL", repositoryURL) + tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) + return + } + + slog.Debug("Found test suite durations", "service", service, "repositoryURL", repositoryURL, "testSuitesCount", totalSuites) + tr.testSuiteDurations = durations +} + +func countTestSuites(durations map[string]map[string]testoptimization.TestSuiteDurationInfo) int { + totalSuites := 0 + for _, suites := range durations { + totalSuites += len(suites) + } + return totalSuites +} + +func logDurationsAPIError(err error) { + slog.Error("Test durations API errored", "error", err) +} diff --git a/internal/runner/dd_test_optimization_test.go b/internal/runner/dd_test_optimization_test.go index f178fc6..df5c58b 100644 --- a/internal/runner/dd_test_optimization_test.go +++ b/internal/runner/dd_test_optimization_test.go @@ -1,20 +1,38 @@ package runner import ( + "bytes" "context" "errors" + "log/slog" "os" "os/exec" "path/filepath" "strings" "testing" + ciConstants "github.com/DataDog/ddtest/civisibility/constants" + ciUtils "github.com/DataDog/ddtest/civisibility/utils" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization" ) +func captureLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + originalLogger := slog.Default() + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + slog.SetDefault(logger) + t.Cleanup(func() { + slog.SetDefault(originalLogger) + }) + return &buf +} + func TestTestRunner_PrepareTestOptimization_Success(t *testing.T) { ctx := context.Background() + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) // Setup mocks mockFramework := &MockFramework{ @@ -46,8 +64,18 @@ func TestTestRunner_PrepareTestOptimization_Success(t *testing.T) { "TestSuite3.test4.": true, // Skip test4 }, } + mockDurationsClient := &MockTestSuiteDurationsClient{ + Durations: map[string]map[string]testoptimization.TestSuiteDurationInfo{ + "module1": { + "suite1": { + SourceFile: "test/file1_test.rb", + Duration: testoptimization.DurationPercentiles{P50: "1", P90: "2"}, + }, + }, + }, + } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -92,6 +120,151 @@ func TestTestRunner_PrepareTestOptimization_Success(t *testing.T) { t.Errorf("PrepareTestOptimization() should calculate skippable percentage as %.2f, got %.2f", expectedPercentage, runner.skippablePercentage) } + + if !mockDurationsClient.Called { + t.Error("PrepareTestOptimization() should fetch test suite durations") + } +} + +func TestTestRunner_PrepareTestOptimization_DurationsErrorContinues(t *testing.T) { + ctx := context.Background() + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) + logs := captureLogs(t) + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "Suite", Name: "test1", Parameters: "", SuiteSourceFile: "spec/file1_test.rb"}, + }, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{ + ciConstants.GitRepositoryURL: "github.com/DataDog/ddtest", + }, + Framework: mockFramework, + } + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{} + mockDurationsClient := &MockTestSuiteDurationsClient{Err: errors.New("durations backend failed")} + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) + + err := runner.PrepareTestOptimization(ctx) + if err != nil { + t.Fatalf("PrepareTestOptimization() should not fail when durations API errors, got: %v", err) + } + + if len(runner.testSuiteDurations) != 0 { + t.Errorf("Expected empty in-memory test suite durations on error, got %v", runner.testSuiteDurations) + } + + if !strings.Contains(logs.String(), "level=ERROR") || !strings.Contains(logs.String(), "Test durations API errored") { + t.Errorf("Expected ERROR log for durations API failure, got logs: %s", logs.String()) + } +} + +func TestTestRunner_PrepareTestOptimization_EmptyDurationsWarns(t *testing.T) { + ctx := context.Background() + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) + logs := captureLogs(t) + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "Suite", Name: "test1", Parameters: "", SuiteSourceFile: "spec/file1_test.rb"}, + }, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{ + ciConstants.GitRepositoryURL: "github.com/DataDog/ddtest", + }, + Framework: mockFramework, + } + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{} + mockDurationsClient := &MockTestSuiteDurationsClient{ + Durations: map[string]map[string]testoptimization.TestSuiteDurationInfo{}, + } + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) + + err := runner.PrepareTestOptimization(ctx) + if err != nil { + t.Fatalf("PrepareTestOptimization() should not fail with empty durations, got: %v", err) + } + + if len(runner.testSuiteDurations) != 0 { + t.Errorf("Expected empty in-memory test suite durations on empty response, got %v", runner.testSuiteDurations) + } + + if !strings.Contains(logs.String(), "level=WARN") || !strings.Contains(logs.String(), "Test durations API returned no test suites") { + t.Errorf("Expected WARN log for empty durations response, got logs: %s", logs.String()) + } +} + +func TestTestRunner_PrepareTestOptimization_NonEmptyDurationsStoredWithoutChangingWeights(t *testing.T) { + ctx := context.Background() + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) + logs := captureLogs(t) + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "Suite1", Name: "test1", Parameters: "", SuiteSourceFile: "spec/file1_test.rb"}, + {Suite: "Suite1", Name: "test2", Parameters: "", SuiteSourceFile: "spec/file1_test.rb"}, + {Suite: "Suite2", Name: "test3", Parameters: "", SuiteSourceFile: "spec/file2_test.rb"}, + }, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{ + ciConstants.GitRepositoryURL: "github.com/DataDog/ddtest", + }, + Framework: mockFramework, + } + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{} + mockDurationsClient := &MockTestSuiteDurationsClient{ + Durations: map[string]map[string]testoptimization.TestSuiteDurationInfo{ + "module1": { + "suite1": { + SourceFile: "spec/file1_test.rb", + Duration: testoptimization.DurationPercentiles{P50: "10", P90: "20"}, + }, + "suite2": { + SourceFile: "spec/file2_test.rb", + Duration: testoptimization.DurationPercentiles{P50: "30", P90: "40"}, + }, + }, + }, + } + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) + + err := runner.PrepareTestOptimization(ctx) + if err != nil { + t.Fatalf("PrepareTestOptimization() should not fail with durations data, got: %v", err) + } + + if len(runner.testSuiteDurations) != 1 { + t.Fatalf("Expected stored durations data, got %v", runner.testSuiteDurations) + } + + if runner.testFiles["spec/file1_test.rb"] != 2 { + t.Errorf("Expected file1 weight to remain test-count based (2), got %d", runner.testFiles["spec/file1_test.rb"]) + } + if runner.testFiles["spec/file2_test.rb"] != 1 { + t.Errorf("Expected file2 weight to remain test-count based (1), got %d", runner.testFiles["spec/file2_test.rb"]) + } + + if !strings.Contains(logs.String(), "level=DEBUG") || !strings.Contains(logs.String(), "Found test suite durations") || !strings.Contains(logs.String(), "testSuitesCount=2") { + t.Errorf("Expected DEBUG log for non-empty durations response, got logs: %s", logs.String()) + } } func TestTestRunner_PrepareTestOptimization_PlatformDetectionError(t *testing.T) { @@ -103,7 +276,7 @@ func TestTestRunner_PrepareTestOptimization_PlatformDetectionError(t *testing.T) mockOptimizationClient := &MockTestOptimizationClient{} - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -130,7 +303,7 @@ func TestTestRunner_PrepareTestOptimization_TagsCreationError(t *testing.T) { mockOptimizationClient := &MockTestOptimizationClient{} - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -166,7 +339,7 @@ func TestTestRunner_PrepareTestOptimization_OptimizationClientInitError(t *testi InitializeErr: errors.New("client initialization failed"), } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -194,7 +367,7 @@ func TestTestRunner_PrepareTestOptimization_FrameworkDetectionError(t *testing.T mockOptimizationClient := &MockTestOptimizationClient{} - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -226,7 +399,7 @@ func TestTestRunner_PrepareTestOptimization_TestDiscoveryError(t *testing.T) { mockOptimizationClient := &MockTestOptimizationClient{} - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -255,7 +428,7 @@ func TestTestRunner_PrepareTestOptimization_EmptyTests(t *testing.T) { mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} mockOptimizationClient := &MockTestOptimizationClient{SkippableTests: map[string]bool{}} - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -296,7 +469,7 @@ func TestTestRunner_PrepareTestOptimization_AllTestsSkipped(t *testing.T) { }, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -354,7 +527,7 @@ func TestTestRunner_PrepareTestOptimization_RuntimeTagsOverride(t *testing.T) { SkippableTests: map[string]bool{}, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -419,7 +592,7 @@ func TestTestRunner_PrepareTestOptimization_RuntimeTagsOverrideInvalidJSON(t *te mockOptimizationClient := &MockTestOptimizationClient{} - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -473,7 +646,7 @@ func TestTestRunner_PrepareTestOptimization_NoRuntimeTagsOverride(t *testing.T) SkippableTests: map[string]bool{}, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) @@ -558,7 +731,7 @@ func TestPrepareTestOptimization_ITRFullDiscovery_SubdirRootRelativePath_Normali SkippableTests: map[string]bool{}, // No tests skipped (ITR enabled but all tests need to run) } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) if err != nil { @@ -622,7 +795,7 @@ func TestPrepareTestOptimization_RepoRootRun_LeavesRepoRelativePathsUnchanged(t SkippableTests: map[string]bool{}, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) if err != nil { @@ -659,7 +832,7 @@ func TestPrepareTestOptimization_FastDiscovery_PathsRemainUnchanged(t *testing.T SkippableTests: map[string]bool{}, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) if err != nil { @@ -724,7 +897,7 @@ func TestPrepareTestOptimization_ITRPathNormalization_PrefixMismatchUnchanged(t SkippableTests: map[string]bool{}, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) if err != nil { @@ -805,7 +978,7 @@ func TestPrepareTestOptimization_ITRSubdir_SkipMatching_WithSuitePathsMatchingCw }, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.PrepareTestOptimization(ctx) if err != nil { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 42140c0..3513dae 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -24,28 +24,39 @@ type Runner interface { type TestRunner struct { // the keys are file paths, the values are "durations" - currently just the number of tests in a file testFiles map[string]int + testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo skippablePercentage float64 platformDetector platform.PlatformDetector optimizationClient testoptimization.TestOptimizationClient + durationsClient testoptimization.TestSuiteDurationsClient ciProviderDetector ciprovider.CIProviderDetector } func New() *TestRunner { return &TestRunner{ testFiles: make(map[string]int), + testSuiteDurations: make(map[string]map[string]testoptimization.TestSuiteDurationInfo), skippablePercentage: 0.0, platformDetector: platform.NewPlatformDetector(), optimizationClient: testoptimization.NewDatadogClient(), + durationsClient: testoptimization.NewDurationsClient(), ciProviderDetector: ciprovider.NewCIProviderDetector(), } } -func NewWithDependencies(platformDetector platform.PlatformDetector, optimizationClient testoptimization.TestOptimizationClient, ciProviderDetector ciprovider.CIProviderDetector) *TestRunner { +func NewWithDependencies( + platformDetector platform.PlatformDetector, + optimizationClient testoptimization.TestOptimizationClient, + durationsClient testoptimization.TestSuiteDurationsClient, + ciProviderDetector ciprovider.CIProviderDetector, +) *TestRunner { return &TestRunner{ testFiles: make(map[string]int), + testSuiteDurations: make(map[string]map[string]testoptimization.TestSuiteDurationInfo), skippablePercentage: 0.0, platformDetector: platformDetector, optimizationClient: optimizationClient, + durationsClient: durationsClient, ciProviderDetector: ciProviderDetector, } } diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index b417fc9..18484d2 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -158,6 +158,27 @@ func (m *MockTestOptimizationClient) StoreCacheAndExit() { m.ShutdownCalled = true } +type MockTestSuiteDurationsClient struct { + Durations map[string]map[string]testoptimization.TestSuiteDurationInfo + Err error + Called bool + RepositoryURL string + Service string +} + +func (m *MockTestSuiteDurationsClient) GetTestSuiteDurations(repositoryURL, service string) (map[string]map[string]testoptimization.TestSuiteDurationInfo, error) { + m.Called = true + m.RepositoryURL = repositoryURL + m.Service = service + if m.Err != nil { + return nil, m.Err + } + if m.Durations == nil { + return map[string]map[string]testoptimization.TestSuiteDurationInfo{}, nil + } + return m.Durations, nil +} + // MockCIProvider mocks a CI provider type MockCIProvider struct { ProviderName string @@ -216,14 +237,19 @@ func TestNew(t *testing.T) { if runner.optimizationClient == nil { t.Error("New() should initialize optimizationClient") } + + if runner.durationsClient == nil { + t.Error("New() should initialize durationsClient") + } } func TestNewWithDependencies(t *testing.T) { mockPlatformDetector := &MockPlatformDetector{} mockOptimizationClient := &MockTestOptimizationClient{} + mockDurationsClient := &MockTestSuiteDurationsClient{} mockCIProviderDetector := newDefaultMockCIProviderDetector() - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockCIProviderDetector) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, mockCIProviderDetector) if runner == nil { t.Error("NewWithDependencies() should return non-nil TestRunner") @@ -237,6 +263,10 @@ func TestNewWithDependencies(t *testing.T) { if runner.optimizationClient != mockOptimizationClient { t.Error("NewWithDependencies() should use injected optimizationClient") } + + if runner.durationsClient != mockDurationsClient { + t.Error("NewWithDependencies() should use injected durationsClient") + } } func TestTestRunner_Setup_WithParallelRunners(t *testing.T) { @@ -286,7 +316,7 @@ func TestTestRunner_Setup_WithParallelRunners(t *testing.T) { }, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) // Run Setup err := runner.Plan(context.Background()) @@ -356,7 +386,7 @@ func TestTestRunner_Setup_WithCIProvider(t *testing.T) { CIProvider: mockCIProvider, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockCIProviderDetector) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) // Run Setup err := runner.Plan(context.Background()) @@ -410,7 +440,7 @@ func TestTestRunner_Setup_CIProviderDetectionFailure(t *testing.T) { Err: errors.New("no CI provider detected"), } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockCIProviderDetector) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) // Run Setup - should succeed even if CI provider detection fails err := runner.Plan(context.Background()) @@ -455,7 +485,7 @@ func TestTestRunner_Setup_CIProviderConfigureFailure(t *testing.T) { CIProvider: mockCIProvider, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockCIProviderDetector) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) // Run Setup - should succeed even if CI provider configuration fails err := runner.Plan(context.Background()) @@ -511,7 +541,7 @@ func TestTestRunner_Setup_WithTestSplit(t *testing.T) { SkippableTests: map[string]bool{}, // No tests skipped } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) // Run Setup err := runner.Plan(context.Background()) @@ -599,7 +629,7 @@ func TestTestRunner_Setup_WithTestSplit(t *testing.T) { // Reinitialize settings to pick up environment variables settings.Init() - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) // Run Setup err := runner.Plan(context.Background()) @@ -716,7 +746,7 @@ func TestTestRunner_Plan_SubdirRootRelativeDiscovery_WritesNormalizedPaths(t *te SkippableTests: map[string]bool{}, } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) err := runner.Plan(context.Background()) if err != nil { diff --git a/internal/testoptimization/durations_client.go b/internal/testoptimization/durations_client.go new file mode 100644 index 0000000..b24ad76 --- /dev/null +++ b/internal/testoptimization/durations_client.go @@ -0,0 +1,295 @@ +package testoptimization + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "math" + "math/rand/v2" + "net/http" + "os" + "strings" + "time" + + "github.com/DataDog/ddtest/civisibility" + "github.com/DataDog/ddtest/civisibility/constants" +) + +const ( + durationsRequestType string = "ci_app_ddtest_test_suite_durations_request" + durationsURLPath string = "api/v2/ci/ddtest/test_suite_durations" + + defaultDurationsPageSize int = 500 + maxDurationsRetries int = 3 +) + +type ( + // request types + + durationsRequest struct { + Data durationsRequestData `json:"data"` + } + + durationsRequestData struct { + Type string `json:"type"` + Attributes durationsRequestAttributes `json:"attributes"` + } + + durationsRequestAttributes struct { + RepositoryURL string `json:"repository_url"` + Service string `json:"service,omitempty"` + PageInfo *durationsRequestPageInfo `json:"page_info,omitempty"` + } + + durationsRequestPageInfo struct { + PageSize int `json:"page_size,omitempty"` + PageState string `json:"page_state,omitempty"` + } + + // response types + + durationsResponse struct { + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes durationsResponseAttributes `json:"attributes"` + } `json:"data"` + } + + durationsResponseAttributes struct { + TestSuites map[string]map[string]TestSuiteDurationInfo `json:"test_suites"` + PageInfo *durationsResponsePageInfo `json:"page_info,omitempty"` + } + + durationsResponsePageInfo struct { + Cursor string `json:"cursor,omitempty"` + Size int `json:"size,omitempty"` + HasNext bool `json:"has_next"` + } + + // public response types + + TestSuiteDurationInfo struct { + SourceFile string `json:"source_file"` + Duration DurationPercentiles `json:"duration"` + } + + DurationPercentiles struct { + P50 string `json:"p50"` + P90 string `json:"p90"` + } +) + +// TestSuiteDurationsClient defines the interface for fetching test suite durations +type TestSuiteDurationsClient interface { + GetTestSuiteDurations(repositoryURL, service string) (map[string]map[string]TestSuiteDurationInfo, error) +} + +// DurationsAPI abstracts the HTTP endpoint for testability (equivalent of CIVisibilityIntegrations) +type DurationsAPI interface { + FetchTestSuiteDurations(repositoryURL, service, cursor string, pageSize int) (*durationsResponseAttributes, error) +} + +// DatadogDurationsClient implements TestSuiteDurationsClient (equivalent of DatadogClient) +type DatadogDurationsClient struct { + api DurationsAPI +} + +func NewDurationsClient() *DatadogDurationsClient { + return &DatadogDurationsClient{ + api: NewDatadogDurationsAPI(), + } +} + +func NewDurationsClientWithDependencies(api DurationsAPI) *DatadogDurationsClient { + return &DatadogDurationsClient{ + api: api, + } +} + +func (c *DatadogDurationsClient) GetTestSuiteDurations(repositoryURL, service string) (map[string]map[string]TestSuiteDurationInfo, error) { + startTime := time.Now() + allSuites := make(map[string]map[string]TestSuiteDurationInfo) + + slog.Debug("Fetching test suite durations...") + + cursor := "" + for { + data, err := c.api.FetchTestSuiteDurations(repositoryURL, service, cursor, defaultDurationsPageSize) + if err != nil { + return nil, fmt.Errorf("fetching test suite durations: %w", err) + } + + for module, suites := range data.TestSuites { + if _, ok := allSuites[module]; !ok { + allSuites[module] = make(map[string]TestSuiteDurationInfo) + } + for suite, info := range suites { + allSuites[module][suite] = info + } + } + + if data.PageInfo == nil || !data.PageInfo.HasNext { + break + } + cursor = data.PageInfo.Cursor + } + + duration := time.Since(startTime) + totalSuites := 0 + for _, suites := range allSuites { + totalSuites += len(suites) + } + slog.Debug("Finished fetching test suite durations", "modules", len(allSuites), "suites", totalSuites, "duration", duration) + + return allSuites, nil +} + +// DatadogDurationsAPI implements DurationsAPI using real HTTP calls (equivalent of DatadogCIVisibilityIntegrations) +type DatadogDurationsAPI struct { + baseURL string + headers map[string]string + httpClient *http.Client +} + +func NewDatadogDurationsAPI() *DatadogDurationsAPI { + headers := map[string]string{} + var baseURL string + + agentlessEnabled := civisibility.BoolEnv(constants.CIVisibilityAgentlessEnabledEnvironmentVariable, false) + if agentlessEnabled { + apiKey := os.Getenv(constants.APIKeyEnvironmentVariable) + if apiKey == "" { + slog.Error("An API key is required for agentless mode. Use the DD_API_KEY env variable to set it") + return nil + } + headers["dd-api-key"] = apiKey + + agentlessURL := os.Getenv(constants.CIVisibilityAgentlessURLEnvironmentVariable) + if agentlessURL == "" { + site := "datadoghq.com" + if v := os.Getenv("DD_SITE"); v != "" { + site = v + } + baseURL = fmt.Sprintf("https://api.%s", site) + } else { + baseURL = agentlessURL + } + } else { + headers["X-Datadog-EVP-Subdomain"] = "api" + agentURL := civisibility.AgentURLFromEnv() + baseURL = agentURL.String() + } + + id := fmt.Sprint(rand.Uint64() & math.MaxInt64) + headers["trace_id"] = id + headers["parent_id"] = id + + slog.Debug("DurationsAPI: client created", + "agentless", agentlessEnabled, "url", baseURL) + + return &DatadogDurationsAPI{ + baseURL: baseURL, + headers: headers, + httpClient: &http.Client{ + Timeout: 45 * time.Second, + }, + } +} + +func (c *DatadogDurationsAPI) FetchTestSuiteDurations(repositoryURL, service, cursor string, pageSize int) (*durationsResponseAttributes, error) { + if repositoryURL == "" { + return nil, fmt.Errorf("repository URL is required") + } + + var pageInfo *durationsRequestPageInfo + if pageSize > 0 || cursor != "" { + pageInfo = &durationsRequestPageInfo{ + PageSize: pageSize, + PageState: cursor, + } + } + + body := durationsRequest{ + Data: durationsRequestData{ + Type: durationsRequestType, + Attributes: durationsRequestAttributes{ + RepositoryURL: repositoryURL, + Service: service, + PageInfo: pageInfo, + }, + }, + } + + requestURL := c.getURLPath(durationsURLPath) + + var lastErr error + for attempt := range maxDurationsRetries { + result, err := c.doPost(requestURL, body) + if err == nil { + return result, nil + } + lastErr = err + slog.Debug("DurationsAPI: request failed, retrying", "attempt", attempt+1, "error", err) + time.Sleep(time.Duration(100*(1<= 300 { + return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, truncateBody(respBody)) + } + + slog.Debug("test_suite_durations", "responseBody", string(respBody)) + + var responseObject durationsResponse + if err := json.Unmarshal(respBody, &responseObject); err != nil { + return nil, fmt.Errorf("unmarshalling response: %w", err) + } + + return &responseObject.Data.Attributes, nil +} + +func truncateBody(body []byte) string { + s := string(body) + if len(s) > 256 { + return s[:256] + "..." + } + return s +} diff --git a/internal/testoptimization/durations_client_test.go b/internal/testoptimization/durations_client_test.go new file mode 100644 index 0000000..b8c0e82 --- /dev/null +++ b/internal/testoptimization/durations_client_test.go @@ -0,0 +1,444 @@ +package testoptimization + +import ( + "fmt" + "testing" +) + +// MockDurationsAPI implements DurationsAPI for testing (equivalent of MockCIVisibilityIntegrations) +type MockDurationsAPI struct { + FetchCalled bool + RepositoryURL string + Service string + Cursors []string + Responses []*durationsResponseAttributes + ResponseErrors []error + callIndex int +} + +func (m *MockDurationsAPI) FetchTestSuiteDurations(repositoryURL, service, cursor string, pageSize int) (*durationsResponseAttributes, error) { + m.FetchCalled = true + m.RepositoryURL = repositoryURL + m.Service = service + m.Cursors = append(m.Cursors, cursor) + + if m.callIndex < len(m.ResponseErrors) && m.ResponseErrors[m.callIndex] != nil { + err := m.ResponseErrors[m.callIndex] + m.callIndex++ + return nil, err + } + + if m.callIndex < len(m.Responses) { + resp := m.Responses[m.callIndex] + m.callIndex++ + return resp, nil + } + + return &durationsResponseAttributes{ + TestSuites: make(map[string]map[string]TestSuiteDurationInfo), + }, nil +} + +func TestNewDurationsClientWithDependencies(t *testing.T) { + mockAPI := &MockDurationsAPI{} + client := NewDurationsClientWithDependencies(mockAPI) + + if client == nil { + t.Error("NewDurationsClientWithDependencies() should return non-nil client") + } +} + +func TestDurationsClient_GetTestSuiteDurations_SinglePage(t *testing.T) { + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite1": { + SourceFile: "spec/user_spec.rb", + Duration: DurationPercentiles{P50: "280000000", P90: "350000000"}, + }, + "suite2": { + SourceFile: "spec/order_spec.rb", + Duration: DurationPercentiles{P50: "100000000", P90: "150000000"}, + }, + }, + "module2": { + "suite3": { + SourceFile: "spec/product_spec.rb", + Duration: DurationPercentiles{P50: "500000000", P90: "600000000"}, + }, + }, + }, + }, + }, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + + if err != nil { + t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + } + + if !mockAPI.FetchCalled { + t.Error("GetTestSuiteDurations() should call FetchTestSuiteDurations") + } + + if mockAPI.RepositoryURL != "github.com/DataDog/foo" { + t.Errorf("Expected repository URL 'github.com/DataDog/foo', got '%s'", mockAPI.RepositoryURL) + } + + if mockAPI.Service != "my-service" { + t.Errorf("Expected service 'my-service', got '%s'", mockAPI.Service) + } + + if len(result) != 2 { + t.Errorf("Expected 2 modules, got %d", len(result)) + } + + module1, exists := result["module1"] + if !exists { + t.Error("Expected module1 to exist") + return + } + + if len(module1) != 2 { + t.Errorf("Expected 2 suites in module1, got %d", len(module1)) + } + + suite1, exists := module1["suite1"] + if !exists { + t.Error("Expected suite1 to exist in module1") + return + } + + if suite1.SourceFile != "spec/user_spec.rb" { + t.Errorf("Expected source file 'spec/user_spec.rb', got '%s'", suite1.SourceFile) + } + if suite1.Duration.P50 != "280000000" { + t.Errorf("Expected P50 '280000000', got '%s'", suite1.Duration.P50) + } + if suite1.Duration.P90 != "350000000" { + t.Errorf("Expected P90 '350000000', got '%s'", suite1.Duration.P90) + } + + module2, exists := result["module2"] + if !exists { + t.Error("Expected module2 to exist") + return + } + + if len(module2) != 1 { + t.Errorf("Expected 1 suite in module2, got %d", len(module2)) + } +} + +func TestDurationsClient_GetTestSuiteDurations_Pagination(t *testing.T) { + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite1": { + SourceFile: "spec/user_spec.rb", + Duration: DurationPercentiles{P50: "280000000", P90: "350000000"}, + }, + }, + }, + PageInfo: &durationsResponsePageInfo{ + Cursor: "abc123", + Size: 500, + HasNext: true, + }, + }, + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite2": { + SourceFile: "spec/order_spec.rb", + Duration: DurationPercentiles{P50: "100000000", P90: "150000000"}, + }, + }, + "module2": { + "suite3": { + SourceFile: "spec/product_spec.rb", + Duration: DurationPercentiles{P50: "500000000", P90: "600000000"}, + }, + }, + }, + PageInfo: &durationsResponsePageInfo{ + Cursor: "", + Size: 500, + HasNext: false, + }, + }, + }, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + + if err != nil { + t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + } + + // Verify pagination cursors were passed correctly + if len(mockAPI.Cursors) != 2 { + t.Errorf("Expected 2 API calls, got %d", len(mockAPI.Cursors)) + } + + if mockAPI.Cursors[0] != "" { + t.Errorf("First call should have empty cursor, got '%s'", mockAPI.Cursors[0]) + } + + if mockAPI.Cursors[1] != "abc123" { + t.Errorf("Second call should have cursor 'abc123', got '%s'", mockAPI.Cursors[1]) + } + + // Verify merged results + if len(result) != 2 { + t.Errorf("Expected 2 modules, got %d", len(result)) + } + + module1, exists := result["module1"] + if !exists { + t.Error("Expected module1 to exist") + return + } + + if len(module1) != 2 { + t.Errorf("Expected 2 suites in module1 (merged from both pages), got %d", len(module1)) + } + + if _, exists := module1["suite1"]; !exists { + t.Error("Expected suite1 to exist in module1 (from page 1)") + } + if _, exists := module1["suite2"]; !exists { + t.Error("Expected suite2 to exist in module1 (from page 2)") + } + + module2, exists := result["module2"] + if !exists { + t.Error("Expected module2 to exist") + return + } + + if len(module2) != 1 { + t.Errorf("Expected 1 suite in module2, got %d", len(module2)) + } +} + +func TestDurationsClient_GetTestSuiteDurations_EmptyResponse(t *testing.T) { + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{}, + }, + }, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + + if err != nil { + t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + } + + if result == nil { + t.Error("GetTestSuiteDurations() should return non-nil map even with empty data") + } + + if len(result) != 0 { + t.Errorf("GetTestSuiteDurations() should return empty map, got %d modules", len(result)) + } +} + +func TestDurationsClient_GetTestSuiteDurations_NilTestSuites(t *testing.T) { + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: nil, + }, + }, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + + if err != nil { + t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + } + + if result == nil { + t.Error("GetTestSuiteDurations() should return non-nil map even with nil test suites") + } + + if len(result) != 0 { + t.Errorf("GetTestSuiteDurations() should return empty map, got %d modules", len(result)) + } +} + +func TestDurationsClient_GetTestSuiteDurations_APIError(t *testing.T) { + mockAPI := &MockDurationsAPI{ + ResponseErrors: []error{fmt.Errorf("connection refused")}, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + + if err == nil { + t.Error("GetTestSuiteDurations() should return error when API fails") + } + + if result != nil { + t.Error("GetTestSuiteDurations() should return nil result when API fails") + } +} + +func TestDurationsClient_GetTestSuiteDurations_PaginationError(t *testing.T) { + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite1": { + SourceFile: "spec/user_spec.rb", + Duration: DurationPercentiles{P50: "280000000", P90: "350000000"}, + }, + }, + }, + PageInfo: &durationsResponsePageInfo{ + Cursor: "abc123", + Size: 500, + HasNext: true, + }, + }, + }, + ResponseErrors: []error{nil, fmt.Errorf("timeout on second page")}, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + + if err == nil { + t.Error("GetTestSuiteDurations() should return error when pagination fails") + } + + if result != nil { + t.Error("GetTestSuiteDurations() should return nil result when pagination fails") + } +} + +func TestDurationsClient_GetTestSuiteDurations_NilPageInfo(t *testing.T) { + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite1": { + SourceFile: "spec/user_spec.rb", + Duration: DurationPercentiles{P50: "280000000", P90: "350000000"}, + }, + }, + }, + PageInfo: nil, + }, + }, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + + if err != nil { + t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + } + + if len(result) != 1 { + t.Errorf("Expected 1 module, got %d", len(result)) + } + + // Should only make one API call (no pagination) + if len(mockAPI.Cursors) != 1 { + t.Errorf("Expected 1 API call when PageInfo is nil, got %d", len(mockAPI.Cursors)) + } +} + +func TestDurationsClient_GetTestSuiteDurations_ThreePages(t *testing.T) { + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite1": { + SourceFile: "spec/a_spec.rb", + Duration: DurationPercentiles{P50: "100", P90: "200"}, + }, + }, + }, + PageInfo: &durationsResponsePageInfo{Cursor: "page2", HasNext: true}, + }, + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite2": { + SourceFile: "spec/b_spec.rb", + Duration: DurationPercentiles{P50: "300", P90: "400"}, + }, + }, + }, + PageInfo: &durationsResponsePageInfo{Cursor: "page3", HasNext: true}, + }, + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite3": { + SourceFile: "spec/c_spec.rb", + Duration: DurationPercentiles{P50: "500", P90: "600"}, + }, + }, + }, + PageInfo: &durationsResponsePageInfo{HasNext: false}, + }, + }, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + + if err != nil { + t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + } + + if len(mockAPI.Cursors) != 3 { + t.Errorf("Expected 3 API calls, got %d", len(mockAPI.Cursors)) + } + + if mockAPI.Cursors[0] != "" { + t.Errorf("First cursor should be empty, got '%s'", mockAPI.Cursors[0]) + } + if mockAPI.Cursors[1] != "page2" { + t.Errorf("Second cursor should be 'page2', got '%s'", mockAPI.Cursors[1]) + } + if mockAPI.Cursors[2] != "page3" { + t.Errorf("Third cursor should be 'page3', got '%s'", mockAPI.Cursors[2]) + } + + module1 := result["module1"] + if len(module1) != 3 { + t.Errorf("Expected 3 suites merged in module1, got %d", len(module1)) + } +} + +func TestDatadogDurationsAPI_FetchTestSuiteDurations_EmptyRepositoryURL(t *testing.T) { + api := &DatadogDurationsAPI{ + baseURL: "https://api.datadoghq.com", + headers: map[string]string{"dd-api-key": "test-key"}, + } + + _, err := api.FetchTestSuiteDurations("", "my-service", "", 100) + + if err == nil { + t.Error("FetchTestSuiteDurations() should return error when repository URL is empty") + } +}