diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..70c9962 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build-and-test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + dotnet-quality: preview + + - name: Restore dependencies + run: dotnet restore + + - name: Build + run: dotnet build --no-restore --configuration Release + + - name: Test + run: dotnet test --no-build --configuration Release --verbosity normal diff --git a/CodeGenesis.Engine.Tests/Claude/ClaudeResponseTests.cs b/CodeGenesis.Engine.Tests/Claude/ClaudeResponseTests.cs new file mode 100644 index 0000000..751d6cf --- /dev/null +++ b/CodeGenesis.Engine.Tests/Claude/ClaudeResponseTests.cs @@ -0,0 +1,255 @@ +using CodeGenesis.Engine.Claude; +using FluentAssertions; + +namespace CodeGenesis.Engine.Tests.Claude; + +public class ClaudeResponseTests +{ + private static readonly TimeSpan TestDuration = TimeSpan.FromSeconds(5); + + #region FailureKind + + [Fact] + public void FailureKind_SuccessResponse_ReturnsNone() + { + var response = new ClaudeResponse { Success = true }; + + response.FailureKind.Should().Be(ClaudeFailureKind.None); + } + + [Theory] + [InlineData("timed out waiting for response")] + [InlineData("Process timed out")] + public void FailureKind_TimeoutWithExitCodeMinus1_ReturnsTimeout(string errorMsg) + { + var response = new ClaudeResponse + { + Success = false, + ExitCode = -1, + ErrorMessage = errorMsg + }; + + response.FailureKind.Should().Be(ClaudeFailureKind.Timeout); + } + + [Fact] + public void FailureKind_TimeoutWithoutExitCodeMinus1_IsNotTimeout() + { + var response = new ClaudeResponse + { + Success = false, + ExitCode = 1, + ErrorMessage = "timed out" + }; + + response.FailureKind.Should().NotBe(ClaudeFailureKind.Timeout); + } + + [Theory] + [InlineData("rate limit exceeded")] + [InlineData("429 Too Many Requests")] + [InlineData("server overloaded")] + [InlineData("too many requests")] + public void FailureKind_RateLimitMessages_ReturnsRateLimit(string errorMsg) + { + var response = new ClaudeResponse + { + Success = false, + ExitCode = 1, + ErrorMessage = errorMsg + }; + + response.FailureKind.Should().Be(ClaudeFailureKind.RateLimit); + } + + [Fact] + public void FailureKind_MaxTurnsMessage_ReturnsMaxTurns() + { + var response = new ClaudeResponse + { + Success = false, + ExitCode = 1, + ErrorMessage = "max_turns reached" + }; + + response.FailureKind.Should().Be(ClaudeFailureKind.MaxTurns); + } + + [Fact] + public void FailureKind_GenericError_ReturnsOther() + { + var response = new ClaudeResponse + { + Success = false, + ExitCode = 1, + ErrorMessage = "something unexpected" + }; + + response.FailureKind.Should().Be(ClaudeFailureKind.Other); + } + + [Fact] + public void FailureKind_NullErrorMessage_ReturnsOther() + { + var response = new ClaudeResponse + { + Success = false, + ExitCode = 1, + ErrorMessage = null + }; + + response.FailureKind.Should().Be(ClaudeFailureKind.Other); + } + + #endregion + + #region FromJson + + [Fact] + public void FromJson_SuccessResponse_ParsesCorrectly() + { + var json = """ + { + "result": "Hello, world!", + "usage": { + "input_tokens": 100, + "output_tokens": 50 + }, + "total_cost_usd": 0.005, + "num_turns": 1 + } + """; + + var response = ClaudeResponse.FromJson(json, TestDuration); + + response.Success.Should().BeTrue(); + response.Result.Should().Be("Hello, world!"); + response.InputTokens.Should().Be(100); + response.OutputTokens.Should().Be(50); + response.CostUsd.Should().Be(0.005); + response.Duration.Should().Be(TestDuration); + response.ExitCode.Should().Be(0); + } + + [Fact] + public void FromJson_WithCacheTokens_IncludesInInputTotal() + { + var json = """ + { + "result": "cached response", + "usage": { + "input_tokens": 100, + "cache_creation_input_tokens": 50, + "cache_read_input_tokens": 25, + "output_tokens": 30 + } + } + """; + + var response = ClaudeResponse.FromJson(json, TestDuration); + + response.Success.Should().BeTrue(); + response.InputTokens.Should().Be(175); // 100 + 50 + 25 + response.OutputTokens.Should().Be(30); + } + + [Fact] + public void FromJson_ErrorMaxTurns_ReturnsFailure() + { + var json = """ + { + "subtype": "error_max_turns", + "result": "max turns exceeded", + "usage": { + "input_tokens": 500, + "output_tokens": 200 + }, + "num_turns": 5 + } + """; + + var response = ClaudeResponse.FromJson(json, TestDuration); + + response.Success.Should().BeFalse(); + response.ErrorMessage.Should().Contain("max_turns limit"); + response.ErrorMessage.Should().Contain("5 turn(s)"); + response.InputTokens.Should().Be(500); + response.OutputTokens.Should().Be(200); + } + + [Fact] + public void FromJson_IsError_ReturnsFailure() + { + var json = """ + { + "is_error": true, + "result": "Something broke", + "usage": { + "input_tokens": 10, + "output_tokens": 5 + } + } + """; + + var response = ClaudeResponse.FromJson(json, TestDuration); + + response.Success.Should().BeFalse(); + response.ErrorMessage.Should().Contain("Something broke"); + } + + [Fact] + public void FromJson_InvalidJson_ReturnsParseFailure() + { + var json = "not valid json {{{"; + + var response = ClaudeResponse.FromJson(json, TestDuration); + + response.Success.Should().BeFalse(); + response.ErrorMessage.Should().Contain("Failed to parse"); + response.RawOutput.Should().Be(json); + response.Duration.Should().Be(TestDuration); + } + + [Fact] + public void FromJson_MissingUsage_DefaultsToZeroTokens() + { + var json = """{"result": "minimal"}"""; + + var response = ClaudeResponse.FromJson(json, TestDuration); + + response.Success.Should().BeTrue(); + response.InputTokens.Should().Be(0); + response.OutputTokens.Should().Be(0); + response.CostUsd.Should().BeNull(); + } + + [Fact] + public void FromJson_MissingResult_ReturnsNullResult() + { + var json = """{"usage": {"input_tokens": 10, "output_tokens": 5}}"""; + + var response = ClaudeResponse.FromJson(json, TestDuration); + + response.Success.Should().BeTrue(); + response.Result.Should().BeNull(); + } + + #endregion + + #region Failure factory + + [Fact] + public void Failure_CreatesFailedResponse() + { + var response = ClaudeResponse.Failure("bad stuff", 1, TestDuration); + + response.Success.Should().BeFalse(); + response.ErrorMessage.Should().Be("bad stuff"); + response.ExitCode.Should().Be(1); + response.Duration.Should().Be(TestDuration); + response.Result.Should().BeNull(); + response.RawOutput.Should().BeNull(); + } + + #endregion +} diff --git a/CodeGenesis.Engine.Tests/CodeGenesis.Engine.Tests.csproj b/CodeGenesis.Engine.Tests/CodeGenesis.Engine.Tests.csproj new file mode 100644 index 0000000..9369517 --- /dev/null +++ b/CodeGenesis.Engine.Tests/CodeGenesis.Engine.Tests.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + CodeGenesis.Engine.Tests + false + true + + + + + + + + + + + + + + + diff --git a/CodeGenesis.Engine.Tests/Config/McpServerConfigTests.cs b/CodeGenesis.Engine.Tests/Config/McpServerConfigTests.cs new file mode 100644 index 0000000..cc9b710 --- /dev/null +++ b/CodeGenesis.Engine.Tests/Config/McpServerConfigTests.cs @@ -0,0 +1,95 @@ +using CodeGenesis.Engine.Config; +using FluentAssertions; + +namespace CodeGenesis.Engine.Tests.Config; + +public class McpServerConfigTests +{ + [Fact] + public void ResolveTemplates_ResolvesCommand() + { + var config = new McpServerConfig + { + Command = "{{tool_path}}/server", + Args = [], + Env = new() + }; + + var resolved = config.ResolveTemplates(s => s.Replace("{{tool_path}}", "/usr/bin")); + + resolved.Command.Should().Be("/usr/bin/server"); + } + + [Fact] + public void ResolveTemplates_ResolvesArgs() + { + var config = new McpServerConfig + { + Command = "node", + Args = ["{{script}}", "--port", "{{port}}"], + Env = new() + }; + + var resolved = config.ResolveTemplates(s => s + .Replace("{{script}}", "server.js") + .Replace("{{port}}", "3000")); + + resolved.Args.Should().Equal("server.js", "--port", "3000"); + } + + [Fact] + public void ResolveTemplates_ResolvesEnvValues() + { + var config = new McpServerConfig + { + Command = "server", + Args = [], + Env = new() + { + ["API_KEY"] = "{{api_key}}", + ["BASE_URL"] = "{{base_url}}" + } + }; + + var resolved = config.ResolveTemplates(s => s + .Replace("{{api_key}}", "secret123") + .Replace("{{base_url}}", "https://api.example.com")); + + resolved.Env["API_KEY"].Should().Be("secret123"); + resolved.Env["BASE_URL"].Should().Be("https://api.example.com"); + } + + [Fact] + public void ResolveTemplates_DoesNotMutateOriginal() + { + var config = new McpServerConfig + { + Command = "{{cmd}}", + Args = ["{{arg}}"], + Env = new() { ["K"] = "{{v}}" } + }; + + config.ResolveTemplates(s => "resolved"); + + config.Command.Should().Be("{{cmd}}"); + config.Args.Should().Equal("{{arg}}"); + config.Env["K"].Should().Be("{{v}}"); + } + + [Fact] + public void ResolveTemplates_NoPlaceholders_ReturnsIdentical() + { + var config = new McpServerConfig + { + Command = "npx", + Args = ["-y", "@modelcontextprotocol/server"], + Env = new() { ["NODE_ENV"] = "production" } + }; + + var resolved = config.ResolveTemplates(s => s); + + resolved.Command.Should().Be("npx"); + resolved.Args.Should().Equal("-y", "@modelcontextprotocol/server"); + resolved.Env["NODE_ENV"].Should().Be("production"); + } +} diff --git a/CodeGenesis.Engine.Tests/Config/PipelineConfigLoaderTests.cs b/CodeGenesis.Engine.Tests/Config/PipelineConfigLoaderTests.cs new file mode 100644 index 0000000..cd901d7 --- /dev/null +++ b/CodeGenesis.Engine.Tests/Config/PipelineConfigLoaderTests.cs @@ -0,0 +1,299 @@ +using CodeGenesis.Engine.Config; +using FluentAssertions; + +namespace CodeGenesis.Engine.Tests.Config; + +public class PipelineConfigLoaderTests +{ + #region ResolveTemplate + + [Fact] + public void ResolveTemplate_NoPlaceholders_ReturnsOriginal() + { + var result = PipelineConfigLoader.ResolveTemplate("plain text", []); + + result.Should().Be("plain text"); + } + + [Fact] + public void ResolveTemplate_SinglePlaceholder_Resolves() + { + var vars = new Dictionary { ["name"] = "Alice" }; + + var result = PipelineConfigLoader.ResolveTemplate("Hello {{name}}!", vars); + + result.Should().Be("Hello Alice!"); + } + + [Fact] + public void ResolveTemplate_MultiplePlaceholders_ResolvesAll() + { + var vars = new Dictionary + { + ["lang"] = "C#", + ["framework"] = ".NET" + }; + + var result = PipelineConfigLoader.ResolveTemplate("Build with {{lang}} on {{framework}}", vars); + + result.Should().Be("Build with C# on .NET"); + } + + [Fact] + public void ResolveTemplate_UnresolvedPlaceholder_LeavesAsIs() + { + var vars = new Dictionary { ["known"] = "yes" }; + + var result = PipelineConfigLoader.ResolveTemplate("{{known}} and {{unknown}}", vars); + + result.Should().Be("yes and {{unknown}}"); + } + + [Fact] + public void ResolveTemplate_PlaceholderWithSpaces_TrimsKey() + { + var vars = new Dictionary { ["key"] = "value" }; + + var result = PipelineConfigLoader.ResolveTemplate("{{ key }}", vars); + + result.Should().Be("value"); + } + + [Fact] + public void ResolveTemplate_DottedKey_Resolves() + { + var vars = new Dictionary { ["steps.plan"] = "my plan" }; + + var result = PipelineConfigLoader.ResolveTemplate("Plan: {{steps.plan}}", vars); + + result.Should().Be("Plan: my plan"); + } + + [Fact] + public void ResolveTemplate_EmptyString_ReturnsEmpty() + { + var result = PipelineConfigLoader.ResolveTemplate("", []); + + result.Should().BeEmpty(); + } + + [Fact] + public void ResolveTemplate_RepeatedPlaceholder_ResolvesAll() + { + var vars = new Dictionary { ["x"] = "1" }; + + var result = PipelineConfigLoader.ResolveTemplate("{{x}} + {{x}} = 2", vars); + + result.Should().Be("1 + 1 = 2"); + } + + #endregion + + #region LoadFromFile + + [Fact] + public void LoadFromFile_FileNotFound_ThrowsFileNotFoundException() + { + var act = () => PipelineConfigLoader.LoadFromFile("/nonexistent/pipeline.yaml"); + + act.Should().Throw(); + } + + [Fact] + public void LoadFromFile_ValidYaml_LoadsConfig() + { + var yaml = """ + pipeline: + name: Test Pipeline + description: A test + version: "1.0" + steps: + - name: step1 + prompt: Do something + """; + + var path = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid()}.yaml"); + File.WriteAllText(path, yaml); + try + { + var config = PipelineConfigLoader.LoadFromFile(path); + + config.Pipeline.Name.Should().Be("Test Pipeline"); + config.Pipeline.Description.Should().Be("A test"); + config.Pipeline.Version.Should().Be("1.0"); + config.Steps.Should().HaveCount(1); + config.Steps[0].Name.Should().Be("step1"); + config.Steps[0].Prompt.Should().Be("Do something"); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void LoadFromFile_EmptySteps_ThrowsInvalidOperation() + { + var yaml = """ + pipeline: + name: Empty + steps: [] + """; + + var path = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid()}.yaml"); + File.WriteAllText(path, yaml); + try + { + var act = () => PipelineConfigLoader.LoadFromFile(path); + + act.Should().Throw() + .WithMessage("*at least one step*"); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void LoadFromFile_StepMissingName_ThrowsInvalidOperation() + { + var yaml = """ + pipeline: + name: Bad + steps: + - prompt: Do something + """; + + var path = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid()}.yaml"); + File.WriteAllText(path, yaml); + try + { + var act = () => PipelineConfigLoader.LoadFromFile(path); + + act.Should().Throw() + .WithMessage("*missing a 'name'*"); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void LoadFromFile_StepMissingPromptAndContext_ThrowsInvalidOperation() + { + var yaml = """ + pipeline: + name: Bad + steps: + - name: empty-step + """; + + var path = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid()}.yaml"); + File.WriteAllText(path, yaml); + try + { + var act = () => PipelineConfigLoader.LoadFromFile(path); + + act.Should().Throw() + .WithMessage("*must have either a 'prompt' or a 'context'*"); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void LoadFromFile_ForeachMissingCollection_ThrowsInvalidOperation() + { + var yaml = """ + pipeline: + name: Bad + steps: + - foreach: + steps: + - name: sub + prompt: do + """; + + var path = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid()}.yaml"); + File.WriteAllText(path, yaml); + try + { + var act = () => PipelineConfigLoader.LoadFromFile(path); + + act.Should().Throw() + .WithMessage("*missing 'collection'*"); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void LoadFromFile_ParallelMissingBranches_ThrowsInvalidOperation() + { + var yaml = """ + pipeline: + name: Bad + steps: + - parallel: + branches: [] + """; + + var path = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid()}.yaml"); + File.WriteAllText(path, yaml); + try + { + var act = () => PipelineConfigLoader.LoadFromFile(path); + + act.Should().Throw() + .WithMessage("*at least one branch*"); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void LoadFromFile_WithInputsAndOutputs_ParsesCorrectly() + { + var yaml = """ + pipeline: + name: Full + inputs: + task: + description: The task + default: hello + steps: + - name: step1 + prompt: "{{task}}" + outputs: + result: + source: step1 + description: The result + """; + + var path = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid()}.yaml"); + File.WriteAllText(path, yaml); + try + { + var config = PipelineConfigLoader.LoadFromFile(path); + + config.Inputs.Should().ContainKey("task"); + config.Inputs["task"].Default.Should().Be("hello"); + config.Outputs.Should().ContainKey("result"); + config.Outputs["result"].Source.Should().Be("step1"); + } + finally + { + File.Delete(path); + } + } + + #endregion +} diff --git a/CodeGenesis.Engine.Tests/Config/StepEntryTests.cs b/CodeGenesis.Engine.Tests/Config/StepEntryTests.cs new file mode 100644 index 0000000..e193049 --- /dev/null +++ b/CodeGenesis.Engine.Tests/Config/StepEntryTests.cs @@ -0,0 +1,142 @@ +using CodeGenesis.Engine.Config; +using FluentAssertions; + +namespace CodeGenesis.Engine.Tests.Config; + +public class StepEntryTests +{ + [Fact] + public void IsSimpleStep_WithNameOnly_ReturnsTrue() + { + var entry = new StepEntry { Name = "step1", Prompt = "do something" }; + + entry.IsSimpleStep.Should().BeTrue(); + entry.IsForeach.Should().BeFalse(); + entry.IsParallel.Should().BeFalse(); + entry.IsParallelForeach.Should().BeFalse(); + entry.IsApproval.Should().BeFalse(); + } + + [Fact] + public void IsForeach_WithForeachConfig_ReturnsTrue() + { + var entry = new StepEntry + { + Foreach = new ForeachConfig { Collection = "a,b,c" } + }; + + entry.IsForeach.Should().BeTrue(); + entry.IsSimpleStep.Should().BeFalse(); + } + + [Fact] + public void IsParallel_WithParallelConfig_ReturnsTrue() + { + var entry = new StepEntry + { + Parallel = new ParallelConfig() + }; + + entry.IsParallel.Should().BeTrue(); + entry.IsSimpleStep.Should().BeFalse(); + } + + [Fact] + public void IsParallelForeach_WithConfig_ReturnsTrue() + { + var entry = new StepEntry + { + ParallelForeach = new ParallelForeachConfig { Collection = "x,y" } + }; + + entry.IsParallelForeach.Should().BeTrue(); + entry.IsSimpleStep.Should().BeFalse(); + } + + [Fact] + public void IsApproval_WithConfig_ReturnsTrue() + { + var entry = new StepEntry + { + Approval = new ApprovalConfig { Name = "review" } + }; + + entry.IsApproval.Should().BeTrue(); + entry.IsSimpleStep.Should().BeFalse(); + } + + [Fact] + public void IsSimpleStep_NullNameNoComposite_ReturnsFalse() + { + var entry = new StepEntry { Prompt = "orphan prompt" }; + + entry.IsSimpleStep.Should().BeFalse(); + } + + [Fact] + public void ToStepConfig_MapsAllFields() + { + var entry = new StepEntry + { + Name = "build", + Agent = "coder", + Description = "Build the app", + SystemPrompt = "You are a builder", + Prompt = "Build it", + Model = "claude-sonnet-4-6", + Context = "./agents/builder", + MaxTurns = 10, + OutputKey = "build_result", + AllowedTools = ["Bash", "Read"], + Optional = true, + FailIf = "ERROR", + FailMessage = "Build failed", + RetryMax = 3, + RetryBackoffSeconds = 15, + RateLimitPauseSeconds = 30, + RateLimitMaxPauses = 2 + }; + + var config = entry.ToStepConfig(); + + config.Name.Should().Be("build"); + config.Agent.Should().Be("coder"); + config.Description.Should().Be("Build the app"); + config.SystemPrompt.Should().Be("You are a builder"); + config.Prompt.Should().Be("Build it"); + config.Model.Should().Be("claude-sonnet-4-6"); + config.Context.Should().Be("./agents/builder"); + config.MaxTurns.Should().Be(10); + config.OutputKey.Should().Be("build_result"); + config.AllowedTools.Should().Equal("Bash", "Read"); + config.Optional.Should().BeTrue(); + config.FailIf.Should().Be("ERROR"); + config.FailMessage.Should().Be("Build failed"); + config.RetryMax.Should().Be(3); + config.RetryBackoffSeconds.Should().Be(15); + config.RateLimitPauseSeconds.Should().Be(30); + config.RateLimitMaxPauses.Should().Be(2); + } + + [Fact] + public void ToStepConfig_NullFields_DefaultsCorrectly() + { + var entry = new StepEntry { Name = "minimal", Prompt = "go" }; + + var config = entry.ToStepConfig(); + + config.Name.Should().Be("minimal"); + config.Prompt.Should().Be("go"); + config.Agent.Should().BeNull(); + config.Description.Should().BeNull(); + config.SystemPrompt.Should().BeNull(); + config.Model.Should().BeNull(); + config.MaxTurns.Should().BeNull(); + config.OutputKey.Should().BeNull(); + config.AllowedTools.Should().BeNull(); + config.McpServers.Should().BeNull(); + config.Optional.Should().BeFalse(); + config.FailIf.Should().BeNull(); + config.RetryMax.Should().BeNull(); + } +} diff --git a/CodeGenesis.Engine.Tests/GlobalUsings.cs b/CodeGenesis.Engine.Tests/GlobalUsings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/CodeGenesis.Engine.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/CodeGenesis.Engine.Tests/Pipeline/CollectionParserTests.cs b/CodeGenesis.Engine.Tests/Pipeline/CollectionParserTests.cs new file mode 100644 index 0000000..23891ac --- /dev/null +++ b/CodeGenesis.Engine.Tests/Pipeline/CollectionParserTests.cs @@ -0,0 +1,100 @@ +using CodeGenesis.Engine.Pipeline; +using FluentAssertions; + +namespace CodeGenesis.Engine.Tests.Pipeline; + +public class CollectionParserTests +{ + [Fact] + public void Parse_EmptyString_ReturnsEmptyList() + { + CollectionParser.Parse("").Should().BeEmpty(); + } + + [Fact] + public void Parse_Whitespace_ReturnsEmptyList() + { + CollectionParser.Parse(" ").Should().BeEmpty(); + } + + [Fact] + public void Parse_JsonArray_ReturnsItems() + { + var result = CollectionParser.Parse("""["a", "b", "c"]"""); + + result.Should().Equal("a", "b", "c"); + } + + [Fact] + public void Parse_JsonArrayWithNumbers_ReturnsRawText() + { + var result = CollectionParser.Parse("[1, 2, 3]"); + + result.Should().Equal("1", "2", "3"); + } + + [Fact] + public void Parse_CommaSeparated_ReturnsItems() + { + var result = CollectionParser.Parse("alpha, beta, gamma"); + + result.Should().Equal("alpha", "beta", "gamma"); + } + + [Fact] + public void Parse_CommaSeparated_TrimsWhitespace() + { + var result = CollectionParser.Parse(" x , y , z "); + + result.Should().Equal("x", "y", "z"); + } + + [Fact] + public void Parse_CommaSeparated_RemovesEmptyEntries() + { + var result = CollectionParser.Parse("a,,b,"); + + result.Should().Equal("a", "b"); + } + + [Fact] + public void Parse_NewlineSeparated_ReturnsItems() + { + var result = CollectionParser.Parse("line1\nline2\nline3"); + + result.Should().Equal("line1", "line2", "line3"); + } + + [Fact] + public void Parse_NewlineSeparated_RemovesEmptyLines() + { + var result = CollectionParser.Parse("line1\n\nline2\n"); + + result.Should().Equal("line1", "line2"); + } + + [Fact] + public void Parse_SingleItem_ReturnsSingleElementList() + { + var result = CollectionParser.Parse("just-one"); + + result.Should().Equal("just-one"); + } + + [Fact] + public void Parse_InvalidJsonArray_FallsBackToCommaSeparated() + { + var result = CollectionParser.Parse("[broken, json"); + + result.Should().Equal("[broken", "json"); + } + + [Fact] + public void Parse_JsonArrayPrioritizedOverComma() + { + // A valid JSON array with commas should parse as JSON, not comma-separated + var result = CollectionParser.Parse("""["a,b", "c,d"]"""); + + result.Should().Equal("a,b", "c,d"); + } +} diff --git a/CodeGenesis.Engine.Tests/Pipeline/PipelineContextTests.cs b/CodeGenesis.Engine.Tests/Pipeline/PipelineContextTests.cs new file mode 100644 index 0000000..2c5223c --- /dev/null +++ b/CodeGenesis.Engine.Tests/Pipeline/PipelineContextTests.cs @@ -0,0 +1,88 @@ +using CodeGenesis.Engine.Pipeline; +using FluentAssertions; + +namespace CodeGenesis.Engine.Tests.Pipeline; + +public class PipelineContextTests +{ + [Fact] + public void Constructor_InitializesWithRequiredTaskDescription() + { + var ctx = new PipelineContext { TaskDescription = "Build something" }; + + ctx.TaskDescription.Should().Be("Build something"); + ctx.WorkingDirectory.Should().BeNull(); + ctx.StepOutputs.Should().BeEmpty(); + } + + [Fact] + public void Metrics_DefaultToZero() + { + var ctx = new PipelineContext { TaskDescription = "test" }; + + ctx.TotalInputTokens.Should().Be(0); + ctx.TotalOutputTokens.Should().Be(0); + ctx.TotalCostUsd.Should().Be(0); + ctx.TotalDuration.Should().Be(TimeSpan.Zero); + ctx.StepsCompleted.Should().Be(0); + ctx.StepsFailed.Should().Be(0); + } + + [Fact] + public void FailureInfo_DefaultsToNull() + { + var ctx = new PipelineContext { TaskDescription = "test" }; + + ctx.FailedStepName.Should().BeNull(); + ctx.FailureReason.Should().BeNull(); + } + + [Fact] + public void StepOutputs_CanStoreAndRetrieveValues() + { + var ctx = new PipelineContext { TaskDescription = "test" }; + + ctx.StepOutputs["plan"] = "my plan output"; + ctx.StepOutputs["code"] = "my code output"; + + ctx.StepOutputs.Should().HaveCount(2); + ctx.StepOutputs["plan"].Should().Be("my plan output"); + ctx.StepOutputs["code"].Should().Be("my code output"); + } + + [Fact] + public void Metrics_CanBeAccumulated() + { + var ctx = new PipelineContext { TaskDescription = "test" }; + + ctx.TotalInputTokens += 100; + ctx.TotalOutputTokens += 50; + ctx.TotalCostUsd += 0.01; + ctx.StepsCompleted++; + + ctx.TotalInputTokens += 200; + ctx.TotalOutputTokens += 75; + ctx.TotalCostUsd += 0.02; + ctx.StepsCompleted++; + + ctx.TotalInputTokens.Should().Be(300); + ctx.TotalOutputTokens.Should().Be(125); + ctx.TotalCostUsd.Should().BeApproximately(0.03, 0.0001); + ctx.StepsCompleted.Should().Be(2); + } + + [Fact] + public void StatusUpdate_CanBeSetAndInvoked() + { + var capturedMessage = ""; + var ctx = new PipelineContext + { + TaskDescription = "test", + StatusUpdate = msg => capturedMessage = msg + }; + + ctx.StatusUpdate?.Invoke("Working on step 1..."); + + capturedMessage.Should().Be("Working on step 1..."); + } +} diff --git a/CodeGenesis.Engine.Tests/Pipeline/RetryPolicyTests.cs b/CodeGenesis.Engine.Tests/Pipeline/RetryPolicyTests.cs new file mode 100644 index 0000000..4358e7a --- /dev/null +++ b/CodeGenesis.Engine.Tests/Pipeline/RetryPolicyTests.cs @@ -0,0 +1,91 @@ +using CodeGenesis.Engine.Config; +using CodeGenesis.Engine.Pipeline; +using FluentAssertions; + +namespace CodeGenesis.Engine.Tests.Pipeline; + +public class RetryPolicyTests +{ + [Fact] + public void Resolve_NoOverrides_ReturnsDefaults() + { + var step = new StepConfig { Name = "test", Prompt = "test" }; + + var policy = RetryPolicy.Resolve(step, null); + + policy.MaxRetries.Should().Be(0); + policy.BackoffSeconds.Should().Be(10); + policy.RateLimitPauseSeconds.Should().Be(60); + policy.MaxRateLimitPauses.Should().Be(5); + } + + [Fact] + public void Resolve_GlobalSettings_OverridesDefaults() + { + var step = new StepConfig { Name = "test", Prompt = "test" }; + var global = new PipelineSettings + { + RetryMax = 3, + RetryBackoffSeconds = 20, + RateLimitPauseSeconds = 120, + RateLimitMaxPauses = 10 + }; + + var policy = RetryPolicy.Resolve(step, global); + + policy.MaxRetries.Should().Be(3); + policy.BackoffSeconds.Should().Be(20); + policy.RateLimitPauseSeconds.Should().Be(120); + policy.MaxRateLimitPauses.Should().Be(10); + } + + [Fact] + public void Resolve_StepSettings_OverridesGlobal() + { + var step = new StepConfig + { + Name = "test", + Prompt = "test", + RetryMax = 5, + RetryBackoffSeconds = 30, + RateLimitPauseSeconds = 90, + RateLimitMaxPauses = 2 + }; + var global = new PipelineSettings + { + RetryMax = 3, + RetryBackoffSeconds = 20, + RateLimitPauseSeconds = 120, + RateLimitMaxPauses = 10 + }; + + var policy = RetryPolicy.Resolve(step, global); + + policy.MaxRetries.Should().Be(5); + policy.BackoffSeconds.Should().Be(30); + policy.RateLimitPauseSeconds.Should().Be(90); + policy.MaxRateLimitPauses.Should().Be(2); + } + + [Fact] + public void Resolve_PartialStepOverride_FallsBackToGlobalThenDefaults() + { + var step = new StepConfig + { + Name = "test", + Prompt = "test", + RetryMax = 2 + }; + var global = new PipelineSettings + { + RetryBackoffSeconds = 15 + }; + + var policy = RetryPolicy.Resolve(step, global); + + policy.MaxRetries.Should().Be(2); // step + policy.BackoffSeconds.Should().Be(15); // global + policy.RateLimitPauseSeconds.Should().Be(60); // default + policy.MaxRateLimitPauses.Should().Be(5); // default + } +} diff --git a/CodeGenesis.Engine.Tests/Pipeline/StepResultTests.cs b/CodeGenesis.Engine.Tests/Pipeline/StepResultTests.cs new file mode 100644 index 0000000..4814c4e --- /dev/null +++ b/CodeGenesis.Engine.Tests/Pipeline/StepResultTests.cs @@ -0,0 +1,94 @@ +using CodeGenesis.Engine.Pipeline; +using FluentAssertions; + +namespace CodeGenesis.Engine.Tests.Pipeline; + +public class StepResultTests +{ + [Fact] + public void SuccessResult_HasCorrectProperties() + { + var result = new StepResult + { + Outcome = StepOutcome.Success, + Output = "done", + Duration = TimeSpan.FromSeconds(5), + TokensUsed = 150, + CostUsd = 0.01 + }; + + result.Outcome.Should().Be(StepOutcome.Success); + result.Output.Should().Be("done"); + result.Error.Should().BeNull(); + result.Duration.Should().Be(TimeSpan.FromSeconds(5)); + result.TokensUsed.Should().Be(150); + result.CostUsd.Should().Be(0.01); + } + + [Fact] + public void FailedResult_HasErrorSet() + { + var result = new StepResult + { + Outcome = StepOutcome.Failed, + Error = "something went wrong" + }; + + result.Outcome.Should().Be(StepOutcome.Failed); + result.Error.Should().Be("something went wrong"); + result.Output.Should().BeNull(); + } + + [Fact] + public void SkippedResult_IsValid() + { + var result = new StepResult + { + Outcome = StepOutcome.Skipped, + Error = "optional step failed" + }; + + result.Outcome.Should().Be(StepOutcome.Skipped); + } + + [Fact] + public void Record_Equality_WorksCorrectly() + { + var a = new StepResult + { + Outcome = StepOutcome.Success, + Output = "done", + Duration = TimeSpan.FromSeconds(1), + TokensUsed = 100, + CostUsd = 0.01 + }; + var b = new StepResult + { + Outcome = StepOutcome.Success, + Output = "done", + Duration = TimeSpan.FromSeconds(1), + TokensUsed = 100, + CostUsd = 0.01 + }; + + a.Should().Be(b); + } + + [Fact] + public void Record_Inequality_DifferentOutcome() + { + var a = new StepResult { Outcome = StepOutcome.Success }; + var b = new StepResult { Outcome = StepOutcome.Failed }; + + a.Should().NotBe(b); + } + + [Fact] + public void StepOutcome_HasExpectedValues() + { + Enum.GetValues().Should().Equal( + StepOutcome.Success, + StepOutcome.Failed, + StepOutcome.Skipped); + } +} diff --git a/CodeGenesis.Engine.Tests/Steps/DynamicStepTests.cs b/CodeGenesis.Engine.Tests/Steps/DynamicStepTests.cs new file mode 100644 index 0000000..ef57269 --- /dev/null +++ b/CodeGenesis.Engine.Tests/Steps/DynamicStepTests.cs @@ -0,0 +1,449 @@ +using CodeGenesis.Engine.Claude; +using CodeGenesis.Engine.Config; +using CodeGenesis.Engine.Pipeline; +using CodeGenesis.Engine.Steps; +using FluentAssertions; +using NSubstitute; + +namespace CodeGenesis.Engine.Tests.Steps; + +public class DynamicStepTests +{ + private readonly IClaudeRunner _claude = Substitute.For(); + + private static StepConfig MakeConfig( + string name = "test-step", + string prompt = "do something", + string? outputKey = null, + bool optional = false, + string? failIf = null, + string? failMessage = null) => new() + { + Name = name, + Prompt = prompt, + OutputKey = outputKey, + Optional = optional, + FailIf = failIf, + FailMessage = failMessage + }; + + private static PipelineContext MakeContext() => + new() { TaskDescription = "test" }; + + private DynamicStep MakeStep( + StepConfig? config = null, + string? model = null, + RetryPolicy? retryPolicy = null) => + new(_claude, config ?? MakeConfig(), "resolved prompt", null, model, null, retryPolicy); + + #region Properties + + [Fact] + public void Name_ReturnsStepConfigName() + { + var step = MakeStep(MakeConfig(name: "my-step")); + + step.Name.Should().Be("my-step"); + } + + [Fact] + public void Description_ReturnsStepConfigDescription() + { + var config = MakeConfig(); + config.Description = "My description"; + var step = MakeStep(config); + + step.Description.Should().Be("My description"); + } + + [Fact] + public void Description_FallsBackToName() + { + var step = MakeStep(MakeConfig(name: "fallback")); + + step.Description.Should().Be("fallback"); + } + + #endregion + + #region Success execution + + [Fact] + public async Task ExecuteAsync_Success_ReturnsSuccessResult() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = true, + Result = "done!", + InputTokens = 100, + OutputTokens = 50, + CostUsd = 0.01, + Duration = TimeSpan.FromSeconds(2) + }); + + var step = MakeStep(); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Success); + result.Output.Should().Be("done!"); + result.TokensUsed.Should().Be(150); + result.CostUsd.Should().Be(0.01); + } + + [Fact] + public async Task ExecuteAsync_Success_AccumulatesMetricsInContext() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = true, + Result = "ok", + InputTokens = 200, + OutputTokens = 100, + CostUsd = 0.05 + }); + + var step = MakeStep(); + var ctx = MakeContext(); + + await step.ExecuteAsync(ctx, CancellationToken.None); + + ctx.TotalInputTokens.Should().Be(200); + ctx.TotalOutputTokens.Should().Be(100); + ctx.TotalCostUsd.Should().Be(0.05); + } + + [Fact] + public async Task ExecuteAsync_WithOutputKey_StoresInStepOutputs() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = true, + Result = "plan output" + }); + + var step = MakeStep(MakeConfig(outputKey: "plan")); + var ctx = MakeContext(); + + await step.ExecuteAsync(ctx, CancellationToken.None); + + ctx.StepOutputs.Should().ContainKey("plan"); + ctx.StepOutputs["plan"].Should().Be("plan output"); + } + + [Fact] + public async Task ExecuteAsync_WithOutputKey_NullResult_StoresEmptyString() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = true, + Result = null + }); + + var step = MakeStep(MakeConfig(outputKey: "result")); + var ctx = MakeContext(); + + await step.ExecuteAsync(ctx, CancellationToken.None); + + ctx.StepOutputs["result"].Should().BeEmpty(); + } + + #endregion + + #region FailIf condition + + [Fact] + public async Task ExecuteAsync_FailIf_MatchesOutput_ReturnsFailed() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = true, + Result = "Found ERROR in the output" + }); + + var step = MakeStep(MakeConfig(failIf: "ERROR", failMessage: "Validation failed")); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Failed); + result.Error.Should().Be("Validation failed"); + } + + [Fact] + public async Task ExecuteAsync_FailIf_CaseInsensitive() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = true, + Result = "found error in the output" + }); + + var step = MakeStep(MakeConfig(failIf: "ERROR")); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Failed); + } + + [Fact] + public async Task ExecuteAsync_FailIf_NoMatch_ReturnsSuccess() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = true, + Result = "All good, no issues" + }); + + var step = MakeStep(MakeConfig(failIf: "ERROR")); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Success); + } + + #endregion + + #region Failure paths + + [Fact] + public async Task ExecuteAsync_NonTransientFailure_ReturnsFailedImmediately() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = false, + ErrorMessage = "max_turns reached", + Duration = TimeSpan.FromSeconds(1) + }); + + var step = MakeStep(retryPolicy: new RetryPolicy(3, 1, 60, 5)); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Failed); + result.Error.Should().Contain("max_turns"); + // Should NOT have retried for non-transient errors + await _claude.Received(1).RunAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_OptionalStep_FailureReturnsSkipped() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = false, + ErrorMessage = "some error" + }); + + var step = MakeStep(MakeConfig(optional: true)); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Skipped); + result.Error.Should().Be("some error"); + } + + [Fact] + public async Task ExecuteAsync_Timeout_RetriesWithBackoff() + { + var calls = 0; + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(_ => + { + calls++; + if (calls <= 2) + return new ClaudeResponse + { + Success = false, + ExitCode = -1, + ErrorMessage = "Process timed out", + Duration = TimeSpan.FromSeconds(1) + }; + + return new ClaudeResponse + { + Success = true, + Result = "finally done" + }; + }); + + var step = MakeStep(retryPolicy: new RetryPolicy(MaxRetries: 3, BackoffSeconds: 0, RateLimitPauseSeconds: 0, MaxRateLimitPauses: 5)); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Success); + calls.Should().Be(3); // 2 failures + 1 success + } + + [Fact] + public async Task ExecuteAsync_Timeout_ExhaustsRetries_ReturnsFailed() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = false, + ExitCode = -1, + ErrorMessage = "Process timed out", + Duration = TimeSpan.FromSeconds(1) + }); + + var step = MakeStep(retryPolicy: new RetryPolicy(MaxRetries: 1, BackoffSeconds: 0, RateLimitPauseSeconds: 0, MaxRateLimitPauses: 5)); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Failed); + // 1 initial attempt + 1 retry = 2 calls + await _claude.Received(2).RunAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_RateLimit_PausesAndRetries() + { + var calls = 0; + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(_ => + { + calls++; + if (calls == 1) + return new ClaudeResponse + { + Success = false, + ExitCode = 1, + ErrorMessage = "429 too many requests", + Duration = TimeSpan.FromSeconds(1) + }; + + return new ClaudeResponse + { + Success = true, + Result = "ok after rate limit" + }; + }); + + var step = MakeStep(retryPolicy: new RetryPolicy(MaxRetries: 0, BackoffSeconds: 0, RateLimitPauseSeconds: 0, MaxRateLimitPauses: 3)); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Success); + calls.Should().Be(2); + } + + [Fact] + public async Task ExecuteAsync_RateLimit_ExceedsMaxPauses_ReturnsFailed() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse + { + Success = false, + ExitCode = 1, + ErrorMessage = "rate limit exceeded", + Duration = TimeSpan.FromSeconds(1) + }); + + var step = MakeStep(retryPolicy: new RetryPolicy(MaxRetries: 0, BackoffSeconds: 0, RateLimitPauseSeconds: 0, MaxRateLimitPauses: 1)); + var ctx = MakeContext(); + + var result = await step.ExecuteAsync(ctx, CancellationToken.None); + + result.Outcome.Should().Be(StepOutcome.Failed); + // 1 initial + 1 rate limit pause then fail = 2 calls + await _claude.Received(2).RunAsync(Arg.Any(), Arg.Any()); + } + + #endregion + + #region Clone + + [Fact] + public void Clone_ReturnsIndependentCopy() + { + var step = MakeStep(MakeConfig(name: "original")); + + var clone = step.Clone(); + + clone.Name.Should().Be("original"); + clone.UpdateResolvedPrompt("updated prompt"); + + // Original should be unaffected + step.OriginalPromptTemplate.Should().Be("do something"); + } + + #endregion + + #region Request construction + + [Fact] + public async Task ExecuteAsync_PassesModelToRequest() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse { Success = true, Result = "ok" }); + + var step = MakeStep(model: "claude-sonnet-4-6"); + var ctx = MakeContext(); + + await step.ExecuteAsync(ctx, CancellationToken.None); + + await _claude.Received(1).RunAsync( + Arg.Is(r => r.Model == "claude-sonnet-4-6"), + Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_PassesWorkingDirectoryFromContext() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse { Success = true, Result = "ok" }); + + var step = MakeStep(); + var ctx = new PipelineContext + { + TaskDescription = "test", + WorkingDirectory = "/my/project" + }; + + await step.ExecuteAsync(ctx, CancellationToken.None); + + await _claude.Received(1).RunAsync( + Arg.Is(r => r.WorkingDirectory == "/my/project"), + Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_PassesAllowedTools() + { + _claude.RunAsync(Arg.Any(), Arg.Any()) + .Returns(new ClaudeResponse { Success = true, Result = "ok" }); + + var config = MakeConfig(); + config.AllowedTools = ["Bash", "Read", "Write"]; + var step = MakeStep(config); + var ctx = MakeContext(); + + await step.ExecuteAsync(ctx, CancellationToken.None); + + await _claude.Received(1).RunAsync( + Arg.Is(r => + r.AllowedTools.Count == 3 && + r.AllowedTools.Contains("Bash")), + Arg.Any()); + } + + #endregion +} diff --git a/README.md b/README.md index 9ea7fe0..25687aa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # CodeGenesis +[![CI](https://github.com/viamus/code-genesis/actions/workflows/ci.yml/badge.svg)](https://github.com/viamus/code-genesis/actions/workflows/ci.yml) + A .NET CLI engine that orchestrates multi-step AI pipelines using [Claude Code](https://docs.anthropic.com/en/docs/claude-code) as the execution backend. Define pipelines in YAML, compose agents with Markdown context bundles, and let Claude handle planning, execution, and validation. ## Prerequisites diff --git a/Solution.slnx b/Solution.slnx index 66446d1..0f24bf2 100644 --- a/Solution.slnx +++ b/Solution.slnx @@ -1,3 +1,4 @@ +