From 9b3abea62dff8c85fe7bf877786f0ad18376bf57 Mon Sep 17 00:00:00 2001 From: Marcus Vorwaller Date: Fri, 17 Apr 2026 04:23:12 -0700 Subject: [PATCH] feat: harden changelog-synth prompts Nightshift-Task: changelog-synth Nightshift-Ref: https://github.com/marcus/nightshift --- internal/orchestrator/orchestrator.go | 15 +++++ internal/orchestrator/orchestrator_test.go | 78 ++++++++++++++++++++++ internal/tasks/tasks.go | 12 ++-- internal/tasks/tasks_test.go | 17 +++++ website/docs/task-reference.md | 2 +- 5 files changed, 119 insertions(+), 5 deletions(-) diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index 519c380..cea2023 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -713,6 +713,13 @@ func (o *Orchestrator) PlanPrompt(task *tasks.Task) string { func taskSpecificPlanGuidance(task *tasks.Task) string { switch task.Type { + case tasks.TaskChangelogSynth: + return "\n\n## Task-Specific Guidance\n" + + "- Inspect the repo's current changelog and release signals before deciding the scope: CHANGELOG.md, release workflow files if present (for example .github/workflows/release.yml), and recent git tags/commits.\n" + + "- Compare the current branch to `main`; derive the commit range from `git merge-base main HEAD` through `HEAD`, and exclude merge commits so the draft is deterministic.\n" + + "- If the release boundary or target changelog section is unclear, state the assumptions you made.\n" + + "- Prefer updating the existing changelog artifact and structure instead of inventing a new format, and preserve prior changelog history.\n" + + "- Plan for stable Markdown sections such as Added, Changed, Fixed, Docs, Refactor, Tests, Chore, and Other. Omit empty sections when appropriate.\n" case tasks.TaskReleaseNotes: return "\n\n## Task-Specific Guidance\n" + "- Inspect the repo's existing release signals before deciding the scope: CHANGELOG.md, .github/workflows/release.yml, and recent git tags/commits.\n" + @@ -726,6 +733,14 @@ func taskSpecificPlanGuidance(task *tasks.Task) string { func taskSpecificImplementGuidance(task *tasks.Task) string { switch task.Type { + case tasks.TaskChangelogSynth: + return "\n\n## Task-Specific Guidance\n" + + "- Inspect the repo's current changelog and release signals before drafting: CHANGELOG.md, release workflow files if present (for example .github/workflows/release.yml), and recent git tags/commits.\n" + + "- Compare the current branch to `main`; derive the commit range from `git merge-base main HEAD` through `HEAD`, and exclude merge commits so the draft is deterministic.\n" + + "- Synthesize only the newest changelog section from supported commits, note assumptions if the release boundary is unclear, and do not invent entries.\n" + + "- Preserve prior changelog sections and structure; prefer updating the existing changelog artifact over creating a new format.\n" + + "- Organize the update into stable Markdown sections such as Added, Changed, Fixed, Docs, Refactor, Tests, Chore, and Other. Omit empty sections when appropriate.\n" + + "- Emit Markdown-ready changelog content only.\n" case tasks.TaskReleaseNotes: return "\n\n## Task-Specific Guidance\n" + "- Inspect the repo's existing release signals before drafting: CHANGELOG.md, .github/workflows/release.yml, and recent git tags/commits.\n" + diff --git a/internal/orchestrator/orchestrator_test.go b/internal/orchestrator/orchestrator_test.go index 6c9ba4d..69f350a 100644 --- a/internal/orchestrator/orchestrator_test.go +++ b/internal/orchestrator/orchestrator_test.go @@ -809,6 +809,78 @@ func TestBuildImplementPrompt_WithoutBranch(t *testing.T) { } } +func TestBuildPlanPrompt_ChangelogSynthIncludesRepoAwareGuidance(t *testing.T) { + o := New() + + task := &tasks.Task{ + ID: "changelog-synth:/tmp/nightshift", + Title: "Changelog Synthesizer", + Description: "Draft the next changelog update from branch commits", + Type: tasks.TaskChangelogSynth, + } + + prompt := o.buildPlanPrompt(task) + + expected := []string{ + "CHANGELOG.md", + "release workflow files if present", + ".github/workflows/release.yml", + "recent git tags/commits", + "current branch to `main`", + "`git merge-base main HEAD`", + "exclude merge commits", + "target changelog section is unclear", + "existing changelog artifact and structure", + "preserve prior changelog history", + "Added, Changed, Fixed, Docs, Refactor, Tests, Chore, and Other", + } + + for _, want := range expected { + if !strings.Contains(prompt, want) { + t.Errorf("plan prompt missing %q\nGot:\n%s", want, prompt) + } + } +} + +func TestBuildImplementPrompt_ChangelogSynthIncludesRepoAwareGuidance(t *testing.T) { + o := New() + + task := &tasks.Task{ + ID: "changelog-synth:/tmp/nightshift", + Title: "Changelog Synthesizer", + Description: "Draft the next changelog update from branch commits", + Type: tasks.TaskChangelogSynth, + } + plan := &PlanOutput{ + Steps: []string{"Inspect CHANGELOG.md", "Draft the next section"}, + Description: "Use the current branch diff against main to update the changelog.", + } + + prompt := o.buildImplementPrompt(task, plan, 1) + + expected := []string{ + "CHANGELOG.md", + "release workflow files if present", + ".github/workflows/release.yml", + "recent git tags/commits", + "current branch to `main`", + "`git merge-base main HEAD`", + "exclude merge commits", + "newest changelog section", + "do not invent entries", + "existing changelog artifact", + "prior changelog sections and structure", + "Added, Changed, Fixed, Docs, Refactor, Tests, Chore, and Other", + "Emit Markdown-ready changelog content only", + } + + for _, want := range expected { + if !strings.Contains(prompt, want) { + t.Errorf("implement prompt missing %q\nGot:\n%s", want, prompt) + } + } +} + func TestBuildPlanPrompt_ReleaseNotesIncludesRepoAwareGuidance(t *testing.T) { o := New() @@ -894,6 +966,12 @@ func TestBuildPrompts_GenericTasksDoNotReceiveReleaseNotesGuidance(t *testing.T) "release boundary", "current release-notes artifact and format", "Features, Fixes, Breaking Changes, and Other", + "release workflow files if present", + "`git merge-base main HEAD`", + "exclude merge commits", + "newest changelog section", + "Added, Changed, Fixed, Docs, Refactor, Tests, Chore, and Other", + "Emit Markdown-ready changelog content only", } for _, notWant := range unexpected { diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index 0959c08..dedbe63 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -338,10 +338,14 @@ Apply safe updates directly, and leave concise follow-ups for anything uncertain DefaultInterval: 24 * time.Hour, }, TaskChangelogSynth: { - Type: TaskChangelogSynth, - Category: CategoryPR, - Name: "Changelog Synthesizer", - Description: "Generate changelog from commits", + Type: TaskChangelogSynth, + Category: CategoryPR, + Name: "Changelog Synthesizer", + Description: `Inspect the current branch against main and synthesize the next changelog update for this repository. +Determine the commit range from git merge-base main HEAD through HEAD, excluding merge commits so the output stays deterministic. +Review the existing changelog artifact and preserve prior history and structure, updating only the newest relevant section instead of rewriting older entries. +Group entries into stable Markdown sections such as Added, Changed, Fixed, Docs, Refactor, Tests, Chore, and Other, omitting empty sections when appropriate. +Base every bullet on actual commits, do not invent changes, and emit Markdown-ready changelog content only.`, CostTier: CostLow, RiskLevel: RiskLow, DefaultInterval: 168 * time.Hour, diff --git a/internal/tasks/tasks_test.go b/internal/tasks/tasks_test.go index 03cdf18..e9d58f6 100644 --- a/internal/tasks/tasks_test.go +++ b/internal/tasks/tasks_test.go @@ -243,6 +243,23 @@ func TestTaskDefinitionEstimatedTokens(t *testing.T) { } } +func TestTaskChangelogSynthDescriptionIsDeterministicAndRepoAware(t *testing.T) { + def, err := GetDefinition(TaskChangelogSynth) + if err != nil { + t.Fatalf("GetDefinition(TaskChangelogSynth) returned error: %v", err) + } + + want := `Inspect the current branch against main and synthesize the next changelog update for this repository. +Determine the commit range from git merge-base main HEAD through HEAD, excluding merge commits so the output stays deterministic. +Review the existing changelog artifact and preserve prior history and structure, updating only the newest relevant section instead of rewriting older entries. +Group entries into stable Markdown sections such as Added, Changed, Fixed, Docs, Refactor, Tests, Chore, and Other, omitting empty sections when appropriate. +Base every bullet on actual commits, do not invent changes, and emit Markdown-ready changelog content only.` + + if def.Description != want { + t.Errorf("TaskChangelogSynth description mismatch\nGot:\n%s\n\nWant:\n%s", def.Description, want) + } +} + func TestRegistryCompleteness(t *testing.T) { // All task type constants should be in registry taskTypes := []TaskType{ diff --git a/website/docs/task-reference.md b/website/docs/task-reference.md index 754a067..89db0da 100644 --- a/website/docs/task-reference.md +++ b/website/docs/task-reference.md @@ -24,7 +24,7 @@ Fully formed, review-ready artifacts. These tasks create branches and open pull | `build-optimize` | Build Time Optimization | Optimize build configuration for faster builds | High | Medium | 7d | | `docs-backfill` | Documentation Backfiller | Generate missing documentation | Low | Low | 7d | | `commit-normalize` | Commit Message Normalizer | Standardize commit message format | Low | Low | 24h | -| `changelog-synth` | Changelog Synthesizer | Generate changelog from commits | Low | Low | 7d | +| `changelog-synth` | Changelog Synthesizer | Draft a deterministic changelog update from current-branch commits since `git merge-base main HEAD`, preserving existing `CHANGELOG.md` structure | Low | Low | 7d | | `release-notes` | Release Note Drafter | Draft release-ready notes for the next version | Low | Low | 7d | | `adr-draft` | ADR Drafter | Draft Architecture Decision Records | Medium | Low | 7d | | `td-review` | TD Review Session | Review open td reviews, fix obvious bugs, create tasks for bigger issues | High | Medium | 72h |