Skip to content

Add changelog-synth prompt guidance #128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
marcus wants to merge 1 commit into
base: main
Choose a base branch
from
+216 −4
Open

Add changelog-synth prompt guidance #128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 48 additions & 4 deletions internal/orchestrator/orchestrator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -711,12 +711,54 @@ func (o *Orchestrator) PlanPrompt(task *tasks.Task) string {
return o.buildPlanPrompt(task)
}

func taskSpecificPlanGuidance(task *tasks.Task) string {
switch task.Type {
case tasks.TaskReleaseNotes:
return "\n\n## Task-Specific Guidance\n" +
"- Inspect the repo's existing release signals before deciding the scope: CHANGELOG.md, .github/workflows/release.yml, and recent git tags/commits.\n" +
"- Use recent tags and commits to infer the release boundary. If the boundary is unclear, state the assumptions you made.\n" +
"- Prefer updating the current release-notes artifact and format instead of inventing a new template.\n" +
"- Plan for release notes with clear sections such as Features, Fixes, Breaking Changes, and Other. Omit empty sections when appropriate.\n"
case tasks.TaskChangelogSynth:
return changelogSynthGuidance("before deciding what changelog update to produce")
default:
return ""
}
}

func taskSpecificImplementGuidance(task *tasks.Task) string {
switch task.Type {
case tasks.TaskReleaseNotes:
return "\n\n## Task-Specific Guidance\n" +
"- Inspect the repo's existing release signals before drafting: CHANGELOG.md, .github/workflows/release.yml, and recent git tags/commits.\n" +
"- Use recent tags and commits to infer the release boundary. If the boundary is unclear, state the assumptions you made.\n" +
"- Prefer updating the current release-notes artifact and format instead of inventing a new template.\n" +
"- Organize the draft into clear sections such as Features, Fixes, Breaking Changes, and Other. Omit empty sections when appropriate.\n"
case tasks.TaskChangelogSynth:
return changelogSynthGuidance("before editing")
default:
return ""
}
}

func changelogSynthGuidance(inspectionContext string) string {
return "\n\n## Task-Specific Guidance\n" +
fmt.Sprintf("- Inspect `CHANGELOG.md`, `.github/workflows/release.yml`, current version signals, and recent git tags/commits %s.\n", inspectionContext) +
"- Use git tags, version signals, the current changelog contents, and release workflow expectations to infer the correct release boundary.\n" +
"- If tags, commits, and existing changelog entries do not line up, call out the mismatch and state the assumptions you made.\n" +
"- Update the existing changelog artifact instead of inventing a new file or format. Preserve the repo's current changelog structure, heading style, and ordering.\n" +
"- Group entries into the most appropriate existing sections, omit empty sections, and keep summaries concise and release-ready.\n" +
"- When commit intent is ambiguous, make the smallest defensible interpretation and state the assumption in your summary or PR notes.\n"
}

func (o *Orchestrator) buildPlanPrompt(task *tasks.Task) string {
branchInstruction := ""
if o.runMeta != nil && o.runMeta.Branch != "" {
branchInstruction = fmt.Sprintf("\n Create your feature branch from `%s`.", o.runMeta.Branch)
}

taskGuidance := taskSpecificPlanGuidance(task)

return fmt.Sprintf(`You are a planning agent. Create a detailed execution plan for this task.

## Task
Expand All @@ -733,15 +775,15 @@ Description: %s
Nightshift-Ref: https://github.com/marcus/nightshift
4. Analyze the task requirements
5. Identify files that need to be modified
6. Create step-by-step implementation plan
6. Create step-by-step implementation plan%s
7. Output only valid JSON (no markdown, no extra text). The output is read by a machine. Use this schema:

{
"steps": ["step1", "step2", ...],
"files": ["file1.go", "file2.go", ...],
"description": "overall approach"
}
`, task.ID, task.Title, task.Description, branchInstruction, task.Type)
`, task.ID, task.Title, task.Description, branchInstruction, task.Type, taskGuidance)
}

func (o *Orchestrator) buildImplementPrompt(task *tasks.Task, plan *PlanOutput, iteration int) string {
Expand All @@ -755,6 +797,8 @@ func (o *Orchestrator) buildImplementPrompt(task *tasks.Task, plan *PlanOutput,
branchInstruction = fmt.Sprintf("\n Checkout `%s` before creating your feature branch.", o.runMeta.Branch)
}

taskGuidance := taskSpecificImplementGuidance(task)

return fmt.Sprintf(`You are an implementation agent. Execute the plan for this task.

## Task
Expand All @@ -776,14 +820,14 @@ Description: %s
Nightshift-Ref: https://github.com/marcus/nightshift
2. Implement the plan step by step
3. Make all necessary code changes
4. Ensure tests pass
4. Ensure tests pass%s
5. Output a summary as JSON:

{
"files_modified": ["file1.go", ...],
"summary": "what was done"
}
`, task.ID, task.Title, task.Description, plan.Description, plan.Steps, iterationNote, branchInstruction, task.Type)
`, task.ID, task.Title, task.Description, plan.Description, plan.Steps, iterationNote, branchInstruction, task.Type, taskGuidance)
}

func (o *Orchestrator) buildReviewPrompt(task *tasks.Task, impl *ImplementOutput) string {
Expand Down
168 changes: 168 additions & 0 deletions internal/orchestrator/orchestrator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -809,6 +809,174 @@ func TestBuildImplementPrompt_WithoutBranch(t *testing.T) {
}
}

func TestBuildPlanPrompt_ReleaseNotesIncludesRepoAwareGuidance(t *testing.T) {
o := New()

task := &tasks.Task{
ID: "release-notes:/tmp/nightshift",
Title: "Release Note Drafter",
Description: "Draft release-ready notes for the next version",
Type: tasks.TaskReleaseNotes,
}

prompt := o.buildPlanPrompt(task)

expected := []string{
"CHANGELOG.md",
".github/workflows/release.yml",
"recent git tags/commits",
"release boundary",
"current release-notes artifact and format",
"Features, Fixes, Breaking Changes, and Other",
"state the assumptions",
}

for _, want := range expected {
if !strings.Contains(prompt, want) {
t.Errorf("plan prompt missing %q\nGot:\n%s", want, prompt)
}
}
}

func TestBuildImplementPrompt_ReleaseNotesIncludesRepoAwareGuidance(t *testing.T) {
o := New()

task := &tasks.Task{
ID: "release-notes:/tmp/nightshift",
Title: "Release Note Drafter",
Description: "Draft release-ready notes for the next version",
Type: tasks.TaskReleaseNotes,
}
plan := &PlanOutput{
Steps: []string{"Inspect release artifacts", "Draft the notes"},
Description: "Use the repo's release signals to draft notes for the next release.",
}

prompt := o.buildImplementPrompt(task, plan, 1)

expected := []string{
"CHANGELOG.md",
".github/workflows/release.yml",
"recent git tags/commits",
"release boundary",
"current release-notes artifact and format",
"Features, Fixes, Breaking Changes, and Other",
"state the assumptions",
}

for _, want := range expected {
if !strings.Contains(prompt, want) {
t.Errorf("implement prompt missing %q\nGot:\n%s", want, prompt)
}
}
}

func TestBuildPlanPrompt_ChangelogSynthIncludesRepoAwareGuidance(t *testing.T) {
o := New()

task := &tasks.Task{
ID: "changelog-synth:/tmp/nightshift",
Title: "Changelog Synthesizer",
Description: "Generate changelog from commits",
Type: tasks.TaskChangelogSynth,
}

prompt := o.buildPlanPrompt(task)

expected := []string{
"CHANGELOG.md",
".github/workflows/release.yml",
"current version signals",
"recent git tags/commits",
"correct release boundary",
"tags, commits, and existing changelog entries do not line up",
"Update the existing changelog artifact",
"Preserve the repo's current changelog structure",
"omit empty sections",
"state the assumptions",
}

for _, want := range expected {
if !strings.Contains(prompt, want) {
t.Errorf("plan prompt missing %q\nGot:\n%s", want, prompt)
}
}
}

func TestBuildImplementPrompt_ChangelogSynthIncludesRepoAwareGuidance(t *testing.T) {
o := New()

task := &tasks.Task{
ID: "changelog-synth:/tmp/nightshift",
Title: "Changelog Synthesizer",
Description: "Generate changelog from commits",
Type: tasks.TaskChangelogSynth,
}
plan := &PlanOutput{
Steps: []string{"Inspect changelog signals", "Update the existing entry"},
Description: "Use repo release signals to update the current changelog artifact.",
}

prompt := o.buildImplementPrompt(task, plan, 1)

expected := []string{
"CHANGELOG.md",
".github/workflows/release.yml",
"current version signals",
"recent git tags/commits",
"correct release boundary",
"tags, commits, and existing changelog entries do not line up",
"Update the existing changelog artifact",
"Preserve the repo's current changelog structure",
"omit empty sections",
"state the assumptions",
}

for _, want := range expected {
if !strings.Contains(prompt, want) {
t.Errorf("implement prompt missing %q\nGot:\n%s", want, prompt)
}
}
}

func TestBuildPrompts_GenericTasksDoNotReceiveReleaseOrChangelogGuidance(t *testing.T) {
o := New()

task := &tasks.Task{
ID: "lint-fix:/tmp/nightshift",
Title: "Linter Fixes",
Description: "Automatically fix linting errors and style issues",
Type: tasks.TaskLintFix,
}
plan := &PlanOutput{
Steps: []string{"Run linters", "Fix issues"},
Description: "Apply lint fixes.",
}

planPrompt := o.buildPlanPrompt(task)
implPrompt := o.buildImplementPrompt(task, plan, 1)

unexpected := []string{
"CHANGELOG.md",
".github/workflows/release.yml",
"current version signals",
"release boundary",
"current release-notes artifact and format",
"existing changelog artifact",
"current changelog structure",
"Features, Fixes, Breaking Changes, and Other",
}

for _, notWant := range unexpected {
if strings.Contains(planPrompt, notWant) {
t.Errorf("generic plan prompt should not contain %q\nGot:\n%s", notWant, planPrompt)
}
if strings.Contains(implPrompt, notWant) {
t.Errorf("generic implement prompt should not contain %q\nGot:\n%s", notWant, implPrompt)
}
}
}

func TestBuildMetadataBlock_WithBranch(t *testing.T) {
o := New()
o.SetRunMetadata(&RunMetadata{
Expand Down
Loading