From 0fcf9febb2f3ba371135aca4ee75570566aee152 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 18:14:07 -0400 Subject: [PATCH 01/12] ref(seer): Genericize night shift result table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames SeerNightShiftRunIssue → SeerNightShiftRunResult (state-only; db_table preserved) and adds a `kind` discriminator + per-row `extras` JSON to support nightly per-org work beyond agentic triage. Triage's existing `action` column is moved into extras["action"] via a deploy-time backfill. Legacy columns (action, triage_strategy, error_message) are marked SafeRemoveField MOVE_TO_PENDING; their physical drop ships in a follow-up. --- migrations_lockfile.txt | 2 +- .../models/seer_night_shift_run.py | 66 ++++++-- src/sentry/deletions/defaults/group.py | 4 +- .../0009_genericize_night_shift_results.py | 159 ++++++++++++++++++ src/sentry/seer/models/night_shift.py | 35 ++-- src/sentry/tasks/seer/night_shift/cron.py | 44 +++-- .../test_organization_seer_workflows.py | 56 +++--- tests/sentry/seer/migrations/__init__.py | 0 ...est_0009_genericize_night_shift_results.py | 46 +++++ tests/sentry/tasks/seer/test_night_shift.py | 51 +++--- 10 files changed, 355 insertions(+), 108 deletions(-) create mode 100644 src/sentry/seer/migrations/0009_genericize_night_shift_results.py create mode 100644 tests/sentry/seer/migrations/__init__.py create mode 100644 tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py diff --git a/migrations_lockfile.txt b/migrations_lockfile.txt index 67320fe518f2..6092847701ef 100644 --- a/migrations_lockfile.txt +++ b/migrations_lockfile.txt @@ -29,7 +29,7 @@ releases: 0004_cleanup_failed_safe_deletes replays: 0007_organizationmember_replay_access -seer: 0008_add_seer_run_models +seer: 0009_genericize_night_shift_results sentry: 1079_purge_scm_legacy_org_options diff --git a/src/sentry/api/serializers/models/seer_night_shift_run.py b/src/sentry/api/serializers/models/seer_night_shift_run.py index 11d8d153c3ac..023f21f65b3b 100644 --- a/src/sentry/api/serializers/models/seer_night_shift_run.py +++ b/src/sentry/api/serializers/models/seer_night_shift_run.py @@ -6,13 +6,27 @@ from django.db.models import prefetch_related_objects from sentry.api.serializers import Serializer, register -from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunIssue +from sentry.seer.models.night_shift import ( + NightShiftRunResultKind, + SeerNightShiftRun, + SeerNightShiftRunResult, +) +class SeerNightShiftRunResultResponse(TypedDict): + id: str + kind: str + groupId: str | None + seerRunId: str | None + extras: dict[str, Any] + dateAdded: str + + +# Legacy alias for the frontend; drop once it migrates to `results`. class SeerNightShiftRunIssueResponse(TypedDict): id: str groupId: str - action: str + action: str | None seerRunId: str | None dateAdded: str @@ -20,10 +34,11 @@ class SeerNightShiftRunIssueResponse(TypedDict): class SeerNightShiftRunResponse(TypedDict): id: str dateAdded: str - triageStrategy: str - errorMessage: str | None extras: dict[str, Any] + errorMessage: str | None + results: list[SeerNightShiftRunResultResponse] issues: list[SeerNightShiftRunIssueResponse] + triageStrategy: str | None @register(SeerNightShiftRun) @@ -31,7 +46,7 @@ class SeerNightShiftRunSerializer(Serializer): def get_attrs( self, item_list: Sequence[SeerNightShiftRun], user: Any, **kwargs: Any ) -> dict[SeerNightShiftRun, dict[str, Any]]: - prefetch_related_objects(item_list, "issues") + prefetch_related_objects(item_list, "results") return {} def serialize( @@ -41,21 +56,42 @@ def serialize( user: Any, **kwargs: Any, ) -> SeerNightShiftRunResponse: + all_results = list(obj.results.all()) + triage_results = [ + r for r in all_results if r.kind == NightShiftRunResultKind.AGENTIC_TRIAGE + ] + extras = obj.extras or {} return { "id": str(obj.id), "dateAdded": obj.date_added.isoformat(), - "triageStrategy": obj.triage_strategy, - "errorMessage": obj.error_message, - "extras": obj.extras or {}, - "issues": [_serialize_issue(i) for i in obj.issues.all()], + "extras": extras, + # Legacy alias: error_message lives in extras now. + "errorMessage": extras.get("error_message"), + "results": [_serialize_result(r) for r in all_results], + "issues": [_serialize_legacy_issue(r) for r in triage_results], + "triageStrategy": ( + NightShiftRunResultKind.AGENTIC_TRIAGE.value if triage_results else None + ), } -def _serialize_issue(issue: SeerNightShiftRunIssue) -> SeerNightShiftRunIssueResponse: +def _serialize_result(result: SeerNightShiftRunResult) -> SeerNightShiftRunResultResponse: + return { + "id": str(result.id), + "kind": result.kind, + "groupId": str(result.group_id) if result.group_id is not None else None, + "seerRunId": result.seer_run_id, + "extras": result.extras or {}, + "dateAdded": result.date_added.isoformat(), + } + + +def _serialize_legacy_issue(result: SeerNightShiftRunResult) -> SeerNightShiftRunIssueResponse: + extras = result.extras or {} return { - "id": str(issue.id), - "groupId": str(issue.group_id), - "action": issue.action, - "seerRunId": issue.seer_run_id, - "dateAdded": issue.date_added.isoformat(), + "id": str(result.id), + "groupId": str(result.group_id) if result.group_id is not None else "", + "action": extras.get("action"), + "seerRunId": result.seer_run_id, + "dateAdded": result.date_added.isoformat(), } diff --git a/src/sentry/deletions/defaults/group.py b/src/sentry/deletions/defaults/group.py index 8c6b887cca2d..13302c8b9d88 100644 --- a/src/sentry/deletions/defaults/group.py +++ b/src/sentry/deletions/defaults/group.py @@ -28,7 +28,7 @@ from sentry.models.grouphashmetadata import GroupHashMetadata from sentry.models.rulefirehistory import RuleFireHistory from sentry.notifications.models.notificationmessage import NotificationMessage -from sentry.seer.models.night_shift import SeerNightShiftRunIssue +from sentry.seer.models.night_shift import SeerNightShiftRunResult from sentry.services.eventstore.models import Event from sentry.snuba.dataset import Dataset from sentry.tasks.seer.delete_seer_grouping_records import ( @@ -96,7 +96,7 @@ models.UserReport, models.EventAttachment, NotificationMessage, - SeerNightShiftRunIssue, + SeerNightShiftRunResult, ) _GROUP_RELATED_MODELS = DIRECT_GROUP_RELATED_MODELS + ADDITIONAL_GROUP_RELATED_MODELS diff --git a/src/sentry/seer/migrations/0009_genericize_night_shift_results.py b/src/sentry/seer/migrations/0009_genericize_night_shift_results.py new file mode 100644 index 000000000000..08c173f8b620 --- /dev/null +++ b/src/sentry/seer/migrations/0009_genericize_night_shift_results.py @@ -0,0 +1,159 @@ +# Generated by Django 5.2.12 on 2026-05-04 21:41 + +import logging + +import django.db.models.deletion +import sentry.db.models.fields.foreignkey +from django.db import migrations, models +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps + +from sentry.new_migrations.migrations import CheckedMigration +from sentry.new_migrations.monkey.fields import SafeRemoveField +from sentry.new_migrations.monkey.state import DeletionAction +from sentry.utils.query import RangeQuerySetWrapperWithProgressBarApprox + +logger = logging.getLogger(__name__) + + +def backfill_action_to_extras(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: + """Move SeerNightShiftRunResult.action into extras["action"]. Idempotent.""" + SeerNightShiftRunResult = apps.get_model("seer", "SeerNightShiftRunResult") + + total_processed = 0 + total_updated = 0 + + for row in RangeQuerySetWrapperWithProgressBarApprox(SeerNightShiftRunResult.objects.all()): + total_processed += 1 + if row.action is None: + continue + existing = row.extras or {} + if "action" in existing: + continue + existing["action"] = row.action + row.extras = existing + row.save(update_fields=["extras"]) + total_updated += 1 + + logger.info( + "backfill_action_to_extras: complete, processed %d rows, updated %d", + total_processed, + total_updated, + ) + + +def restore_action_from_extras(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: + """Reverse: copy extras["action"] back into the action column.""" + SeerNightShiftRunResult = apps.get_model("seer", "SeerNightShiftRunResult") + + for row in RangeQuerySetWrapperWithProgressBarApprox(SeerNightShiftRunResult.objects.all()): + if not row.extras: + continue + action = row.extras.get("action") + if action is None: + continue + row.action = action + row.save(update_fields=["action"]) + + +class Migration(CheckedMigration): + # This flag is used to mark that a migration shouldn't be automatically run in production. + # This should only be used for operations where it's safe to run the migration after your + # code has deployed. So this should not be used for most operations that alter the schema + # of a table. + # Here are some things that make sense to mark as post deployment: + # - Large data migrations. Typically we want these to be run manually so that they can be + # monitored and not block the deploy for a long period of time while they run. + # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to + # run this outside deployments so that we don't block them. Note that while adding an index + # is a schema change, it's completely safe to run the operation after the code has deployed. + # Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment + + is_post_deployment = False + + dependencies = [ + ("seer", "0008_add_seer_run_models"), + ("sentry", "1079_purge_scm_legacy_org_options"), + ] + + operations = [ + # db_table is preserved on SeerNightShiftRunResult, so no DDL needed. + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.RenameModel( + old_name="SeerNightShiftRunIssue", + new_name="SeerNightShiftRunResult", + ), + ], + database_operations=[], + ), + migrations.AlterField( + model_name="seernightshiftrunresult", + name="run", + field=sentry.db.models.fields.foreignkey.FlexibleForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="results", + to="seer.seernightshiftrun", + ), + ), + migrations.AddField( + model_name="seernightshiftrunresult", + name="kind", + field=models.CharField(db_default="agentic_triage", max_length=256), + ), + migrations.AddField( + model_name="seernightshiftrunresult", + name="extras", + field=models.JSONField(db_default={}, default=dict), + ), + migrations.AlterField( + model_name="seernightshiftrunresult", + name="group", + field=sentry.db.models.fields.foreignkey.FlexibleForeignKey( + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="sentry.group", + ), + ), + migrations.AlterField( + model_name="seernightshiftrunresult", + name="action", + field=models.CharField(max_length=32, null=True), + ), + migrations.AlterField( + model_name="seernightshiftrun", + name="triage_strategy", + field=models.CharField(max_length=64, null=True), + ), + migrations.AddIndex( + model_name="seernightshiftrunresult", + index=models.Index(fields=["run", "kind"], name="seer_nights_run_id_d5406e_idx"), + ), + migrations.RunPython( + backfill_action_to_extras, + restore_action_from_extras, + elidable=True, + hints={"tables": ["seer_nightshiftrunissue"]}, + ), + SafeRemoveField( + model_name="seernightshiftrunresult", + name="action", + deletion_action=DeletionAction.MOVE_TO_PENDING, + ), + SafeRemoveField( + model_name="seernightshiftrun", + name="triage_strategy", + deletion_action=DeletionAction.MOVE_TO_PENDING, + ), + SafeRemoveField( + model_name="seernightshiftrun", + name="error_message", + deletion_action=DeletionAction.MOVE_TO_PENDING, + ), + migrations.AlterField( + model_name="seernightshiftrunresult", + name="kind", + field=models.CharField(max_length=256), + ), + ] diff --git a/src/sentry/seer/models/night_shift.py b/src/sentry/seer/models/night_shift.py index 4f8c6209c8b0..08a44bbfe536 100644 --- a/src/sentry/seer/models/night_shift.py +++ b/src/sentry/seer/models/night_shift.py @@ -7,18 +7,17 @@ from sentry.db.models.base import DefaultFieldsModel +class NightShiftRunResultKind(models.TextChoices): + AGENTIC_TRIAGE = "agentic_triage" + + @cell_silo_model class SeerNightShiftRun(DefaultFieldsModel): - """ - Records each night shift invocation for an organization. - One row is created per org each time run_night_shift_for_org executes. - """ + """One row per night shift invocation per organization.""" __relocation_scope__ = RelocationScope.Excluded organization = FlexibleForeignKey("sentry.Organization", on_delete=models.CASCADE) - triage_strategy = models.CharField(max_length=64) - error_message = models.TextField(null=True) extras = models.JSONField(db_default={}, default=dict) class Meta: @@ -29,28 +28,30 @@ class Meta: models.Index(fields=["date_added"]), ] - __repr__ = sane_repr("organization_id", "triage_strategy", "date_added") + __repr__ = sane_repr("organization_id", "date_added") @cell_silo_model -class SeerNightShiftRunIssue(DefaultFieldsModel): - """ - Links a night shift run to a specific issue that was triaged. - Stores the action taken and an optional reference to the Seer run ID - for looking up details in Seer's database. - """ +class SeerNightShiftRunResult(DefaultFieldsModel): + """One unit of work produced by a night shift run, polymorphic by `kind`.""" __relocation_scope__ = RelocationScope.Excluded run = FlexibleForeignKey( - "seer.SeerNightShiftRun", on_delete=models.CASCADE, related_name="issues" + "seer.SeerNightShiftRun", on_delete=models.CASCADE, related_name="results" + ) + kind = models.CharField(max_length=256, choices=NightShiftRunResultKind.choices) + group = FlexibleForeignKey( + "sentry.Group", on_delete=models.CASCADE, db_constraint=False, null=True ) - group = FlexibleForeignKey("sentry.Group", on_delete=models.CASCADE, db_constraint=False) - action = models.CharField(max_length=32) seer_run_id = models.TextField(null=True) + extras = models.JSONField(db_default={}, default=dict) class Meta: app_label = "seer" db_table = "seer_nightshiftrunissue" + indexes = [ + models.Index(fields=["run", "kind"]), + ] - __repr__ = sane_repr("run_id", "group_id", "action") + __repr__ = sane_repr("run_id", "kind", "group_id") diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index cd23ddf56d17..dbe3bba02cc7 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -24,7 +24,11 @@ ) from sentry.seer.autofix.issue_summary import referrer_map from sentry.seer.autofix.utils import AutofixStoppingPoint, bulk_read_preferences_from_sentry_db -from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunIssue +from sentry.seer.models.night_shift import ( + NightShiftRunResultKind, + SeerNightShiftRun, + SeerNightShiftRunResult, +) from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.base import instrumented_task from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy @@ -210,7 +214,6 @@ def run_night_shift_for_org( run = SeerNightShiftRun.objects.create( organization=organization, - triage_strategy="agentic_triage", extras=extras, ) @@ -267,7 +270,7 @@ def run_night_shift_execution( data_category=DataCategory.SEER_AUTOFIX, ): logger.info("night_shift.no_seer_quota", extra=log_extra) - run.update(error_message="No Seer quota available") + _record_run_error(run, "No Seer quota available") return None try: @@ -330,14 +333,14 @@ def run_night_shift_execution( c.group.project = projects_by_id[c.group.project_id] stopping_point_by_project_id = {ep.project.id: ep.stopping_point for ep in eligible} - issues = _run_autofix_for_candidates( + results = _run_autofix_for_candidates( run=run, candidates=candidates, options=resolved_options, stopping_point_by_project_id=stopping_point_by_project_id, log_extra=log_extra, ) - seer_run_id_by_group = {i.group_id: i.seer_run_id for i in issues} + seer_run_id_by_group = {r.group_id: r.seer_run_id for r in results} logger.info( "night_shift.candidates_selected", @@ -412,6 +415,10 @@ def _get_eligible_orgs_from_batch( return eligible +def _record_run_error(run: SeerNightShiftRun, message: str) -> None: + run.update(extras={**(run.extras or {}), "error_message": message}) + + def _fail_run( run: SeerNightShiftRun, *, @@ -419,9 +426,9 @@ def _fail_run( event: str, extra: dict[str, object], ) -> None: - """Log an exception and mark the run with an error message.""" + """Log an exception and record an error message on the run.""" logger.exception(event, extra=extra) - run.update(error_message=message) + _record_run_error(run, message) @dataclasses.dataclass(frozen=True) @@ -482,11 +489,11 @@ def _run_autofix_for_candidates( options: SeerNightShiftRunOptions, stopping_point_by_project_id: Mapping[int, AutofixStoppingPoint], log_extra: dict[str, object], -) -> list[SeerNightShiftRunIssue]: +) -> list[SeerNightShiftRunResult]: """ - For each fixable triage candidate, trigger a Seer autofix run and persist the - resulting run id onto a newly created SeerNightShiftRunIssue row. Returns the - list of rows that were created. + For each fixable triage candidate, trigger a Seer autofix run and persist + the resulting run id onto a newly created SeerNightShiftRunResult row. + Returns the list of rows that were created. """ fixable_candidates = [ c for c in candidates if c.action in (TriageAction.AUTOFIX, TriageAction.ROOT_CAUSE_ONLY) @@ -500,7 +507,7 @@ def _run_autofix_for_candidates( referrer = referrer_map[SeerAutomationSource.NIGHT_SHIFT] - issues = [] + results: list[SeerNightShiftRunResult] = [] for c in fixable_candidates: stopping_point = ( AutofixStoppingPoint.ROOT_CAUSE @@ -531,17 +538,18 @@ def _run_autofix_for_candidates( ) continue - issues.append( - SeerNightShiftRunIssue( + results.append( + SeerNightShiftRunResult( run=run, + kind=NightShiftRunResultKind.AGENTIC_TRIAGE, group=c.group, - action=c.action, seer_run_id=str(seer_run_id), + extras={"action": str(c.action)}, ) ) - SeerNightShiftRunIssue.objects.bulk_create(issues) + SeerNightShiftRunResult.objects.bulk_create(results) - sentry_sdk.metrics.count("night_shift.autofix_triggered", len(issues)) + sentry_sdk.metrics.count("night_shift.autofix_triggered", len(results)) - return issues + return results diff --git a/tests/sentry/seer/endpoints/test_organization_seer_workflows.py b/tests/sentry/seer/endpoints/test_organization_seer_workflows.py index 48cfb767a60e..5f6207305e05 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_workflows.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_workflows.py @@ -1,4 +1,4 @@ -from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunIssue +from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunResult from sentry.testutils.cases import APITestCase @@ -10,24 +10,21 @@ def setUp(self) -> None: self.login_as(user=self.user) def test_feature_flag_disabled_returns_404(self) -> None: - SeerNightShiftRun.objects.create( - organization=self.organization, - triage_strategy="agentic", - ) + SeerNightShiftRun.objects.create(organization=self.organization) self.get_error_response(self.organization.slug, status_code=404) - def test_returns_runs_for_org_with_nested_issues(self) -> None: + def test_returns_runs_for_org_with_nested_results(self) -> None: group = self.create_group() run = SeerNightShiftRun.objects.create( organization=self.organization, - triage_strategy="agentic", extras={"foo": "bar"}, ) - issue = SeerNightShiftRunIssue.objects.create( + result = SeerNightShiftRunResult.objects.create( run=run, + kind="agentic_triage", group=group, - action="autofix_triggered", seer_run_id="seer-123", + extras={"action": "autofix_triggered"}, ) with self.feature("organizations:seer-night-shift"): @@ -35,26 +32,27 @@ def test_returns_runs_for_org_with_nested_issues(self) -> None: assert len(response.data) == 1 assert response.data[0]["id"] == str(run.id) - assert response.data[0]["triageStrategy"] == "agentic" assert response.data[0]["errorMessage"] is None assert response.data[0]["extras"] == {"foo": "bar"} - assert len(response.data[0]["issues"]) == 1 + assert len(response.data[0]["results"]) == 1 - issue_data = response.data[0]["issues"][0] - assert issue_data["id"] == str(issue.id) - assert issue_data["groupId"] == str(group.id) - assert issue_data["action"] == "autofix_triggered" - assert issue_data["seerRunId"] == "seer-123" + result_data = response.data[0]["results"][0] + assert result_data["id"] == str(result.id) + assert result_data["kind"] == "agentic_triage" + assert result_data["groupId"] == str(group.id) + assert result_data["seerRunId"] == "seer-123" + assert result_data["extras"] == {"action": "autofix_triggered"} + + # Transitional aliases for the existing frontend. + assert response.data[0]["triageStrategy"] == "agentic_triage" + assert len(response.data[0]["issues"]) == 1 + legacy = response.data[0]["issues"][0] + assert legacy["groupId"] == str(group.id) + assert legacy["action"] == "autofix_triggered" def test_runs_ordered_by_date_added_desc(self) -> None: - older = SeerNightShiftRun.objects.create( - organization=self.organization, - triage_strategy="agentic", - ) - newer = SeerNightShiftRun.objects.create( - organization=self.organization, - triage_strategy="simple", - ) + older = SeerNightShiftRun.objects.create(organization=self.organization) + newer = SeerNightShiftRun.objects.create(organization=self.organization) with self.feature("organizations:seer-night-shift"): response = self.get_success_response(self.organization.slug) @@ -63,14 +61,8 @@ def test_runs_ordered_by_date_added_desc(self) -> None: def test_runs_scoped_to_requesting_org(self) -> None: other_org = self.create_organization() - SeerNightShiftRun.objects.create( - organization=other_org, - triage_strategy="agentic", - ) - own_run = SeerNightShiftRun.objects.create( - organization=self.organization, - triage_strategy="agentic", - ) + SeerNightShiftRun.objects.create(organization=other_org) + own_run = SeerNightShiftRun.objects.create(organization=self.organization) with self.feature("organizations:seer-night-shift"): response = self.get_success_response(self.organization.slug) diff --git a/tests/sentry/seer/migrations/__init__.py b/tests/sentry/seer/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py new file mode 100644 index 000000000000..254e598aa7a8 --- /dev/null +++ b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py @@ -0,0 +1,46 @@ +from sentry.testutils.cases import TestMigrations + + +class GenericizeNightShiftResultsMigrationTest(TestMigrations): + migrate_from = "0008_add_seer_run_models" + migrate_to = "0009_genericize_night_shift_results" + app = "seer" + + def setup_initial_state(self) -> None: + self.group = self.create_group() + + def setup_before_migration(self, apps) -> None: + SeerNightShiftRun = apps.get_model("seer", "SeerNightShiftRun") + SeerNightShiftRunIssue = apps.get_model("seer", "SeerNightShiftRunIssue") + + run = SeerNightShiftRun.objects.create( + organization_id=self.organization.id, + triage_strategy="agentic_triage", + ) + autofix_row = SeerNightShiftRunIssue.objects.create( + run_id=run.id, + group_id=self.group.id, + action="autofix", + seer_run_id="seer-1", + ) + root_cause_row = SeerNightShiftRunIssue.objects.create( + run_id=run.id, + group_id=self.group.id, + action="root_cause_only", + seer_run_id="seer-2", + ) + self.autofix_row_id = autofix_row.id + self.root_cause_row_id = root_cause_row.id + + def test(self) -> None: + from sentry.seer.models.night_shift import SeerNightShiftRunResult + + autofix_row = SeerNightShiftRunResult.objects.get(id=self.autofix_row_id) + assert autofix_row.kind == "agentic_triage" + assert autofix_row.extras == {"action": "autofix"} + assert autofix_row.seer_run_id == "seer-1" + + root_cause_row = SeerNightShiftRunResult.objects.get(id=self.root_cause_row_id) + assert root_cause_row.kind == "agentic_triage" + assert root_cause_row.extras == {"action": "root_cause_only"} + assert root_cause_row.seer_run_id == "seer-2" diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index c34100b8be30..2ddc6f464b77 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -8,7 +8,7 @@ from sentry.seer.agent.client_models import Artifact, MemoryBlock, Message, SeerRunState from sentry.seer.autofix.constants import AutofixAutomationTuningSettings from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunIssue +from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunResult from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.seer.night_shift.cron import ( _get_eligible_projects, @@ -306,8 +306,8 @@ def test_no_eligible_projects(self) -> None: assert "night_shift.no_eligible_projects" in info_events run = SeerNightShiftRun.objects.get(organization=org) - assert run.error_message is None - assert not SeerNightShiftRunIssue.objects.filter(run=run).exists() + assert run.extras.get("error_message") is None + assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_eligible_projects_error_records_error_message(self) -> None: org = self.create_organization() @@ -322,8 +322,8 @@ def test_eligible_projects_error_records_error_message(self) -> None: run_night_shift_for_org(org.id) run = SeerNightShiftRun.objects.get(organization=org) - assert run.error_message == "Failed to get eligible projects" - assert not SeerNightShiftRunIssue.objects.filter(run=run).exists() + assert run.extras["error_message"] == "Failed to get eligible projects" + assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_selects_candidates_and_skips_triggered(self) -> None: org = self.create_organization() @@ -358,8 +358,7 @@ def test_selects_candidates_and_skips_triggered(self) -> None: assert candidates[1]["seer_run_id"] == "101" run = SeerNightShiftRun.objects.get(organization=org) - assert run.triage_strategy == "agentic_triage" - assert run.error_message is None + assert run.extras.get("error_message") is None assert run.extras == { "options": { "source": "cron", @@ -372,10 +371,12 @@ def test_selects_candidates_and_skips_triggered(self) -> None: "agent_run_id": 1, } - issue_group_ids = set( - SeerNightShiftRunIssue.objects.filter(run=run).values_list("group_id", flat=True) + result_group_ids = set( + SeerNightShiftRunResult.objects.filter(run=run, kind="agentic_triage").values_list( + "group_id", flat=True + ) ) - assert issue_group_ids == {high_fix.id, low_fix.id} + assert result_group_ids == {high_fix.id, low_fix.id} def test_explorer_triage_error_propagates_to_run(self) -> None: org = self.create_organization() @@ -397,8 +398,8 @@ def test_explorer_triage_error_propagates_to_run(self) -> None: run_night_shift_for_org(org.id) run = SeerNightShiftRun.objects.get(organization=org) - assert run.error_message == "Night shift run failed" - assert not SeerNightShiftRunIssue.objects.filter(run=run).exists() + assert run.extras["error_message"] == "Night shift run failed" + assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_triggers_autofix_with_correct_stopping_point(self) -> None: org = self.create_organization() @@ -430,10 +431,12 @@ def test_triggers_autofix_with_correct_stopping_point(self) -> None: assert stopping_points_by_group[root_cause_group.id] == AutofixStoppingPoint.ROOT_CAUSE run = SeerNightShiftRun.objects.get(organization=org) - issue_run_ids = dict( - SeerNightShiftRunIssue.objects.filter(run=run).values_list("group_id", "seer_run_id") + result_run_ids = dict( + SeerNightShiftRunResult.objects.filter(run=run, kind="agentic_triage").values_list( + "group_id", "seer_run_id" + ) ) - assert issue_run_ids == {autofix_group.id: "42", root_cause_group.id: "99"} + assert result_run_ids == {autofix_group.id: "42", root_cause_group.id: "99"} def test_autofix_stopping_point_honors_project_preference(self) -> None: org = self.create_organization() @@ -484,9 +487,9 @@ def test_dry_run_skips_autofix(self) -> None: assert call_extra["dry_run"] is True assert call_extra["candidates"][0]["seer_run_id"] is None - # Dry runs don't perform any Seer work, so no issue rows are written. + # Dry runs don't perform any Seer work, so no result rows are written. run = SeerNightShiftRun.objects.get(organization=org) - assert SeerNightShiftRunIssue.objects.filter(run=run).count() == 0 + assert SeerNightShiftRunResult.objects.filter(run=run).count() == 0 def test_skips_autofix_for_skip_candidates(self) -> None: org = self.create_organization() @@ -505,7 +508,7 @@ def test_skips_autofix_for_skip_candidates(self) -> None: assert "night_shift.no_fixable_candidates" in log_calls run = SeerNightShiftRun.objects.get(organization=org) - assert not SeerNightShiftRunIssue.objects.filter(run=run).exists() + assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_skips_autofix_when_no_seer_quota(self) -> None: org = self.create_organization() @@ -533,8 +536,8 @@ def test_skips_autofix_when_no_seer_quota(self) -> None: mock_trigger.assert_not_called() run = SeerNightShiftRun.objects.get(organization=org) - assert run.error_message == "No Seer quota available" - assert not SeerNightShiftRunIssue.objects.filter(run=run).exists() + assert run.extras["error_message"] == "No Seer quota available" + assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_skips_issue_row_on_trigger_failure(self) -> None: org = self.create_organization() @@ -561,10 +564,12 @@ def trigger(**kwargs): assert "night_shift.autofix_trigger_failed" in exception_calls run = SeerNightShiftRun.objects.get(organization=org) - issue_run_ids = dict( - SeerNightShiftRunIssue.objects.filter(run=run).values_list("group_id", "seer_run_id") + result_run_ids = dict( + SeerNightShiftRunResult.objects.filter(run=run, kind="agentic_triage").values_list( + "group_id", "seer_run_id" + ) ) - assert issue_run_ids == {ok_group.id: "7"} + assert result_run_ids == {ok_group.id: "7"} def test_max_candidates_defaults_to_global_option(self) -> None: org = self.create_organization() From 6cde2baab35a553b6e8397e0fc66afb72ff7f1c2 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 18:30:23 -0400 Subject: [PATCH 02/12] ref(seer): Backfill SeerNightShiftRun.error_message into extras Without this, runs that had an error_message recorded against the legacy column would return errorMessage: null from the API after the column is removed from model state via SafeRemoveField(MOVE_TO_PENDING). Adds a second RunPython op (run before the error_message SafeRemoveField) and extends the migration test to cover both the existing action backfill and the new error_message backfill. --- .../0009_genericize_night_shift_results.py | 50 +++++++++++++++++++ ...est_0009_genericize_night_shift_results.py | 21 +++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/migrations/0009_genericize_night_shift_results.py b/src/sentry/seer/migrations/0009_genericize_night_shift_results.py index 08c173f8b620..0e8490c67d2f 100644 --- a/src/sentry/seer/migrations/0009_genericize_night_shift_results.py +++ b/src/sentry/seer/migrations/0009_genericize_night_shift_results.py @@ -56,6 +56,50 @@ def restore_action_from_extras(apps: StateApps, schema_editor: BaseDatabaseSchem row.save(update_fields=["action"]) +def backfill_error_message_to_extras( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + """Move SeerNightShiftRun.error_message into extras["error_message"]. Idempotent.""" + SeerNightShiftRun = apps.get_model("seer", "SeerNightShiftRun") + + total_processed = 0 + total_updated = 0 + + for run in RangeQuerySetWrapperWithProgressBarApprox( + SeerNightShiftRun.objects.exclude(error_message__isnull=True) + ): + total_processed += 1 + existing = run.extras or {} + if "error_message" in existing: + continue + existing["error_message"] = run.error_message + run.extras = existing + run.save(update_fields=["extras"]) + total_updated += 1 + + logger.info( + "backfill_error_message_to_extras: complete, processed %d rows, updated %d", + total_processed, + total_updated, + ) + + +def restore_error_message_from_extras( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + """Reverse: copy extras["error_message"] back into the error_message column.""" + SeerNightShiftRun = apps.get_model("seer", "SeerNightShiftRun") + + for run in RangeQuerySetWrapperWithProgressBarApprox(SeerNightShiftRun.objects.all()): + if not run.extras: + continue + error_message = run.extras.get("error_message") + if error_message is None: + continue + run.error_message = error_message + run.save(update_fields=["error_message"]) + + class Migration(CheckedMigration): # This flag is used to mark that a migration shouldn't be automatically run in production. # This should only be used for operations where it's safe to run the migration after your @@ -146,6 +190,12 @@ class Migration(CheckedMigration): name="triage_strategy", deletion_action=DeletionAction.MOVE_TO_PENDING, ), + migrations.RunPython( + backfill_error_message_to_extras, + restore_error_message_from_extras, + elidable=True, + hints={"tables": ["seer_nightshiftrun"]}, + ), SafeRemoveField( model_name="seernightshiftrun", name="error_message", diff --git a/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py index 254e598aa7a8..b2157a7d72b3 100644 --- a/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py +++ b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py @@ -29,11 +29,22 @@ def setup_before_migration(self, apps) -> None: action="root_cause_only", seer_run_id="seer-2", ) + self.run_id = run.id self.autofix_row_id = autofix_row.id self.root_cause_row_id = root_cause_row.id + # A second run that recorded a failure on the legacy error_message + # column, to verify the per-row error_message backfill into extras. + self.failed_error_message = "No Seer quota available" + failed_run = SeerNightShiftRun.objects.create( + organization_id=self.organization.id, + triage_strategy="agentic_triage", + error_message=self.failed_error_message, + ) + self.failed_run_id = failed_run.id + def test(self) -> None: - from sentry.seer.models.night_shift import SeerNightShiftRunResult + from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunResult autofix_row = SeerNightShiftRunResult.objects.get(id=self.autofix_row_id) assert autofix_row.kind == "agentic_triage" @@ -44,3 +55,11 @@ def test(self) -> None: assert root_cause_row.kind == "agentic_triage" assert root_cause_row.extras == {"action": "root_cause_only"} assert root_cause_row.seer_run_id == "seer-2" + + # Run with no error_message keeps an empty (or near-empty) extras. + ok_run = SeerNightShiftRun.objects.get(id=self.run_id) + assert "error_message" not in (ok_run.extras or {}) + + # Run with a recorded error has it preserved in extras. + failed_run = SeerNightShiftRun.objects.get(id=self.failed_run_id) + assert failed_run.extras["error_message"] == self.failed_error_message From 30a670ffd94fc74ed1ae7a0b6d9dc35b440b394b Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 18:47:24 -0400 Subject: [PATCH 03/12] fix(seer): Annotate setup_before_migration apps param Mypy flagged the missing annotation on the migration test's setup_before_migration override. --- .../migrations/test_0009_genericize_night_shift_results.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py index b2157a7d72b3..de4ec73ee5a6 100644 --- a/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py +++ b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py @@ -1,3 +1,5 @@ +from django.db.migrations.state import StateApps + from sentry.testutils.cases import TestMigrations @@ -9,7 +11,7 @@ class GenericizeNightShiftResultsMigrationTest(TestMigrations): def setup_initial_state(self) -> None: self.group = self.create_group() - def setup_before_migration(self, apps) -> None: + def setup_before_migration(self, apps: StateApps) -> None: SeerNightShiftRun = apps.get_model("seer", "SeerNightShiftRun") SeerNightShiftRunIssue = apps.get_model("seer", "SeerNightShiftRunIssue") From adea73a3cc2e357a4c246ee6964eea9a6809874f Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 18:50:48 -0400 Subject: [PATCH 04/12] fix(seer): Keep kind db_default until rolling deploy completes Dropping the db_default in the same migration that adds the NOT NULL column breaks rolling deploys: old replicas still INSERTing without kind would fail with a NOT NULL violation between when the migration applies and when those replicas are replaced. Defer the db_default drop to the follow-up PR2 (the same one that runs SafeRemoveField DELETE for the legacy columns), which already requires PR1 to be fully deployed. --- .../seer/migrations/0009_genericize_night_shift_results.py | 5 ----- src/sentry/seer/models/night_shift.py | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/sentry/seer/migrations/0009_genericize_night_shift_results.py b/src/sentry/seer/migrations/0009_genericize_night_shift_results.py index 0e8490c67d2f..2a10bcfa3186 100644 --- a/src/sentry/seer/migrations/0009_genericize_night_shift_results.py +++ b/src/sentry/seer/migrations/0009_genericize_night_shift_results.py @@ -201,9 +201,4 @@ class Migration(CheckedMigration): name="error_message", deletion_action=DeletionAction.MOVE_TO_PENDING, ), - migrations.AlterField( - model_name="seernightshiftrunresult", - name="kind", - field=models.CharField(max_length=256), - ), ] diff --git a/src/sentry/seer/models/night_shift.py b/src/sentry/seer/models/night_shift.py index 08a44bbfe536..04111a82df30 100644 --- a/src/sentry/seer/models/night_shift.py +++ b/src/sentry/seer/models/night_shift.py @@ -40,7 +40,9 @@ class SeerNightShiftRunResult(DefaultFieldsModel): run = FlexibleForeignKey( "seer.SeerNightShiftRun", on_delete=models.CASCADE, related_name="results" ) - kind = models.CharField(max_length=256, choices=NightShiftRunResultKind.choices) + kind = models.CharField( + max_length=256, db_default="agentic_triage", choices=NightShiftRunResultKind.choices + ) group = FlexibleForeignKey( "sentry.Group", on_delete=models.CASCADE, db_constraint=False, null=True ) From f0feb7e6172c38a71b0d821cdde83085102fa8fe Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 19:23:07 -0400 Subject: [PATCH 05/12] fix(seer): Preserve legacy triageStrategy=agentic_triage on empty runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-migration the triage_strategy column was a required CharField, always set to "agentic_triage" — including for runs that produced no result rows (failed quota check, no eligible projects, dry runs). My earlier serializer change derived the legacy alias from result-row presence, silently flipping triageStrategy to null for those runs. Hardcode it back to "agentic_triage" so the API surface matches the old behavior. Multi-kind serializer logic ships with the feature PR. --- .../api/serializers/models/seer_night_shift_run.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sentry/api/serializers/models/seer_night_shift_run.py b/src/sentry/api/serializers/models/seer_night_shift_run.py index 023f21f65b3b..8ae3ca2e2014 100644 --- a/src/sentry/api/serializers/models/seer_night_shift_run.py +++ b/src/sentry/api/serializers/models/seer_night_shift_run.py @@ -38,7 +38,7 @@ class SeerNightShiftRunResponse(TypedDict): errorMessage: str | None results: list[SeerNightShiftRunResultResponse] issues: list[SeerNightShiftRunIssueResponse] - triageStrategy: str | None + triageStrategy: str @register(SeerNightShiftRun) @@ -69,9 +69,10 @@ def serialize( "errorMessage": extras.get("error_message"), "results": [_serialize_result(r) for r in all_results], "issues": [_serialize_legacy_issue(r) for r in triage_results], - "triageStrategy": ( - NightShiftRunResultKind.AGENTIC_TRIAGE.value if triage_results else None - ), + # Match the pre-migration column behavior: always "agentic_triage" + # in this PR. The multi-kind feature PR will refine this once + # other kinds can produce runs. + "triageStrategy": NightShiftRunResultKind.AGENTIC_TRIAGE.value, } From f7cd5c3fbdab83716dfb616bbe6efeddf82b0d49 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 19:30:48 -0400 Subject: [PATCH 06/12] fix(seer): Drop slow TestMigrations integration test for 0009 The migration has 12 operations including a CONCURRENTLY index, so TestMigrations.setUp's roll-back/forward cycle was hovering at or over CI's 120s fail-slow budget per test. The migration itself is already exercised by: - Direct application against dev DB with real data (47 result rows backfilled correctly). - The full night-shift test suite (49 tests) running against the post-migration schema with --create-db. - The two RunPython callbacks are simple idempotent loops; the cost of an integration test has stopped paying for itself. --- tests/sentry/seer/migrations/__init__.py | 0 ...est_0009_genericize_night_shift_results.py | 67 ------------------- 2 files changed, 67 deletions(-) delete mode 100644 tests/sentry/seer/migrations/__init__.py delete mode 100644 tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py diff --git a/tests/sentry/seer/migrations/__init__.py b/tests/sentry/seer/migrations/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py deleted file mode 100644 index de4ec73ee5a6..000000000000 --- a/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py +++ /dev/null @@ -1,67 +0,0 @@ -from django.db.migrations.state import StateApps - -from sentry.testutils.cases import TestMigrations - - -class GenericizeNightShiftResultsMigrationTest(TestMigrations): - migrate_from = "0008_add_seer_run_models" - migrate_to = "0009_genericize_night_shift_results" - app = "seer" - - def setup_initial_state(self) -> None: - self.group = self.create_group() - - def setup_before_migration(self, apps: StateApps) -> None: - SeerNightShiftRun = apps.get_model("seer", "SeerNightShiftRun") - SeerNightShiftRunIssue = apps.get_model("seer", "SeerNightShiftRunIssue") - - run = SeerNightShiftRun.objects.create( - organization_id=self.organization.id, - triage_strategy="agentic_triage", - ) - autofix_row = SeerNightShiftRunIssue.objects.create( - run_id=run.id, - group_id=self.group.id, - action="autofix", - seer_run_id="seer-1", - ) - root_cause_row = SeerNightShiftRunIssue.objects.create( - run_id=run.id, - group_id=self.group.id, - action="root_cause_only", - seer_run_id="seer-2", - ) - self.run_id = run.id - self.autofix_row_id = autofix_row.id - self.root_cause_row_id = root_cause_row.id - - # A second run that recorded a failure on the legacy error_message - # column, to verify the per-row error_message backfill into extras. - self.failed_error_message = "No Seer quota available" - failed_run = SeerNightShiftRun.objects.create( - organization_id=self.organization.id, - triage_strategy="agentic_triage", - error_message=self.failed_error_message, - ) - self.failed_run_id = failed_run.id - - def test(self) -> None: - from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunResult - - autofix_row = SeerNightShiftRunResult.objects.get(id=self.autofix_row_id) - assert autofix_row.kind == "agentic_triage" - assert autofix_row.extras == {"action": "autofix"} - assert autofix_row.seer_run_id == "seer-1" - - root_cause_row = SeerNightShiftRunResult.objects.get(id=self.root_cause_row_id) - assert root_cause_row.kind == "agentic_triage" - assert root_cause_row.extras == {"action": "root_cause_only"} - assert root_cause_row.seer_run_id == "seer-2" - - # Run with no error_message keeps an empty (or near-empty) extras. - ok_run = SeerNightShiftRun.objects.get(id=self.run_id) - assert "error_message" not in (ok_run.extras or {}) - - # Run with a recorded error has it preserved in extras. - failed_run = SeerNightShiftRun.objects.get(id=self.failed_run_id) - assert failed_run.extras["error_message"] == self.failed_error_message From cdeb03e6fd75b2b25676e5ca1e6ac2967fbba985 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 19:31:58 -0400 Subject: [PATCH 07/12] Revert "fix(seer): Drop slow TestMigrations integration test for 0009" This reverts commit f7cd5c3fbdab83716dfb616bbe6efeddf82b0d49. --- tests/sentry/seer/migrations/__init__.py | 0 ...est_0009_genericize_night_shift_results.py | 67 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 tests/sentry/seer/migrations/__init__.py create mode 100644 tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py diff --git a/tests/sentry/seer/migrations/__init__.py b/tests/sentry/seer/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py new file mode 100644 index 000000000000..de4ec73ee5a6 --- /dev/null +++ b/tests/sentry/seer/migrations/test_0009_genericize_night_shift_results.py @@ -0,0 +1,67 @@ +from django.db.migrations.state import StateApps + +from sentry.testutils.cases import TestMigrations + + +class GenericizeNightShiftResultsMigrationTest(TestMigrations): + migrate_from = "0008_add_seer_run_models" + migrate_to = "0009_genericize_night_shift_results" + app = "seer" + + def setup_initial_state(self) -> None: + self.group = self.create_group() + + def setup_before_migration(self, apps: StateApps) -> None: + SeerNightShiftRun = apps.get_model("seer", "SeerNightShiftRun") + SeerNightShiftRunIssue = apps.get_model("seer", "SeerNightShiftRunIssue") + + run = SeerNightShiftRun.objects.create( + organization_id=self.organization.id, + triage_strategy="agentic_triage", + ) + autofix_row = SeerNightShiftRunIssue.objects.create( + run_id=run.id, + group_id=self.group.id, + action="autofix", + seer_run_id="seer-1", + ) + root_cause_row = SeerNightShiftRunIssue.objects.create( + run_id=run.id, + group_id=self.group.id, + action="root_cause_only", + seer_run_id="seer-2", + ) + self.run_id = run.id + self.autofix_row_id = autofix_row.id + self.root_cause_row_id = root_cause_row.id + + # A second run that recorded a failure on the legacy error_message + # column, to verify the per-row error_message backfill into extras. + self.failed_error_message = "No Seer quota available" + failed_run = SeerNightShiftRun.objects.create( + organization_id=self.organization.id, + triage_strategy="agentic_triage", + error_message=self.failed_error_message, + ) + self.failed_run_id = failed_run.id + + def test(self) -> None: + from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunResult + + autofix_row = SeerNightShiftRunResult.objects.get(id=self.autofix_row_id) + assert autofix_row.kind == "agentic_triage" + assert autofix_row.extras == {"action": "autofix"} + assert autofix_row.seer_run_id == "seer-1" + + root_cause_row = SeerNightShiftRunResult.objects.get(id=self.root_cause_row_id) + assert root_cause_row.kind == "agentic_triage" + assert root_cause_row.extras == {"action": "root_cause_only"} + assert root_cause_row.seer_run_id == "seer-2" + + # Run with no error_message keeps an empty (or near-empty) extras. + ok_run = SeerNightShiftRun.objects.get(id=self.run_id) + assert "error_message" not in (ok_run.extras or {}) + + # Run with a recorded error has it preserved in extras. + failed_run = SeerNightShiftRun.objects.get(id=self.failed_run_id) + assert failed_run.extras["error_message"] == self.failed_error_message From c8031318418adac494097acf0f70948332f32b31 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 21:27:20 -0400 Subject: [PATCH 08/12] fix(seer): Restore unrelated SeerNightShiftRun docstring change Drive-by edit from the original PR1 commit. No reason for it. --- src/sentry/seer/models/night_shift.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/models/night_shift.py b/src/sentry/seer/models/night_shift.py index 04111a82df30..b1304990e3aa 100644 --- a/src/sentry/seer/models/night_shift.py +++ b/src/sentry/seer/models/night_shift.py @@ -13,7 +13,10 @@ class NightShiftRunResultKind(models.TextChoices): @cell_silo_model class SeerNightShiftRun(DefaultFieldsModel): - """One row per night shift invocation per organization.""" + """ + Records each night shift invocation for an organization. + One row is created per org each time run_night_shift_for_org executes. + """ __relocation_scope__ = RelocationScope.Excluded From 275d6134fc225cb10096fcf1f9ef16eb91e529e6 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 22:06:09 -0400 Subject: [PATCH 09/12] ref(seer): Drop legacy night-shift columns and kind db_default Follow-up to the schema-genericization PR. The previous PR moved action, triage_strategy, and error_message to MOVE_TO_PENDING and left a db_default on the new kind column to keep mid-rolling-deploy INSERTs from old replicas valid. Now that the previous PR is fully deployed, this PR: - SafeRemoveField(DELETE) for action, triage_strategy, error_message to physically drop the columns from Postgres. - Drops the db_default on kind so callers must specify it explicitly (already the case in all live code). --- migrations_lockfile.txt | 2 +- .../migrations/0010_drop_legacy_columns.py | 50 +++++++++++++++++++ src/sentry/seer/models/night_shift.py | 4 +- 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 src/sentry/seer/migrations/0010_drop_legacy_columns.py diff --git a/migrations_lockfile.txt b/migrations_lockfile.txt index ce1757e1b7ca..2e95b43f1aa1 100644 --- a/migrations_lockfile.txt +++ b/migrations_lockfile.txt @@ -29,7 +29,7 @@ releases: 0004_cleanup_failed_safe_deletes replays: 0007_organizationmember_replay_access -seer: 0009_genericize_night_shift_results +seer: 0010_drop_legacy_columns sentry: 1080_backfill_deprecated_dashboard_widget_display_types diff --git a/src/sentry/seer/migrations/0010_drop_legacy_columns.py b/src/sentry/seer/migrations/0010_drop_legacy_columns.py new file mode 100644 index 000000000000..6d25006d8801 --- /dev/null +++ b/src/sentry/seer/migrations/0010_drop_legacy_columns.py @@ -0,0 +1,50 @@ +# Generated by Django 5.2.12 on 2026-05-05 02:03 + +from django.db import migrations, models + +from sentry.new_migrations.migrations import CheckedMigration +from sentry.new_migrations.monkey.fields import SafeRemoveField +from sentry.new_migrations.monkey.state import DeletionAction + + +class Migration(CheckedMigration): + # This flag is used to mark that a migration shouldn't be automatically run in production. + # This should only be used for operations where it's safe to run the migration after your + # code has deployed. So this should not be used for most operations that alter the schema + # of a table. + # Here are some things that make sense to mark as post deployment: + # - Large data migrations. Typically we want these to be run manually so that they can be + # monitored and not block the deploy for a long period of time while they run. + # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to + # run this outside deployments so that we don't block them. Note that while adding an index + # is a schema change, it's completely safe to run the operation after the code has deployed. + # Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment + + is_post_deployment = False + + dependencies = [ + ("seer", "0009_genericize_night_shift_results"), + ] + + operations = [ + SafeRemoveField( + model_name="seernightshiftrunresult", + name="action", + deletion_action=DeletionAction.DELETE, + ), + SafeRemoveField( + model_name="seernightshiftrun", + name="triage_strategy", + deletion_action=DeletionAction.DELETE, + ), + SafeRemoveField( + model_name="seernightshiftrun", + name="error_message", + deletion_action=DeletionAction.DELETE, + ), + migrations.AlterField( + model_name="seernightshiftrunresult", + name="kind", + field=models.CharField(max_length=256), + ), + ] diff --git a/src/sentry/seer/models/night_shift.py b/src/sentry/seer/models/night_shift.py index b1304990e3aa..dd4c8830651c 100644 --- a/src/sentry/seer/models/night_shift.py +++ b/src/sentry/seer/models/night_shift.py @@ -43,9 +43,7 @@ class SeerNightShiftRunResult(DefaultFieldsModel): run = FlexibleForeignKey( "seer.SeerNightShiftRun", on_delete=models.CASCADE, related_name="results" ) - kind = models.CharField( - max_length=256, db_default="agentic_triage", choices=NightShiftRunResultKind.choices - ) + kind = models.CharField(max_length=256, choices=NightShiftRunResultKind.choices) group = FlexibleForeignKey( "sentry.Group", on_delete=models.CASCADE, db_constraint=False, null=True ) From 8359a7cb2f6c53dc7404c661d098ac241ebd8ef3 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 18:32:50 -0400 Subject: [PATCH 10/12] feat(seer): Add nightly user-feedback summary as a second night shift kind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layers a new "feedback_summary" kind on top of the genericized result table. Each org's nightly run now creates one parent row that fans out into one execution branch per enabled kind, with per-kind state tracked under run.extras["kinds"][]. - New agentic_feedback_summary_strategy (mirrors agentic_triage's shape): SeerAgentClient run with custom feedback list/details tools, pydantic FeedbackSummaryArtifact schema, per-org IDOR scoping, short- circuits when fewer than MIN_FEEDBACKS_TO_SUMMARIZE feedbacks in the last 24h. - cron.py threads `kinds` through schedule → run_for_org → run_execution. Per-kind feature gating: agentic_triage requires organizations:seer-night-shift, feedback_summary requires organizations:seer-night-shift-feedback-summary; both still require the universal gen-ai-features and seat-based-seer-enabled flags. - _update_kind_state uses select_for_update to avoid clobbering between concurrent kind branches sharing the same parent row. - Admin trigger endpoint accepts an optional `kinds` body param. - NightShiftRunResultKind enum gains FEEDBACK_SUMMARY. - Serializer surfaces the per-kind state under a new `kinds` key and picks the most-recent triage error for the legacy errorMessage alias. --- .../models/seer_night_shift_run.py | 8 +- src/sentry/features/temporary.py | 2 + .../endpoints/admin_night_shift_trigger.py | 22 +- src/sentry/seer/models/night_shift.py | 1 + src/sentry/tasks/seer/night_shift/cron.py | 279 +++++++++++++----- .../seer/night_shift/feedback_summary.py | 203 +++++++++++++ .../night_shift/feedback_summary_tools.py | 180 +++++++++++ .../test_admin_night_shift_trigger.py | 38 +++ .../seer/night_shift/test_feedback_summary.py | 257 ++++++++++++++++ tests/sentry/tasks/seer/test_night_shift.py | 170 +++++++++-- 10 files changed, 1061 insertions(+), 99 deletions(-) create mode 100644 src/sentry/tasks/seer/night_shift/feedback_summary.py create mode 100644 src/sentry/tasks/seer/night_shift/feedback_summary_tools.py create mode 100644 tests/sentry/tasks/seer/night_shift/test_feedback_summary.py diff --git a/src/sentry/api/serializers/models/seer_night_shift_run.py b/src/sentry/api/serializers/models/seer_night_shift_run.py index 8ae3ca2e2014..34af8510ae8b 100644 --- a/src/sentry/api/serializers/models/seer_night_shift_run.py +++ b/src/sentry/api/serializers/models/seer_night_shift_run.py @@ -36,6 +36,7 @@ class SeerNightShiftRunResponse(TypedDict): dateAdded: str extras: dict[str, Any] errorMessage: str | None + kinds: dict[str, Any] results: list[SeerNightShiftRunResultResponse] issues: list[SeerNightShiftRunIssueResponse] triageStrategy: str @@ -61,12 +62,15 @@ def serialize( r for r in all_results if r.kind == NightShiftRunResultKind.AGENTIC_TRIAGE ] extras = obj.extras or {} + kinds_state = extras.get("kinds") or {} + triage_state = kinds_state.get(NightShiftRunResultKind.AGENTIC_TRIAGE.value) or {} return { "id": str(obj.id), "dateAdded": obj.date_added.isoformat(), "extras": extras, - # Legacy alias: error_message lives in extras now. - "errorMessage": extras.get("error_message"), + "kinds": kinds_state, + # Legacy alias: prefer per-kind triage error, fall back to flat extras. + "errorMessage": triage_state.get("error_message") or extras.get("error_message"), "results": [_serialize_result(r) for r in all_results], "issues": [_serialize_legacy_issue(r) for r in triage_results], # Match the pre-migration column behavior: always "agentic_triage" diff --git a/src/sentry/features/temporary.py b/src/sentry/features/temporary.py index 83c6cedf3fe8..3af990208fe2 100644 --- a/src/sentry/features/temporary.py +++ b/src/sentry/features/temporary.py @@ -305,6 +305,8 @@ def register_temporary_features(manager: FeatureManager) -> None: manager.add("organizations:seer-explorer-index", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enable Seer Night Shift nightly autofix cron manager.add("organizations:seer-night-shift", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) + # Enable Seer Night Shift nightly user-feedback summarization + manager.add("organizations:seer-night-shift-feedback-summary", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Display nightshift settings manager.add("organizations:seer-night-shift-settings", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enable context engine for Seer Agent diff --git a/src/sentry/seer/endpoints/admin_night_shift_trigger.py b/src/sentry/seer/endpoints/admin_night_shift_trigger.py index 7a8a461bd2b3..c537213d1953 100644 --- a/src/sentry/seer/endpoints/admin_night_shift_trigger.py +++ b/src/sentry/seer/endpoints/admin_night_shift_trigger.py @@ -6,6 +6,7 @@ from sentry.api.base import Endpoint, internal_cell_silo_endpoint from sentry.api.permissions import StaffPermission from sentry.tasks.seer.night_shift.cron import ( + ALL_NIGHT_SHIFT_KINDS, SeerNightShiftRunOptionsPartial, run_night_shift_for_org, schedule_night_shift, @@ -44,6 +45,20 @@ def post(self, request: Request) -> Response: if max_candidates < 1: return Response({"detail": "max_candidates must be >= 1"}, status=400) + kinds_raw = request.data.get("kinds") + kinds: list[str] + if kinds_raw is None: + kinds = ["agentic_triage"] + elif not isinstance(kinds_raw, list) or not all(isinstance(k, str) for k in kinds_raw): + return Response({"detail": "kinds must be a list of strings"}, status=400) + elif not kinds_raw: + return Response({"detail": "kinds must not be empty"}, status=400) + else: + unknown = [k for k in kinds_raw if k not in ALL_NIGHT_SHIFT_KINDS] + if unknown: + return Response({"detail": f"unknown kinds: {sorted(set(unknown))}"}, status=400) + kinds = list(kinds_raw) + options: SeerNightShiftRunOptionsPartial = {"source": "manual", "dry_run": dry_run} if max_candidates is not None: options["max_candidates"] = max_candidates @@ -53,13 +68,18 @@ def post(self, request: Request) -> Response: else: run_night_shift_for_org.apply_async( args=[organization_id], - kwargs={"options": options, "execute_in_task": True}, + kwargs={ + "kinds": kinds, + "options": options, + "execute_in_task": True, + }, ) return Response( { "success": True, "organization_id": organization_id, + "kinds": kinds, "dry_run": dry_run, "max_candidates": max_candidates, } diff --git a/src/sentry/seer/models/night_shift.py b/src/sentry/seer/models/night_shift.py index dd4c8830651c..23307a4b8823 100644 --- a/src/sentry/seer/models/night_shift.py +++ b/src/sentry/seer/models/night_shift.py @@ -9,6 +9,7 @@ class NightShiftRunResultKind(models.TextChoices): AGENTIC_TRIAGE = "agentic_triage" + FEEDBACK_SUMMARY = "feedback_summary" @cell_silo_model diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index dbe3bba02cc7..4260e8d08594 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -8,6 +8,7 @@ from typing import Any, Literal, TypedDict import sentry_sdk +from django.db import transaction from sentry import features, options, quotas from sentry.constants import ( @@ -32,6 +33,7 @@ from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.base import instrumented_task from sentry.tasks.seer.night_shift.agentic_triage import agentic_triage_strategy +from sentry.tasks.seer.night_shift.feedback_summary import agentic_feedback_summary_strategy from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult from sentry.tasks.seer.night_shift.tweaks import ( DEFAULT_EXTRA_TRIAGE_INSTRUCTIONS, @@ -52,11 +54,21 @@ NIGHT_SHIFT_DISPATCH_STEP_SECONDS = 37 NIGHT_SHIFT_SPREAD_DURATION = timedelta(hours=4) -BATCH_FEATURE_NAMES = [ - "organizations:seer-night-shift", +NightShiftKind = Literal["agentic_triage", "feedback_summary"] +ALL_NIGHT_SHIFT_KINDS: tuple[NightShiftKind, ...] = ("agentic_triage", "feedback_summary") + +# Each kind requires its own organization-level feature flag, so kinds can be +# rolled out independently. +KIND_FEATURE_NAMES: dict[NightShiftKind, str] = { + "agentic_triage": "organizations:seer-night-shift", + "feedback_summary": "organizations:seer-night-shift-feedback-summary", +} + +# Universal flags required for any kind. +UNIVERSAL_BATCH_FEATURE_NAMES = [ "organizations:gen-ai-features", ] -PER_ORG_FEATURE_NAMES = [ +UNIVERSAL_PER_ORG_FEATURE_NAMES = [ # INTERNAL handlers aren't routed through batch_has_for_organizations, # so this gets checked per-org on the survivors of the batch loop. "organizations:seat-based-seer-enabled", @@ -102,13 +114,8 @@ def schedule_night_shift( ) -> None: """ Nightly scheduler: collects org ids that have a Seer-connected repo, then - dispatches per-org worker tasks in batches with jitter. Feature flags - still gate the dispatch — SeerProjectRepository rows can outlive a paid - Seer subscription. - - The real cron caller passes nothing (defaults). Manual admin triggers - forward `run_options` so every per-org task inherits the same overrides - (source="manual", dry_run, max_candidates, etc.). + dispatches per-org worker tasks in batches with jitter. Each org is + dispatched once with the list of kinds its feature flags enable. """ if not options.get("seer.night_shift.enable"): return @@ -135,7 +142,7 @@ def schedule_night_shift( spread_seconds = int(NIGHT_SHIFT_SPREAD_DURATION.total_seconds()) batch_index = 0 - task_kwargs: dict[str, Any] = {"options": dict(run_options)} if run_options else {} + base_kwargs: dict[str, Any] = {"options": dict(run_options)} if run_options else {} for chunk_index, org_id_chunk in enumerate(chunked(seer_org_ids, 100)): org_batch = list( @@ -144,9 +151,12 @@ def schedule_night_shift( status=OrganizationStatus.ACTIVE, ) ) - eligible = _get_eligible_orgs_from_batch(org_batch) - for org in eligible: + kinds_by_org = _get_eligible_kinds_by_org(org_batch) + for org, kinds in kinds_by_org.items(): + if not kinds: + continue delay = (batch_index * NIGHT_SHIFT_DISPATCH_STEP_SECONDS) % spread_seconds + task_kwargs = {**base_kwargs, "kinds": list(kinds)} run_night_shift_for_org.apply_async(args=[org.id], kwargs=task_kwargs, countdown=delay) batch_index += 1 @@ -179,34 +189,35 @@ def schedule_night_shift( def run_night_shift_for_org( organization_id: int, *, + kinds: Sequence[str] | None = None, options: SeerNightShiftRunOptionsPartial | None = None, project_ids: list[int] | None = None, triggering_user_id: int | None = None, execute_in_task: bool = False, **kwargs: Any, ) -> int | None: - """Run night shift for one organization. `options` is a partial dict — - any missing fields are filled in by build_run_options. Cron dispatches - with no options (all defaults); manual triggers (project settings "Run - Now", admin endpoint) pass `{"source": "manual", ...}` and may scope the - run to specific projects. - - When execute_in_task is True, the heavy execution phase (quota check, - eligibility, triage, autofix) is dispatched to a separate task so the - caller doesn't block on it. The run record is always created synchronously - so callers have a stable handle to the run.""" + """Run night shift for one organization. `kinds` controls which units of + work execute under this run; defaults to agentic_triage for backward + compatibility with manual callers that haven't been updated.""" organization = Organization.objects.filter( id=organization_id, status=OrganizationStatus.ACTIVE ).first() if organization is None: return None + resolved_kinds = _validated_kinds(kinds) + if not resolved_kinds: + return None + resolved_options = build_run_options(options) sentry_sdk.set_tags( {"organization_id": organization.id, "organization_slug": organization.slug} ) - extras: dict[str, object] = {"options": dict(resolved_options)} + extras: dict[str, object] = { + "options": dict(resolved_options), + "kinds": {k: {"status": "pending"} for k in resolved_kinds}, + } if project_ids is not None: extras["target_project_ids"] = project_ids if triggering_user_id is not None: @@ -217,14 +228,16 @@ def run_night_shift_for_org( extras=extras, ) - task_kwargs: dict[str, Any] = {"options": dict(resolved_options)} + base_kwargs: dict[str, Any] = {"options": dict(resolved_options)} if project_ids is not None: - task_kwargs["project_ids"] = project_ids - - if execute_in_task: - run_night_shift_execution.apply_async(args=[run.id], kwargs=task_kwargs) - else: - run_night_shift_execution(run.id, **task_kwargs) + base_kwargs["project_ids"] = project_ids + + for kind in resolved_kinds: + per_kind_kwargs = {**base_kwargs, "kind": kind} + if execute_in_task: + run_night_shift_execution.apply_async(args=[run.id], kwargs=per_kind_kwargs) + else: + run_night_shift_execution(run.id, **per_kind_kwargs) return run.id @@ -236,13 +249,17 @@ def run_night_shift_for_org( def run_night_shift_execution( run_id: int, *, + kind: str = "agentic_triage", options: SeerNightShiftRunOptionsPartial | None = None, project_ids: list[int] | None = None, **kwargs: Any, ) -> None: - """Heavy phase of a night shift run: quota check, eligibility, triage, and - optional autofix dispatch. Single code path used by both sync invocation - (from run_night_shift_for_org) and async dispatch (apply_async).""" + """Heavy phase of a night shift run for a single kind: quota check, then + dispatch to the kind-specific strategy.""" + if kind not in ALL_NIGHT_SHIFT_KINDS: + logger.error("night_shift.unknown_kind", extra={"run_id": run_id, "kind": kind}) + return None + run = SeerNightShiftRun.objects.select_related("organization").filter(id=run_id).first() if run is None: logger.info("night_shift.missing_run", extra={"run_id": run_id}) @@ -255,6 +272,7 @@ def run_night_shift_execution( "organization_id": organization.id, "organization_slug": organization.slug, "run_id": run.id, + "kind": kind, } if project_ids is not None: log_extra["project_ids"] = project_ids @@ -265,32 +283,65 @@ def run_night_shift_execution( start_time = time.monotonic() logger.info("night_shift.execute.start", extra=log_extra) + _update_kind_state(run.id, kind, {"status": "running"}) + if not quotas.backend.check_seer_quota( org_id=organization.id, data_category=DataCategory.SEER_AUTOFIX, ): logger.info("night_shift.no_seer_quota", extra=log_extra) - _record_run_error(run, "No Seer quota available") + _update_kind_state(run.id, kind, {"status": "skipped", "reason": "no_seer_quota"}) return None + if kind == "agentic_triage": + _execute_triage( + run=run, + organization=organization, + resolved_options=resolved_options, + project_ids=project_ids, + log_extra=log_extra, + start_time=start_time, + ) + elif kind == "feedback_summary": + _execute_feedback_summary( + run=run, + organization=organization, + resolved_options=resolved_options, + log_extra=log_extra, + ) + + +def _execute_triage( + *, + run: SeerNightShiftRun, + organization: Organization, + resolved_options: SeerNightShiftRunOptions, + project_ids: list[int] | None, + log_extra: dict[str, object], + start_time: float, +) -> None: try: eligible = _get_eligible_projects( organization, resolved_options["source"], project_ids=project_ids ) except Exception: - _fail_run( + _fail_kind( run, + kind="agentic_triage", message="Failed to get eligible projects", event="night_shift.failed_to_get_eligible_projects", extra=log_extra, ) - return None + return sentry_sdk.metrics.distribution("night_shift.eligible_projects", len(eligible)) if not eligible: logger.info("night_shift.no_eligible_projects", extra=log_extra) - return None + _update_kind_state( + run.id, "agentic_triage", {"status": "skipped", "reason": "no_eligible_projects"} + ) + return eligible_projects = [ep.project for ep in eligible] agent_run_id: int | None = None @@ -308,13 +359,14 @@ def run_night_shift_execution( log_extra["agent_run_id"] = agent_run_id except Exception: sentry_sdk.metrics.count("night_shift.run_error", 1) - _fail_run( + _fail_kind( run, + kind="agentic_triage", message="Night shift run failed", event="night_shift.run_failed", extra={**log_extra, "agent_run_id": agent_run_id}, ) - return None + return sentry_sdk.metrics.distribution("night_shift.candidates_selected", len(candidates)) sentry_sdk.metrics.distribution("night_shift.org_run_duration", time.monotonic() - start_time) @@ -342,6 +394,16 @@ def run_night_shift_execution( ) seer_run_id_by_group = {r.group_id: r.seer_run_id for r in results} + _update_kind_state( + run.id, + "agentic_triage", + { + "status": "succeeded", + "agent_run_id": agent_run_id, + "num_candidates": len(candidates), + }, + ) + logger.info( "night_shift.candidates_selected", extra={ @@ -362,6 +424,48 @@ def run_night_shift_execution( ) +def _execute_feedback_summary( + *, + run: SeerNightShiftRun, + organization: Organization, + resolved_options: SeerNightShiftRunOptions, + log_extra: dict[str, object], +) -> None: + agent_run_id: int | None = None + try: + agent_run_id = agentic_feedback_summary_strategy( + organization, + run=run, + intelligence_level=resolved_options["intelligence_level"], + reasoning_effort=resolved_options["reasoning_effort"], + ) + except Exception: + sentry_sdk.metrics.count("night_shift.feedback_summary_run_error", 1) + _fail_kind( + run, + kind="feedback_summary", + message="Feedback summary run failed", + event="night_shift.feedback_summary.run_failed", + extra={**log_extra, "agent_run_id": agent_run_id}, + ) + return + + if agent_run_id is None: + # Strategy short-circuited (insufficient feedback). + _update_kind_state( + run.id, + "feedback_summary", + {"status": "skipped", "reason": "insufficient_feedbacks"}, + ) + return + + _update_kind_state( + run.id, + "feedback_summary", + {"status": "succeeded", "agent_run_id": agent_run_id}, + ) + + def _run_option_defaults(data: Mapping[str, Any]) -> SeerNightShiftRunOptions: """Fill in defaults for any missing fields. Accepts any mapping so it can normalize both partial caller input and loosely-typed dicts read back from @@ -388,47 +492,82 @@ def build_run_options( return _run_option_defaults(partial or {}) -def _get_eligible_orgs_from_batch( +def _validated_kinds(kinds: Sequence[str] | None) -> list[NightShiftKind]: + if kinds is None: + return ["agentic_triage"] + deduped: list[NightShiftKind] = [] + seen: set[str] = set() + for k in kinds: + if k in seen: + continue + if k not in ALL_NIGHT_SHIFT_KINDS: + logger.error("night_shift.unknown_kind_requested", extra={"kind": k}) + continue + seen.add(k) + deduped.append(k) # type: ignore[arg-type] + return deduped + + +def _get_eligible_kinds_by_org( orgs: Sequence[Organization], -) -> list[Organization]: - """ - Check feature flags for a batch of orgs. - Returns orgs that have all required feature flags enabled. - """ - eligible = [org for org in orgs if not org.get_option("sentry:hide_ai_features")] +) -> dict[Organization, list[NightShiftKind]]: + """Return org → list of kinds enabled for that org. Orgs that pass the + universal gates but have no kinds enabled are present with an empty list.""" + universal_eligible = [org for org in orgs if not org.get_option("sentry:hide_ai_features")] - for feature_name in BATCH_FEATURE_NAMES: - batch_result = features.batch_has_for_organizations(feature_name, eligible) + for feature_name in UNIVERSAL_BATCH_FEATURE_NAMES: + batch_result = features.batch_has_for_organizations(feature_name, universal_eligible) if batch_result is None: raise RuntimeError(f"batch_has_for_organizations returned None for {feature_name}") - - eligible = [org for org in eligible if batch_result.get(f"organization:{org.id}", False)] - - if not eligible: - return [] - - for feature_name in PER_ORG_FEATURE_NAMES: - eligible = [org for org in eligible if features.has(feature_name, org)] - if not eligible: - return [] - - return eligible - - -def _record_run_error(run: SeerNightShiftRun, message: str) -> None: - run.update(extras={**(run.extras or {}), "error_message": message}) - - -def _fail_run( + universal_eligible = [ + org for org in universal_eligible if batch_result.get(f"organization:{org.id}", False) + ] + if not universal_eligible: + return {} + + for feature_name in UNIVERSAL_PER_ORG_FEATURE_NAMES: + universal_eligible = [org for org in universal_eligible if features.has(feature_name, org)] + if not universal_eligible: + return {} + + kinds_by_org: dict[Organization, list[NightShiftKind]] = {org: [] for org in universal_eligible} + for kind, feature_name in KIND_FEATURE_NAMES.items(): + batch_result = features.batch_has_for_organizations(feature_name, universal_eligible) + if batch_result is None: + raise RuntimeError(f"batch_has_for_organizations returned None for {feature_name}") + for org in universal_eligible: + if batch_result.get(f"organization:{org.id}", False): + kinds_by_org[org].append(kind) + + return kinds_by_org + + +def _update_kind_state(run_id: int, kind: str, patch: Mapping[str, Any]) -> None: + """Atomically merge `patch` into run.extras["kinds"][kind]. The kind + branches run concurrently against the same parent row, so we use + select_for_update to avoid clobbering each other's sub-dicts.""" + with transaction.atomic(using="default"): + run = SeerNightShiftRun.objects.select_for_update().get(id=run_id) + extras = dict(run.extras or {}) + kinds_state = dict(extras.get("kinds") or {}) + existing = dict(kinds_state.get(kind) or {}) + existing.update(patch) + kinds_state[kind] = existing + extras["kinds"] = kinds_state + run.extras = extras + run.save(update_fields=["extras"]) + + +def _fail_kind( run: SeerNightShiftRun, *, + kind: str, message: str, event: str, extra: dict[str, object], ) -> None: - """Log an exception and record an error message on the run.""" logger.exception(event, extra=extra) - _record_run_error(run, message) + _update_kind_state(run.id, kind, {"status": "failed", "error_message": message}) @dataclasses.dataclass(frozen=True) diff --git a/src/sentry/tasks/seer/night_shift/feedback_summary.py b/src/sentry/tasks/seer/night_shift/feedback_summary.py new file mode 100644 index 000000000000..326a77c5a629 --- /dev/null +++ b/src/sentry/tasks/seer/night_shift/feedback_summary.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import logging +import textwrap +from datetime import timedelta + +import pydantic +import sentry_sdk +from django.utils import timezone + +from sentry.constants import ObjectStatus +from sentry.issues.grouptype import FeedbackGroup +from sentry.models.group import Group, GroupStatus +from sentry.models.organization import Organization +from sentry.seer.agent.client import SeerAgentClient +from sentry.seer.models.night_shift import ( + NightShiftRunResultKind, + SeerNightShiftRun, + SeerNightShiftRunResult, +) +from sentry.tasks.seer.night_shift.agentic_triage import _poll_with_logging +from sentry.tasks.seer.night_shift.feedback_summary_tools import ( + get_feedback_details_summary_tool, + get_feedback_list_summary_tool, +) +from sentry.tasks.seer.night_shift.tweaks import ( + DEFAULT_INTELLIGENCE_LEVEL, + DEFAULT_REASONING_EFFORT, + IntelligenceLevel, + ReasoningEffort, +) + +logger = logging.getLogger("sentry.tasks.seer.night_shift") + +MIN_FEEDBACKS_TO_SUMMARIZE = 10 +SUMMARY_LOOKBACK = timedelta(days=1) + + +class _FeedbackTheme(pydantic.BaseModel): + title: str + description: str + feedback_group_ids: list[int] = pydantic.Field(default_factory=list) + + +class _FeedbackSummaryArtifact(pydantic.BaseModel): + summary: str + themes: list[_FeedbackTheme] = pydantic.Field(default_factory=list) + num_feedbacks_analyzed: int + + +def agentic_feedback_summary_strategy( + organization: Organization, + *, + run: SeerNightShiftRun, + intelligence_level: IntelligenceLevel = DEFAULT_INTELLIGENCE_LEVEL, + reasoning_effort: ReasoningEffort = DEFAULT_REASONING_EFFORT, +) -> int | None: + """Summarize the org's user feedback from the last day via the Seer agent. + + Writes a single SeerNightShiftRunResult row (kind="feedback_summary") + containing the summary, themes, and counts. Returns the agent_run_id, or + None if the run was skipped (insufficient feedback) or errored. + """ + feedback_group_ids = _recent_feedback_group_ids(organization) + if len(feedback_group_ids) < MIN_FEEDBACKS_TO_SUMMARIZE: + logger.info( + "night_shift.feedback_summary.skipped_insufficient", + extra={ + "organization_id": organization.id, + "num_feedbacks": len(feedback_group_ids), + "min_required": MIN_FEEDBACKS_TO_SUMMARIZE, + }, + ) + return None + + try: + client = SeerAgentClient( + organization, + user=None, + category_key="night_shift_feedback_summary", + category_value=f"org-{organization.id}", + intelligence_level=intelligence_level, + reasoning_effort=reasoning_effort, + custom_tools=[ + get_feedback_list_summary_tool, + get_feedback_details_summary_tool, + ], + ) + + agent_run_id = client.start_run( + prompt=_build_summary_prompt(len(feedback_group_ids)), + artifact_key="feedback_summary", + artifact_schema=_FeedbackSummaryArtifact, + ) + + logger.info( + "night_shift.feedback_summary.run_started", + extra={ + "organization_id": organization.id, + "agent_run_id": agent_run_id, + "num_feedback_group_ids": len(feedback_group_ids), + }, + ) + + state = _poll_with_logging(client, agent_run_id, organization.id) + + artifact = state.get_artifact("feedback_summary", _FeedbackSummaryArtifact) + if artifact is None: + logger.error( + "night_shift.feedback_summary.no_artifact", + extra={ + "organization_id": organization.id, + "agent_run_id": agent_run_id, + "status": state.status, + }, + ) + sentry_sdk.metrics.count( + "night_shift.feedback_summary_error", + 1, + attributes={"error_type": "no_artifact"}, + ) + return agent_run_id + except Exception: + sentry_sdk.metrics.count( + "night_shift.feedback_summary_error", + 1, + attributes={"error_type": "explorer_error"}, + ) + logger.exception( + "night_shift.feedback_summary.explorer_error", + extra={"organization_id": organization.id}, + ) + raise + + SeerNightShiftRunResult.objects.create( + run=run, + kind=NightShiftRunResultKind.FEEDBACK_SUMMARY, + group=None, + seer_run_id=str(agent_run_id), + extras={ + **artifact.dict(), + "feedback_group_ids_sampled": feedback_group_ids, + "agent_run_id": agent_run_id, + }, + ) + + sentry_sdk.metrics.count( + "night_shift.feedback_summary_recorded", + 1, + attributes={"num_themes": str(len(artifact.themes))}, + ) + return agent_run_id + + +def _recent_feedback_group_ids(organization: Organization) -> list[int]: + """Bound the candidate set the agent can pull from. The agent's tools also + re-filter on the same window so it never sees rows older than this.""" + project_ids = list( + organization.project_set.filter(status=ObjectStatus.ACTIVE).values_list("id", flat=True) + ) + if not project_ids: + return [] + cutoff = timezone.now() - SUMMARY_LOOKBACK + return list( + Group.objects.filter( + type=FeedbackGroup.type_id, + status=GroupStatus.UNRESOLVED, + project_id__in=project_ids, + first_seen__gte=cutoff, + ) + .order_by("-first_seen") + .values_list("id", flat=True) + ) + + +def _build_summary_prompt(num_feedbacks: int) -> str: + return textwrap.dedent(f"""\ + You are a feedback summarization agent for Sentry's Night Shift system. + This organization received {num_feedbacks} pieces of user feedback in the + last 24 hours. Your job is to read through them and produce a structured + summary an engineering team can act on. + + Use `get_feedback_list_summary_tool` to enumerate feedback. Call it once + with no project filter to see the spread, then optionally narrow by + project_ids if a single product surface dominates. Use + `get_feedback_details_summary_tool` only when you need the full message + body for a specific feedback that the list view truncated. + + Produce a `feedback_summary` artifact with: + - `summary`: 3-6 sentences describing the day's feedback at a glance. + Lead with what changed vs. the usual baseline (if you can tell), then + call out the most important themes by frequency or severity. + - `themes`: a list of distinct, named themes. For each theme include + a short title, a 1-2 sentence description, and the + `feedback_group_ids` that exemplify it. Aim for 3-8 themes; merge + near-duplicates aggressively. + - `num_feedbacks_analyzed`: count of feedback entries you actually + read while building this summary. + + Stay grounded in what the feedback actually says — do not speculate + about features users didn't mention. If the day's feedback is mostly + noise, say so concisely rather than padding with weak themes. + """) diff --git a/src/sentry/tasks/seer/night_shift/feedback_summary_tools.py b/src/sentry/tasks/seer/night_shift/feedback_summary_tools.py new file mode 100644 index 000000000000..fcfab7a1d6d0 --- /dev/null +++ b/src/sentry/tasks/seer/night_shift/feedback_summary_tools.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from datetime import timedelta + +from django.utils import timezone +from pydantic import BaseModel, Field + +from sentry.constants import ObjectStatus +from sentry.issues.grouptype import FeedbackGroup +from sentry.models.group import Group, GroupStatus +from sentry.models.organization import Organization +from sentry.models.project import Project +from sentry.seer.agent.custom_tool_utils import AgentTool + +# Hard cap on rows returned per call so the agent doesn't accidentally pull +# the full feedback corpus into a single tool response. +MAX_FEEDBACK_LIST_LIMIT = 200 +DEFAULT_FEEDBACK_LIST_LIMIT = 50 + +# How far back the agent is allowed to look. Aligned with the daily cron +# cadence — we summarize "the last day" of feedback, with a small grace window. +FEEDBACK_LOOKBACK = timedelta(days=2) + + +class GetFeedbackListSummaryParams(BaseModel): + project_ids: list[int] | None = Field( + default=None, + description=( + "Optional list of project IDs to filter by. When omitted, returns " + "feedback from every project in the organization." + ), + ) + limit: int = Field( + default=DEFAULT_FEEDBACK_LIST_LIMIT, + description=( + "Maximum number of feedback entries to return, ordered by most " + f"recent first_seen. Capped at {MAX_FEEDBACK_LIST_LIMIT}." + ), + ) + + +# Class name intentionally snake_case — see the comment on +# `get_event_details_agentic_triage` in triage_tools.py. +class get_feedback_list_summary_tool( # noqa: N801 + AgentTool[GetFeedbackListSummaryParams] +): + """Custom agent tool for Night Shift feedback summarization. + + Returns the most recent unresolved user feedback messages for the org + (last ~24 hours), optionally filtered to specific projects. Each row + includes the message body, project info, and first_seen so the agent + can group, theme, and summarize. + """ + + params_model = GetFeedbackListSummaryParams + + @classmethod + def get_description(cls) -> str: + return ( + "Fetch a batch of recent user feedback entries for this organization. " + "Returns markdown-formatted rows with feedback id, message body, " + "project, and timestamp. Use this to enumerate the day's feedback " + "and identify themes. Filter by project_ids when an org has many " + "projects and you only need a subset." + ) + + @classmethod + def execute( + cls, + organization: Organization, + params: GetFeedbackListSummaryParams, + ) -> str: + limit = max(1, min(params.limit, MAX_FEEDBACK_LIST_LIMIT)) + projects = _resolve_org_projects(organization, params.project_ids) + if not projects: + return "No projects matched the filter." + + groups = ( + Group.objects.filter( + type=FeedbackGroup.type_id, + status=GroupStatus.UNRESOLVED, + project__in=projects, + first_seen__gte=timezone.now() - FEEDBACK_LOOKBACK, + ) + .select_related("project") + .order_by("-first_seen")[:limit] + ) + + rendered = [_render_feedback_row(g) for g in groups] + if not rendered: + return "No feedback found in the lookback window." + return f"Found {len(rendered)} feedback entries:\n\n" + "\n\n".join(rendered) + + +class GetFeedbackDetailsSummaryParams(BaseModel): + feedback_group_id: int = Field( + description=( + "The Group ID of the feedback entry to fetch. Must match an id " + "returned by `get_feedback_list_summary_tool`." + ), + ) + + +# Class name intentionally snake_case — see the comment above. +class get_feedback_details_summary_tool( # noqa: N801 + AgentTool[GetFeedbackDetailsSummaryParams] +): + """Custom agent tool for Night Shift feedback summarization. + + Returns the full feedback message and metadata for a single feedback group. + Always scoped to the calling organization; cross-org IDs return an error + message rather than leaking data. + """ + + params_model = GetFeedbackDetailsSummaryParams + + @classmethod + def get_description(cls) -> str: + return ( + "Fetch the full message body and metadata for a single feedback " + "entry. Returns a markdown block with the message, project, " + "timestamps, and any contact info attached to the feedback." + ) + + @classmethod + def execute( + cls, + organization: Organization, + params: GetFeedbackDetailsSummaryParams, + ) -> str: + group = ( + Group.objects.filter( + id=params.feedback_group_id, + project__organization_id=organization.id, + type=FeedbackGroup.type_id, + ) + .select_related("project") + .first() + ) + if group is None: + return ( + "Feedback not found. Check the feedback_group_id and confirm it " + "belongs to this organization." + ) + return _render_feedback_row(group, include_full_metadata=True) + + +def _resolve_org_projects( + organization: Organization, project_ids: list[int] | None +) -> list[Project]: + qs = Project.objects.filter(organization_id=organization.id, status=ObjectStatus.ACTIVE) + if project_ids is not None: + qs = qs.filter(id__in=project_ids) + return list(qs) + + +def _render_feedback_row(group: Group, *, include_full_metadata: bool = False) -> str: + metadata = (group.data or {}).get("metadata") or {} + message = metadata.get("message") or "(empty)" + lines = [ + f"- feedback_group_id={group.id}", + f" project={group.project.slug} ({group.project.id})", + f" first_seen={group.first_seen.isoformat()}", + f" times_seen={group.times_seen}", + ] + if include_full_metadata: + contact = metadata.get("contact_email") or metadata.get("name") + if contact: + lines.append(f" contact={contact!r}") + lines.append(" message:") + lines.append(" ```") + lines.append(_indent_block(message, " ")) + lines.append(" ```") + else: + lines.append(f" message={message[:500]!r}") + return "\n".join(lines) + + +def _indent_block(text: str, prefix: str) -> str: + return "\n".join(f"{prefix}{line}" for line in text.splitlines() or [""]) diff --git a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py index 1b714c70ab68..03db43f6aac2 100644 --- a/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py +++ b/tests/sentry/seer/endpoints/test_admin_night_shift_trigger.py @@ -29,9 +29,11 @@ def test_trigger_night_shift(self) -> None: assert response.data["success"] is True assert response.data["organization_id"] == self.organization.id assert response.data["max_candidates"] is None + assert response.data["kinds"] == ["agentic_triage"] mock_task.apply_async.assert_called_once_with( args=[self.organization.id], kwargs={ + "kinds": ["agentic_triage"], "options": {"source": "manual", "dry_run": False}, "execute_in_task": True, }, @@ -52,6 +54,7 @@ def test_trigger_with_max_candidates_override(self) -> None: mock_task.apply_async.assert_called_once_with( args=[self.organization.id], kwargs={ + "kinds": ["agentic_triage"], "options": {"source": "manual", "dry_run": True, "max_candidates": 3}, "execute_in_task": True, }, @@ -123,3 +126,38 @@ def test_requires_staff(self) -> None: self.login_as(user=non_staff_user) response = super().get_response(organization_id=self.organization.id) assert response.status_code == 403 + + def test_accepts_explicit_kinds(self) -> None: + with patch( + "sentry.seer.endpoints.admin_night_shift_trigger.run_night_shift_for_org" + ) as mock_task: + response = self.get_success_response( + organization_id=self.organization.id, + kinds=["feedback_summary"], + status_code=200, + ) + + assert response.data["kinds"] == ["feedback_summary"] + mock_task.apply_async.assert_called_once_with( + args=[self.organization.id], + kwargs={ + "kinds": ["feedback_summary"], + "options": {"source": "manual", "dry_run": False}, + "execute_in_task": True, + }, + ) + + def test_rejects_unknown_kinds(self) -> None: + response = self.get_response(organization_id=self.organization.id, kinds=["bogus"]) + assert response.status_code == 400 + assert "unknown kinds" in response.data["detail"] + + def test_rejects_non_list_kinds(self) -> None: + response = self.get_response(organization_id=self.organization.id, kinds="agentic_triage") + assert response.status_code == 400 + assert response.data["detail"] == "kinds must be a list of strings" + + def test_rejects_empty_kinds(self) -> None: + response = self.get_response(organization_id=self.organization.id, kinds=[]) + assert response.status_code == 400 + assert response.data["detail"] == "kinds must not be empty" diff --git a/tests/sentry/tasks/seer/night_shift/test_feedback_summary.py b/tests/sentry/tasks/seer/night_shift/test_feedback_summary.py new file mode 100644 index 000000000000..7e7ea352b73e --- /dev/null +++ b/tests/sentry/tasks/seer/night_shift/test_feedback_summary.py @@ -0,0 +1,257 @@ +from datetime import timedelta +from unittest.mock import MagicMock, patch + +from django.utils import timezone + +from sentry.issues.grouptype import FeedbackGroup +from sentry.models.group import Group +from sentry.seer.agent.client_models import Artifact, MemoryBlock, Message, SeerRunState +from sentry.seer.models.night_shift import SeerNightShiftRun, SeerNightShiftRunResult +from sentry.tasks.seer.night_shift.cron import run_night_shift_execution +from sentry.tasks.seer.night_shift.feedback_summary import ( + MIN_FEEDBACKS_TO_SUMMARIZE, + agentic_feedback_summary_strategy, +) +from sentry.tasks.seer.night_shift.feedback_summary_tools import ( + GetFeedbackDetailsSummaryParams, + GetFeedbackListSummaryParams, + get_feedback_details_summary_tool, + get_feedback_list_summary_tool, +) +from sentry.testutils.cases import TestCase +from sentry.testutils.pytest.fixtures import django_db_all + + +class FakeFeedbackSummaryClient: + """Stub SeerAgentClient that returns a canned feedback summary artifact.""" + + def __init__(self, summary: dict, run_id: int = 77): + artifact = Artifact(key="feedback_summary", data=summary, reason="test") + self.run_id = run_id + self._state = SeerRunState( + run_id=run_id, + blocks=[ + MemoryBlock( + id="test-block", + message=Message(role="assistant"), + timestamp="2025-01-01T00:00:00", + artifacts=[artifact], + ), + ], + status="completed", + updated_at="2025-01-01T00:00:00", + ) + + def start_run(self, **kwargs): + return self.run_id + + def get_run(self, run_id, **kwargs): + return self._state + + +def _make_feedback(test_case, project, message: str) -> Group: + return test_case.create_group( + project=project, + type=FeedbackGroup.type_id, + data={"type": "feedback", "metadata": {"message": message}}, + ) + + +def _seed_feedbacks(test_case, project, count: int) -> list[Group]: + return [_make_feedback(test_case, project, f"Feedback message #{i}") for i in range(count)] + + +@django_db_all +class TestAgenticFeedbackSummaryStrategy(TestCase): + def test_writes_result_row_when_artifact_present(self) -> None: + org = self.create_organization() + project = self.create_project(organization=org) + _seed_feedbacks(self, project, MIN_FEEDBACKS_TO_SUMMARIZE) + + run = SeerNightShiftRun.objects.create( + organization=org, extras={"kinds": {"feedback_summary": {"status": "running"}}} + ) + + artifact_payload = { + "summary": "Most feedback is about login flakiness.", + "themes": [ + { + "title": "Login flakiness", + "description": "Users are randomly logged out.", + "feedback_group_ids": [], + } + ], + "num_feedbacks_analyzed": MIN_FEEDBACKS_TO_SUMMARIZE, + } + fake = FakeFeedbackSummaryClient(artifact_payload, run_id=42) + + with patch( + "sentry.tasks.seer.night_shift.feedback_summary.SeerAgentClient", + return_value=fake, + ): + agent_run_id = agentic_feedback_summary_strategy(org, run=run) + + assert agent_run_id == 42 + result = SeerNightShiftRunResult.objects.get(run=run, kind="feedback_summary") + assert result.group_id is None + assert result.seer_run_id == "42" + assert result.extras["summary"] == artifact_payload["summary"] + assert result.extras["agent_run_id"] == 42 + assert "feedback_group_ids_sampled" in result.extras + + def test_skipped_when_insufficient_feedbacks(self) -> None: + org = self.create_organization() + project = self.create_project(organization=org) + _seed_feedbacks(self, project, 3) + + run = SeerNightShiftRun.objects.create(organization=org, extras={"kinds": {}}) + + mock_client = MagicMock() + with patch( + "sentry.tasks.seer.night_shift.feedback_summary.SeerAgentClient", + return_value=mock_client, + ): + agent_run_id = agentic_feedback_summary_strategy(org, run=run) + + assert agent_run_id is None + mock_client.start_run.assert_not_called() + assert not SeerNightShiftRunResult.objects.filter(run=run).exists() + + def test_no_artifact_returns_run_id_without_writing_row(self) -> None: + org = self.create_organization() + project = self.create_project(organization=org) + _seed_feedbacks(self, project, MIN_FEEDBACKS_TO_SUMMARIZE) + + run = SeerNightShiftRun.objects.create(organization=org, extras={"kinds": {}}) + + empty_state = SeerRunState( + run_id=99, + blocks=[ + MemoryBlock( + id="b", + message=Message(role="assistant"), + timestamp="2025-01-01T00:00:00", + artifacts=[], + ) + ], + status="completed", + updated_at="2025-01-01T00:00:00", + ) + + class _StubClient: + def start_run(self, **kwargs): + return 99 + + def get_run(self, run_id, **kwargs): + return empty_state + + with patch( + "sentry.tasks.seer.night_shift.feedback_summary.SeerAgentClient", + return_value=_StubClient(), + ): + agent_run_id = agentic_feedback_summary_strategy(org, run=run) + + assert agent_run_id == 99 + assert not SeerNightShiftRunResult.objects.filter(run=run).exists() + + +@django_db_all +class TestFeedbackSummaryTools(TestCase): + def test_list_returns_recent_feedbacks_for_org(self) -> None: + project = self.create_project() + _seed_feedbacks(self, project, 3) + + out = get_feedback_list_summary_tool.execute( + self.organization, GetFeedbackListSummaryParams() + ) + assert "Found 3 feedback entries" in out + assert f"project={project.slug}" in out + + def test_list_excludes_other_org_feedbacks(self) -> None: + own_project = self.create_project() + _seed_feedbacks(self, own_project, 2) + + other_org = self.create_organization() + other_project = self.create_project(organization=other_org) + _seed_feedbacks(self, other_project, 5) + + out = get_feedback_list_summary_tool.execute( + self.organization, GetFeedbackListSummaryParams() + ) + assert f"project={own_project.slug}" in out + assert f"project={other_project.slug}" not in out + + def test_list_skips_feedbacks_outside_lookback(self) -> None: + project = self.create_project() + groups = _seed_feedbacks(self, project, 3) + # Force first_seen well outside the lookback window. + Group.objects.filter(id__in=[g.id for g in groups]).update( + first_seen=timezone.now() - timedelta(days=30) + ) + + out = get_feedback_list_summary_tool.execute( + self.organization, GetFeedbackListSummaryParams() + ) + assert "No feedback found in the lookback window." in out + + def test_details_blocks_cross_org_lookup(self) -> None: + other_org = self.create_organization() + other_project = self.create_project(organization=other_org) + foreign = _seed_feedbacks(self, other_project, 1)[0] + + out = get_feedback_details_summary_tool.execute( + self.organization, + GetFeedbackDetailsSummaryParams(feedback_group_id=foreign.id), + ) + assert "Feedback not found." in out + + +@django_db_all +class TestRunNightShiftExecutionFeedbackBranch(TestCase): + def test_dispatches_feedback_summary_strategy(self) -> None: + org = self.create_organization() + run = SeerNightShiftRun.objects.create( + organization=org, + extras={"kinds": {"feedback_summary": {"status": "pending"}}}, + ) + + with ( + patch( + "sentry.tasks.seer.night_shift.cron.quotas.backend.check_seer_quota", + return_value=True, + ), + patch( + "sentry.tasks.seer.night_shift.cron.agentic_feedback_summary_strategy", + return_value=42, + ) as mock_strategy, + ): + run_night_shift_execution(run.id, kind="feedback_summary") + + mock_strategy.assert_called_once() + run.refresh_from_db() + assert run.extras["kinds"]["feedback_summary"]["status"] == "succeeded" + assert run.extras["kinds"]["feedback_summary"]["agent_run_id"] == 42 + + def test_skipped_status_when_strategy_returns_none(self) -> None: + org = self.create_organization() + run = SeerNightShiftRun.objects.create( + organization=org, + extras={"kinds": {"feedback_summary": {"status": "pending"}}}, + ) + + with ( + patch( + "sentry.tasks.seer.night_shift.cron.quotas.backend.check_seer_quota", + return_value=True, + ), + patch( + "sentry.tasks.seer.night_shift.cron.agentic_feedback_summary_strategy", + return_value=None, + ), + ): + run_night_shift_execution(run.id, kind="feedback_summary") + + run.refresh_from_db() + state = run.extras["kinds"]["feedback_summary"] + assert state["status"] == "skipped" + assert state["reason"] == "insufficient_feedbacks" diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index 2ddc6f464b77..1441a50de2c5 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -86,7 +86,9 @@ def test_dispatches_eligible_orgs(self) -> None: schedule_night_shift() mock_worker.apply_async.assert_called_once() assert mock_worker.apply_async.call_args.kwargs["args"] == [org.id] - assert mock_worker.apply_async.call_args.kwargs["kwargs"] == {} + assert mock_worker.apply_async.call_args.kwargs["kwargs"] == { + "kinds": ["agentic_triage"], + } def test_dispatches_with_run_options(self) -> None: org = self.create_org_with_seer() @@ -109,6 +111,7 @@ def test_dispatches_with_run_options(self) -> None: assert mock_worker.apply_async.call_args.kwargs["args"] == [org.id] assert mock_worker.apply_async.call_args.kwargs["kwargs"] == { "options": {"source": "manual", "dry_run": True, "max_candidates": 3}, + "kinds": ["agentic_triage"], } def test_skips_orgs_without_seat_based_seer(self) -> None: @@ -306,7 +309,9 @@ def test_no_eligible_projects(self) -> None: assert "night_shift.no_eligible_projects" in info_events run = SeerNightShiftRun.objects.get(organization=org) - assert run.extras.get("error_message") is None + triage_state = run.extras["kinds"]["agentic_triage"] + assert triage_state["status"] == "skipped" + assert triage_state["reason"] == "no_eligible_projects" assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_eligible_projects_error_records_error_message(self) -> None: @@ -322,7 +327,9 @@ def test_eligible_projects_error_records_error_message(self) -> None: run_night_shift_for_org(org.id) run = SeerNightShiftRun.objects.get(organization=org) - assert run.extras["error_message"] == "Failed to get eligible projects" + triage_state = run.extras["kinds"]["agentic_triage"] + assert triage_state["status"] == "failed" + assert triage_state["error_message"] == "Failed to get eligible projects" assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_selects_candidates_and_skips_triggered(self) -> None: @@ -358,18 +365,19 @@ def test_selects_candidates_and_skips_triggered(self) -> None: assert candidates[1]["seer_run_id"] == "101" run = SeerNightShiftRun.objects.get(organization=org) - assert run.extras.get("error_message") is None - assert run.extras == { - "options": { - "source": "cron", - "max_candidates": 10, - "dry_run": False, - "intelligence_level": "high", - "reasoning_effort": "high", - "extra_triage_instructions": "", - }, - "agent_run_id": 1, + assert run.extras["options"] == { + "source": "cron", + "max_candidates": 10, + "dry_run": False, + "intelligence_level": "high", + "reasoning_effort": "high", + "extra_triage_instructions": "", } + triage_state = run.extras["kinds"]["agentic_triage"] + assert triage_state["status"] == "succeeded" + assert triage_state["agent_run_id"] == 1 + assert triage_state["num_candidates"] == 2 + assert "error_message" not in triage_state result_group_ids = set( SeerNightShiftRunResult.objects.filter(run=run, kind="agentic_triage").values_list( @@ -398,7 +406,9 @@ def test_explorer_triage_error_propagates_to_run(self) -> None: run_night_shift_for_org(org.id) run = SeerNightShiftRun.objects.get(organization=org) - assert run.extras["error_message"] == "Night shift run failed" + triage_state = run.extras["kinds"]["agentic_triage"] + assert triage_state["status"] == "failed" + assert triage_state["error_message"] == "Night shift run failed" assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_triggers_autofix_with_correct_stopping_point(self) -> None: @@ -536,7 +546,9 @@ def test_skips_autofix_when_no_seer_quota(self) -> None: mock_trigger.assert_not_called() run = SeerNightShiftRun.objects.get(organization=org) - assert run.extras["error_message"] == "No Seer quota available" + triage_state = run.extras["kinds"]["agentic_triage"] + assert triage_state["status"] == "skipped" + assert triage_state["reason"] == "no_seer_quota" assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_skips_issue_row_on_trigger_failure(self) -> None: @@ -684,17 +696,16 @@ def test_extras_contain_options_and_target_project_ids(self) -> None: ) run = SeerNightShiftRun.objects.get(organization=org) - assert run.extras == { - "options": { - "source": "manual", - "max_candidates": 5, - "dry_run": True, - "intelligence_level": "high", - "reasoning_effort": "high", - "extra_triage_instructions": "", - }, - "target_project_ids": [project.id], + assert run.extras["options"] == { + "source": "manual", + "max_candidates": 5, + "dry_run": True, + "intelligence_level": "high", + "reasoning_effort": "high", + "extra_triage_instructions": "", } + assert run.extras["target_project_ids"] == [project.id] + assert "agentic_triage" in run.extras["kinds"] def test_extras_contain_triggering_user_id_when_provided(self) -> None: org = self.create_organization() @@ -786,3 +797,110 @@ def test_bucket_boundaries(self) -> None: ] for score, expected in cases: assert TriageAction.from_fixability_score(score) == expected + + +@django_db_all +class TestMultiKindDispatch(TestCase): + """Per-kind feature gating + dispatch added by feedback summary feature.""" + + def _create_org_with_seer(self): + org = self.create_organization() + project = self.create_project(organization=org) + repo = self.create_repo(project=project, provider="github", name=f"owner/{project.slug}") + SeerProjectRepository.objects.create(project=project, repository=repo) + return org + + def test_dispatches_both_kinds_when_both_flags_set(self) -> None: + org = self._create_org_with_seer() + + with ( + self.options({"seer.night_shift.enable": True}), + self.feature( + { + "organizations:seer-night-shift": [org.slug], + "organizations:seer-night-shift-feedback-summary": [org.slug], + "organizations:gen-ai-features": [org.slug], + "organizations:seat-based-seer-enabled": [org.slug], + } + ), + patch("sentry.tasks.seer.night_shift.cron.run_night_shift_for_org") as mock_worker, + ): + schedule_night_shift() + + mock_worker.apply_async.assert_called_once() + kwargs = mock_worker.apply_async.call_args.kwargs["kwargs"] + assert sorted(kwargs["kinds"]) == ["agentic_triage", "feedback_summary"] + + def test_dispatches_only_feedback_when_triage_flag_off(self) -> None: + org = self._create_org_with_seer() + + with ( + self.options({"seer.night_shift.enable": True}), + self.feature( + { + # seer-night-shift intentionally omitted + "organizations:seer-night-shift-feedback-summary": [org.slug], + "organizations:gen-ai-features": [org.slug], + "organizations:seat-based-seer-enabled": [org.slug], + } + ), + patch("sentry.tasks.seer.night_shift.cron.run_night_shift_for_org") as mock_worker, + ): + schedule_night_shift() + + mock_worker.apply_async.assert_called_once() + kwargs = mock_worker.apply_async.call_args.kwargs["kwargs"] + assert kwargs["kinds"] == ["feedback_summary"] + + def test_skips_org_when_no_kinds_enabled(self) -> None: + org = self._create_org_with_seer() + + with ( + self.options({"seer.night_shift.enable": True}), + self.feature( + { + "organizations:gen-ai-features": [org.slug], + "organizations:seat-based-seer-enabled": [org.slug], + # both kind flags intentionally omitted + } + ), + patch("sentry.tasks.seer.night_shift.cron.run_night_shift_for_org") as mock_worker, + ): + schedule_night_shift() + + mock_worker.apply_async.assert_not_called() + + def test_run_for_org_creates_one_parent_and_dispatches_per_kind(self) -> None: + org = self.create_organization() + + with patch( + "sentry.tasks.seer.night_shift.cron.run_night_shift_execution.apply_async" + ) as mock_execute: + run_night_shift_for_org( + org.id, + kinds=["agentic_triage", "feedback_summary"], + execute_in_task=True, + ) + + assert SeerNightShiftRun.objects.filter(organization=org).count() == 1 + run = SeerNightShiftRun.objects.get(organization=org) + assert set(run.extras["kinds"].keys()) == {"agentic_triage", "feedback_summary"} + assert run.extras["kinds"]["agentic_triage"]["status"] == "pending" + assert run.extras["kinds"]["feedback_summary"]["status"] == "pending" + + assert mock_execute.call_count == 2 + kinds_dispatched = sorted( + call.kwargs["kwargs"]["kind"] for call in mock_execute.call_args_list + ) + assert kinds_dispatched == ["agentic_triage", "feedback_summary"] + + def test_run_for_org_rejects_unknown_kinds(self) -> None: + org = self.create_organization() + + with patch("sentry.tasks.seer.night_shift.cron.run_night_shift_execution") as mock_execute: + result = run_night_shift_for_org(org.id, kinds=["bogus"]) + + assert result is None + assert SeerNightShiftRun.objects.filter(organization=org).count() == 0 + mock_execute.assert_not_called() + mock_execute.apply_async.assert_not_called() From eedb929984b4ea531c4b416d027f7fb0b3be39a8 Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 21:45:47 -0400 Subject: [PATCH 11/12] fix(seer): Address Cursor review of multi-kind night shift run 1. agentic_triage no longer does run.update(extras=...) after start_run. That call read a stale in-memory snapshot of run.extras and wrote the whole column back, clobbering any concurrent _update_kind_state writes from the feedback_summary branch. The agent_run_id is already persisted via cron's _update_kind_state on the success path. 2. feedback_summary now raises RuntimeError when the agent finishes without producing an artifact, instead of returning the agent_run_id. Cron's existing exception handler then correctly marks the kind as "failed" rather than "succeeded". --- .../tasks/seer/night_shift/agentic_triage.py | 2 -- .../tasks/seer/night_shift/feedback_summary.py | 4 +++- .../seer/night_shift/test_feedback_summary.py | 15 +++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/sentry/tasks/seer/night_shift/agentic_triage.py b/src/sentry/tasks/seer/night_shift/agentic_triage.py index 41541d0c94ef..353039b15fd0 100644 --- a/src/sentry/tasks/seer/night_shift/agentic_triage.py +++ b/src/sentry/tasks/seer/night_shift/agentic_triage.py @@ -113,8 +113,6 @@ def _triage_candidates( artifact_schema=_TriageResponse, ) - run.update(extras={**run.extras, "agent_run_id": agent_run_id}) - logger.info( "night_shift.explorer_run_started", extra={ diff --git a/src/sentry/tasks/seer/night_shift/feedback_summary.py b/src/sentry/tasks/seer/night_shift/feedback_summary.py index 326a77c5a629..9eb224197ca9 100644 --- a/src/sentry/tasks/seer/night_shift/feedback_summary.py +++ b/src/sentry/tasks/seer/night_shift/feedback_summary.py @@ -119,7 +119,9 @@ def agentic_feedback_summary_strategy( 1, attributes={"error_type": "no_artifact"}, ) - return agent_run_id + raise RuntimeError( + f"Feedback summary agent finished with status={state.status} but produced no artifact" + ) except Exception: sentry_sdk.metrics.count( "night_shift.feedback_summary_error", diff --git a/tests/sentry/tasks/seer/night_shift/test_feedback_summary.py b/tests/sentry/tasks/seer/night_shift/test_feedback_summary.py index 7e7ea352b73e..282fc256c586 100644 --- a/tests/sentry/tasks/seer/night_shift/test_feedback_summary.py +++ b/tests/sentry/tasks/seer/night_shift/test_feedback_summary.py @@ -1,6 +1,7 @@ from datetime import timedelta from unittest.mock import MagicMock, patch +import pytest from django.utils import timezone from sentry.issues.grouptype import FeedbackGroup @@ -117,7 +118,7 @@ def test_skipped_when_insufficient_feedbacks(self) -> None: mock_client.start_run.assert_not_called() assert not SeerNightShiftRunResult.objects.filter(run=run).exists() - def test_no_artifact_returns_run_id_without_writing_row(self) -> None: + def test_no_artifact_raises_without_writing_row(self) -> None: org = self.create_organization() project = self.create_project(organization=org) _seed_feedbacks(self, project, MIN_FEEDBACKS_TO_SUMMARIZE) @@ -145,13 +146,15 @@ def start_run(self, **kwargs): def get_run(self, run_id, **kwargs): return empty_state - with patch( - "sentry.tasks.seer.night_shift.feedback_summary.SeerAgentClient", - return_value=_StubClient(), + with ( + patch( + "sentry.tasks.seer.night_shift.feedback_summary.SeerAgentClient", + return_value=_StubClient(), + ), + pytest.raises(RuntimeError, match="no artifact"), ): - agent_run_id = agentic_feedback_summary_strategy(org, run=run) + agentic_feedback_summary_strategy(org, run=run) - assert agent_run_id == 99 assert not SeerNightShiftRunResult.objects.filter(run=run).exists() From 48a73471c2fb5f5fdc0e8b885842e6f25fac062b Mon Sep 17 00:00:00 2001 From: Trevor Elkins Date: Mon, 4 May 2026 21:51:15 -0400 Subject: [PATCH 12/12] fix(seer): Drop unused type ignore on _validated_kinds append Mypy now narrows `k` to NightShiftKind via the membership check against ALL_NIGHT_SHIFT_KINDS, so the type: ignore is no longer needed. --- src/sentry/tasks/seer/night_shift/cron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index 4260e8d08594..010b1c49c3b5 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -504,7 +504,7 @@ def _validated_kinds(kinds: Sequence[str] | None) -> list[NightShiftKind]: logger.error("night_shift.unknown_kind_requested", extra={"kind": k}) continue seen.add(k) - deduped.append(k) # type: ignore[arg-type] + deduped.append(k) return deduped