diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 79dec6b951e9..23b62cfad424 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -184,14 +184,6 @@ jobs: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 if: needs.select-tests.outputs.has-selected-tests == 'true' - - name: Setup sentry env - if: needs.select-tests.outputs.has-selected-tests == 'true' - uses: ./.github/actions/setup-sentry - id: setup - with: - mode: backend-ci - skip-devservices: true - - name: Download selected tests artifact if: needs.select-tests.outputs.has-selected-tests == 'true' uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 diff --git a/.github/workflows/scripts/calculate-backend-test-shards.py b/.github/workflows/scripts/calculate-backend-test-shards.py index 46dcdafbad16..ca75354bfa3a 100755 --- a/.github/workflows/scripts/calculate-backend-test-shards.py +++ b/.github/workflows/scripts/calculate-backend-test-shards.py @@ -1,9 +1,18 @@ #!/usr/bin/env python3 +"""Calculate the number of backend test shards needed for CI. + +Uses AST-based static analysis to count tests instead of running +pytest --collect-only, which requires importing every module and +bootstrapping Django (~100s). AST parsing takes a few seconds. +""" + +from __future__ import annotations + +import ast import json import math import os import re -import subprocess import sys from pathlib import Path @@ -12,85 +21,126 @@ MAX_SHARDS = 22 DEFAULT_SHARDS = MAX_SHARDS +IGNORED_DIRS = frozenset(("tests/acceptance/", "tests/apidocs/", "tests/js/", "tests/tools/")) + + +def _resolve(node: ast.expr, scope: dict[str, ast.expr]) -> ast.expr: + """Chase Name and Subscript references back to a concrete AST node.""" + if isinstance(node, ast.Name) and node.id in scope: + return _resolve(scope[node.id], scope) + if ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and isinstance(node.slice, ast.Constant) + and isinstance(node.slice.value, int) + and node.value.id in scope + ): + target = _resolve(scope[node.value.id], scope) + i = node.slice.value + if isinstance(target, (ast.List, ast.Tuple)) and 0 <= i < len(target.elts): + return _resolve(target.elts[i], scope) + return node + + +def _parametrize_count(dec: ast.expr, scope: dict[str, ast.expr]) -> int | None: + """If *dec* is a ``@pytest.mark.parametrize``, return the case count.""" + dec = _resolve(dec, scope) + if not isinstance(dec, ast.Call) or len(dec.args) < 2: + return None + f = dec.func + if not ( + isinstance(f, ast.Attribute) + and f.attr == "parametrize" + and isinstance(f.value, ast.Attribute) + and f.value.attr == "mark" + and isinstance(f.value.value, ast.Name) + and f.value.value.id == "pytest" + ): + return None + argvals = _resolve(dec.args[1], scope) + return len(argvals.elts) if isinstance(argvals, (ast.List, ast.Tuple)) else None + + +_TEST_FUNC_RE = re.compile(r"^\s*(?:async\s+)?def\s+test_", re.MULTILINE) + + +def count_tests_in_file(filepath: Path) -> int: + """Count the test items *filepath* would produce. + + Accounts for ``@pytest.mark.parametrize`` multipliers including + stacked decorators. + """ + try: + source = filepath.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + return 0 + + # Fast path: no parametrize means each def test_ is exactly one test. + if "parametrize" not in source: + return len(_TEST_FUNC_RE.findall(source)) + + try: + tree = ast.parse(source, filename=str(filepath)) + except SyntaxError: + return len(_TEST_FUNC_RE.findall(source)) + + scope: dict[str, ast.expr] = {} + for node in ast.iter_child_nodes(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + scope[target.id] = node.value + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.value: + scope[node.target.id] = node.value + + total = 0 + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith( + "test_" + ): + counts = ( + c for d in node.decorator_list if (c := _parametrize_count(d, scope)) is not None + ) + total += math.prod(counts, start=1) + return total + def collect_test_count() -> int | None: - """Collect the number of tests to run, either from selected files or full suite.""" + """Count tests via AST analysis of test files.""" selected_tests_file = os.environ.get("SELECTED_TESTS_FILE") if selected_tests_file: path = Path(selected_tests_file) if not path.exists(): - print(f"Selected tests file not found: {selected_tests_file}", file=sys.stderr) + print( + f"Selected tests file not found: {selected_tests_file}", + file=sys.stderr, + ) return None - with path.open() as f: - selected_files = [line.strip() for line in f if line.strip()] + test_files = [Path(line.strip()) for line in path.read_text().splitlines() if line.strip()] - if not selected_files: + if not test_files: print("No selected test files, running 0 tests", file=sys.stderr) return 0 - print(f"Counting tests in {len(selected_files)} selected files", file=sys.stderr) - - pytest_args = [ - "pytest", - # Always pass tests/ directory to ensure proper conftest loading order. - # SELECTED_TESTS_FILE env var triggers filtering in pytest_collection_modifyitems. - "tests", - "--collect-only", - "--quiet", - "--ignore=tests/acceptance", - "--ignore=tests/apidocs", - "--ignore=tests/js", - "--ignore=tests/tools", - ] + print(f"Counting tests in {len(test_files)} selected files", file=sys.stderr) + else: + tests_dir = Path("tests") + if not tests_dir.is_dir(): + print("tests/ directory not found", file=sys.stderr) + return None - try: - result = subprocess.run( - pytest_args, - capture_output=True, - text=True, - check=False, + test_files = sorted( + p + for p in tests_dir.rglob("test_*.py") + if not any(str(p).startswith(d) for d in IGNORED_DIRS) ) + print(f"Found {len(test_files)} test files", file=sys.stderr) - # Parse output for test count - # Format without deselection: "27000 tests collected in 18.53s" - # Format with deselection: "29/31510 tests collected (31481 deselected) in 18.13s" - output = result.stdout + result.stderr - - # Try format with deselection first (selected/total) - match = re.search(r"(\d+)/\d+ tests? collected", output) - if match: - count = int(match.group(1)) - print(f"Collected {count} tests", file=sys.stderr) - return count - - # Fall back to format without deselection - match = re.search(r"(\d+) tests? collected", output) - if match: - count = int(match.group(1)) - print(f"Collected {count} tests", file=sys.stderr) - return count - - if result.returncode == 5: - # Exit code 5 indicates no tests collected (https://docs.pytest.org/en/stable/reference/exit-codes.html) - # This can stem from files being deleted in a branch/PR. - print("No tests collected (exit 5)", file=sys.stderr) - return 0 - - if result.returncode != 0: - print( - f"Pytest collection failed (exit {result.returncode})", - file=sys.stderr, - ) - print(result.stderr, file=sys.stderr) - return None - - print("No tests collected", file=sys.stderr) - return 0 - except Exception as e: - print(f"Error collecting tests: {e}", file=sys.stderr) - return None + total = sum(count_tests_in_file(f) for f in test_files) + print(f"Counted {total} tests via AST analysis", file=sys.stderr) + return total def calculate_shards(test_count: int | None) -> int: diff --git a/.github/workflows/scripts/compute-sentry-selected-tests.py b/.github/workflows/scripts/compute-sentry-selected-tests.py index 24d1c626f8ca..b5912bb05c2e 100644 --- a/.github/workflows/scripts/compute-sentry-selected-tests.py +++ b/.github/workflows/scripts/compute-sentry-selected-tests.py @@ -41,12 +41,36 @@ "tests/integration/", ) +# Most of these won't have coverage info because they're evaluated at +# module load time and app warmup, before any per-test coverage context is active. +# +# Tracking a "startup" coverage context doesn't work: django.setup() +# eagerly imports models, fields, validators, utils, etc. We also have +# large dynamic __init__'s so a startup context would select nearly every +# test. FULL_SUITE_TRIGGERS: list[str | re.Pattern[str]] = [ - "src/sentry/testutils/pytest/sentry.py", + re.compile(r"^src/sentry/testutils/pytest/"), + re.compile(r"(^|/)conftest\.py$"), + "src/sentry/runner/initializer.py", "src/sentry/constants.py", - "pyproject.toml", + # option defaults registered at startup via initialize_app() + re.compile(r"^src/sentry/options/"), + # feature flags registered via manager.add() at import time + re.compile(r"^src/sentry/features/"), + # signal definitions created at module level; receivers depend on these + "src/sentry/signals.py", + # signal handlers registered globally via initialize_receivers() + re.compile(r"^src/sentry/receivers/"), + # stdlib/third-party monkey-patches applied before Django setup + re.compile(r"^src/sentry/monkey/"), + # monkeypatches transaction.atomic for silo-aware DB routing + re.compile(r"^src/sentry/silo/patches/"), + # SiloRouter loaded via DATABASE_ROUTERS; affects every DB query + "src/sentry/db/router.py", "src/sentry/conf/server.py", "src/sentry/web/urls.py", + "pyproject.toml", + "uv.lock", re.compile(r"/migrations/\d{4}_[^/]+\.py$"), ] diff --git a/.github/workflows/scripts/test_calculate_backend_test_shards.py b/.github/workflows/scripts/test_calculate_backend_test_shards.py new file mode 100644 index 000000000000..fbba4260072e --- /dev/null +++ b/.github/workflows/scripts/test_calculate_backend_test_shards.py @@ -0,0 +1,494 @@ +from __future__ import annotations + +import importlib +import textwrap +from pathlib import Path + +import pytest + +# Module has a hyphen in its name, so use importlib. +_mod = importlib.import_module("calculate-backend-test-shards") +count_tests_in_file = _mod.count_tests_in_file +calculate_shards = _mod.calculate_shards +collect_test_count = _mod.collect_test_count + + +def _write(tmp_path: Path, source: str) -> Path: + p = tmp_path / "test_example.py" + p.write_text(textwrap.dedent(source)) + return p + + +class TestCountTestsInFile: + def test_plain_functions(self, tmp_path): + p = _write( + tmp_path, + """\ + def test_a(): + pass + + def test_b(): + pass + + def helper(): + pass + """, + ) + assert count_tests_in_file(p) == 2 + + def test_methods_in_class(self, tmp_path): + p = _write( + tmp_path, + """\ + class TestFoo: + def test_one(self): + pass + + def test_two(self): + pass + + def helper(self): + pass + """, + ) + assert count_tests_in_file(p) == 2 + + def test_parametrize_list(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.parametrize("x", [1, 2, 3]) + def test_vals(x): + pass + """, + ) + assert count_tests_in_file(p) == 3 + + def test_parametrize_tuple(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.parametrize("x", (True, False)) + def test_vals(x): + pass + """, + ) + assert count_tests_in_file(p) == 2 + + def test_parametrize_with_pytest_param(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.parametrize( + "x", + [ + pytest.param(1, id="one"), + pytest.param(2, id="two"), + ], + ) + def test_vals(x): + pass + """, + ) + assert count_tests_in_file(p) == 2 + + def test_parametrize_empty_list(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.parametrize("x", []) + def test_vals(x): + pass + """, + ) + assert count_tests_in_file(p) == 0 + + def test_stacked_parametrize(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.parametrize("a", [1, 2]) + @pytest.mark.parametrize("b", ["x", "y", "z"]) + def test_combo(a, b): + pass + """, + ) + assert count_tests_in_file(p) == 6 + + def test_variable_reference(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + CASES = [1, 2, 3, 4] + + @pytest.mark.parametrize("x", CASES) + def test_vals(x): + pass + """, + ) + assert count_tests_in_file(p) == 4 + + def test_annotated_assignment_variable(self, tmp_path): + p = _write( + tmp_path, + """\ + from typing import Any + import pytest + + CASES: list[dict[str, Any]] = [{"a": 1}, {"a": 2}, {"a": 3}] + + @pytest.mark.parametrize("case", CASES) + def test_vals(case): + pass + """, + ) + assert count_tests_in_file(p) == 3 + + def test_stored_decorator(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + clients = pytest.mark.parametrize( + "client", + [ + pytest.param("redis", id="redis"), + pytest.param("memcached", id="memcached"), + ], + ) + + @clients + def test_get(client): + pass + + @clients + def test_set(client): + pass + """, + ) + assert count_tests_in_file(p) == 4 + + def test_subscript_reference(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + CASES = ( + ["query", "mode"], + [("q1", "m1"), ("q2", "m2"), ("q3", "m3")], + ) + + @pytest.mark.parametrize(CASES[0], CASES[1]) + def test_query(query, mode): + pass + """, + ) + assert count_tests_in_file(p) == 3 + + def test_unresolvable_parametrize_counts_as_one(self, tmp_path): + """Function calls and attribute access can't be resolved statically.""" + p = _write( + tmp_path, + """\ + import os + import pytest + + @pytest.mark.parametrize("f", os.listdir("/tmp")) + def test_files(f): + pass + """, + ) + assert count_tests_in_file(p) == 1 + + def test_mixed_resolvable_and_unresolvable(self, tmp_path): + """One stacked parametrize resolved, one not — resolved one still multiplies.""" + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.parametrize("a", [1, 2]) + @pytest.mark.parametrize("b", some_func()) + def test_combo(a, b): + pass + """, + ) + # [1, 2] resolves to 2; some_func() does not, so treated as 1. + # Total = 2 * 1 = 2. + assert count_tests_in_file(p) == 2 + + def test_parametrize_with_ids_kwarg(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.parametrize( + "enabled", + [True, False], + ids=["with_feature", "without_feature"], + ) + def test_feature(enabled): + pass + """, + ) + assert count_tests_in_file(p) == 2 + + def test_async_def(self, tmp_path): + p = _write( + tmp_path, + """\ + async def test_async(): + pass + """, + ) + assert count_tests_in_file(p) == 1 + + def test_non_test_functions_ignored(self, tmp_path): + p = _write( + tmp_path, + """\ + def helper(): + pass + + def setup_module(): + pass + + def teardown_function(): + pass + """, + ) + assert count_tests_in_file(p) == 0 + + def test_empty_file(self, tmp_path): + p = tmp_path / "test_empty.py" + p.write_text("") + assert count_tests_in_file(p) == 0 + + def test_syntax_error(self, tmp_path): + p = tmp_path / "test_bad.py" + p.write_text("def test_a(:\n") + # Regex fast path still finds the def — best-effort count. + assert count_tests_in_file(p) == 1 + + def test_nonexistent_file(self, tmp_path): + p = tmp_path / "test_nope.py" + assert count_tests_in_file(p) == 0 + + def test_parametrize_tuple_argnames(self, tmp_path): + """Argnames passed as tuple instead of string.""" + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.parametrize( + ("name", "value"), + [("a", 1), ("b", 2)], + ) + def test_pairs(name, value): + pass + """, + ) + assert count_tests_in_file(p) == 2 + + def test_class_with_parametrize_on_method(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + class TestMath: + @pytest.mark.parametrize("n", [1, 2, 3]) + def test_square(self, n): + pass + + def test_plain(self): + pass + """, + ) + assert count_tests_in_file(p) == 4 + + def test_multiple_classes_and_functions(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + def test_standalone(): + pass + + class TestA: + def test_one(self): + pass + + class TestB: + @pytest.mark.parametrize("x", [1, 2]) + def test_two(self, x): + pass + """, + ) + assert count_tests_in_file(p) == 4 + + def test_non_parametrize_decorators_ignored(self, tmp_path): + p = _write( + tmp_path, + """\ + import pytest + + @pytest.mark.slow + @pytest.mark.django_db + def test_decorated(): + pass + """, + ) + assert count_tests_in_file(p) == 1 + + +class TestCalculateShards: + def test_none_returns_default(self): + assert calculate_shards(None) == 22 + + def test_zero_returns_zero(self): + assert calculate_shards(0) == 0 + + def test_small_count(self): + assert calculate_shards(100) == 1 + + def test_exact_boundary(self): + assert calculate_shards(300) == 1 + + def test_just_over_boundary(self): + assert calculate_shards(301) == 2 + + def test_large_count_capped(self): + assert calculate_shards(100_000) == 22 + + def test_mid_range(self): + # 1500 / 300 = 5 + assert calculate_shards(1500) == 5 + + def test_rounds_up(self): + # 301 / 300 = 1.003 → ceil = 2 + assert calculate_shards(301) == 2 + + +class TestCollectTestCount: + def test_selected_tests_file(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + # Create two test files. + (tmp_path / "test_a.py").write_text("def test_one(): pass\ndef test_two(): pass\n") + (tmp_path / "test_b.py").write_text("def test_three(): pass\n") + + selected = tmp_path / "selected.txt" + selected.write_text(f"{tmp_path / 'test_a.py'}\n{tmp_path / 'test_b.py'}\n") + monkeypatch.setenv("SELECTED_TESTS_FILE", str(selected)) + + assert collect_test_count() == 3 + + def test_selected_tests_file_empty(self, tmp_path, monkeypatch): + selected = tmp_path / "selected.txt" + selected.write_text("\n") + monkeypatch.setenv("SELECTED_TESTS_FILE", str(selected)) + + assert collect_test_count() == 0 + + def test_selected_tests_file_missing(self, tmp_path, monkeypatch): + monkeypatch.setenv("SELECTED_TESTS_FILE", str(tmp_path / "nope.txt")) + assert collect_test_count() is None + + def test_full_suite_walks_tests_dir(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("SELECTED_TESTS_FILE", raising=False) + + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_foo.py").write_text("def test_a(): pass\n") + + sub = tests / "sub" + sub.mkdir() + (sub / "test_bar.py").write_text("def test_b(): pass\ndef test_c(): pass\n") + + assert collect_test_count() == 3 + + def test_full_suite_ignores_excluded_dirs(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("SELECTED_TESTS_FILE", raising=False) + + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_ok.py").write_text("def test_a(): pass\n") + + for excluded in ("acceptance", "apidocs", "js", "tools"): + d = tests / excluded + d.mkdir() + (d / "test_skip.py").write_text("def test_no(): pass\n") + + assert collect_test_count() == 1 + + def test_ignored_dirs_prefix_does_not_over_match(self, tmp_path, monkeypatch): + """tests/js/ must not exclude tests/json/.""" + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("SELECTED_TESTS_FILE", raising=False) + + tests = tmp_path / "tests" + (tests / "js").mkdir(parents=True) + (tests / "js" / "test_skip.py").write_text("def test_no(): pass\n") + (tests / "json").mkdir() + (tests / "json" / "test_keep.py").write_text("def test_yes(): pass\n") + + assert collect_test_count() == 1 + + def test_no_tests_dir(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("SELECTED_TESTS_FILE", raising=False) + assert collect_test_count() is None + + +SENTRY_TESTS = Path(__file__).resolve().parent.parent.parent.parent / "tests" + + +@pytest.mark.skipif( + not SENTRY_TESTS.is_dir(), + reason="sentry tests/ directory not found", +) +def test_all_sentry_test_files(): + """Parse every test file in the repo — no crashes, every file returns >= 0.""" + ignored = {"tests/acceptance", "tests/apidocs", "tests/js", "tests/tools"} + files = sorted( + p + for p in SENTRY_TESTS.rglob("test_*.py") + if not any(str(p.relative_to(SENTRY_TESTS.parent)).startswith(d) for d in ignored) + ) + assert len(files) > 2000, f"expected >2000 test files, found {len(files)}" + + failures = [] + total = 0 + for f in files: + try: + n = count_tests_in_file(f) + assert n >= 0 + total += n + except Exception as exc: + failures.append((f, exc)) + + assert not failures, f"{len(failures)} files failed:\n" + "\n".join( + f" {f}: {e}" for f, e in failures[:20] + ) + # Sanity-check: the sentry test suite has ~30k tests at the time of writing. + assert total > 29_000, f"total {total} seems too low" diff --git a/src/sentry/api/serializers/models/rule.py b/src/sentry/api/serializers/models/rule.py index 1a518cdfc172..0473baf6192c 100644 --- a/src/sentry/api/serializers/models/rule.py +++ b/src/sentry/api/serializers/models/rule.py @@ -796,9 +796,15 @@ def serialize(self, obj: Workflow, attrs, user, **kwargs) -> RuleSerializerRespo "createdBy": attrs.get("created_by", None), "environment": environment.name if environment is not None else None, "projects": [p.slug for p in attrs["projects"]], - "status": "active" if obj.enabled else "disabled", - "snooze": "snooze" in attrs, + # Workflow.enabled is toggled by snooze-for-everyone, but "disabled" in the + # UI means a broken/misconfigured rule (matching legacy Rule.status/ObjectStatus). + # Snooze state is communicated via the snooze fields instead. + "status": "active", + "snooze": not obj.enabled, } + if not obj.enabled: + workflow_rule["snoozeForEveryone"] = True + if "last_triggered" in attrs: workflow_rule["lastTriggered"] = attrs["last_triggered"] diff --git a/src/sentry/incidents/subscription_processor.py b/src/sentry/incidents/subscription_processor.py index 6276ab2701ef..25751bbcc68e 100644 --- a/src/sentry/incidents/subscription_processor.py +++ b/src/sentry/incidents/subscription_processor.py @@ -15,6 +15,7 @@ get_comparison_aggregation_value, get_crash_rate_alert_metrics_aggregation_value_helper, ) +from sentry.incidents.utils.subscription_limits import is_metric_subscription_allowed from sentry.incidents.utils.types import ( DATA_SOURCE_SNUBA_QUERY_SUBSCRIPTION, AnomalyDetectionUpdate, @@ -53,46 +54,6 @@ class MetricIssueDetectorConfig(TypedDict): detection_type: Literal["static", "percent", "dynamic"] -def has_downgraded(dataset: str, organization: Organization) -> bool: - """ - Check if the organization has downgraded since the subscription was created. - """ - supports_metrics_issues = features.has("organizations:incidents", organization) - if dataset == Dataset.Events.value and not supports_metrics_issues: - metrics.incr("incidents.alert_rules.ignore_update_missing_incidents") - return True - - supports_performance_view = features.has("organizations:performance-view", organization) - if dataset == Dataset.Transactions.value and not ( - supports_metrics_issues and supports_performance_view - ): - metrics.incr("incidents.alert_rules.ignore_update_missing_incidents_performance") - return True - - supports_explore_view = features.has("organizations:visibility-explore-view", organization) - if dataset == Dataset.EventsAnalyticsPlatform.value and not ( - supports_metrics_issues and supports_explore_view - ): - metrics.incr("incidents.alert_rules.ignore_update_missing_incidents_eap") - return True - - if dataset == Dataset.PerformanceMetrics.value and not features.has( - "organizations:on-demand-metrics-extraction", organization - ): - metrics.incr("incidents.alert_rules.ignore_update_missing_on_demand") - return True - - if not supports_metrics_issues: - # These should probably be downgraded, but we should know the impact first. - metrics.incr( - "incidents.alert_rules.no_incidents_not_downgraded", - sample_rate=1.0, - tags={"dataset": dataset}, - ) - - return False - - class SubscriptionProcessor: """ Class for processing subscription updates for workflow engine. @@ -268,9 +229,20 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> bool: dataset = self.subscription.snuba_query.dataset organization = self.subscription.project.organization - if has_downgraded(dataset, organization): + if not is_metric_subscription_allowed(dataset, organization): + metrics.incr( + "incidents.alert_rules.ignore_update_not_enabled", + tags={"dataset": dataset}, + ) return False + if not features.has("organizations:incidents", organization): + metrics.incr( + "incidents.alert_rules.no_incidents_not_downgraded", + sample_rate=1.0, + tags={"dataset": dataset}, + ) + if subscription_update["timestamp"] <= self.last_update: metrics.incr("incidents.alert_rules.skipping_already_processed_update") return False diff --git a/src/sentry/incidents/utils/subscription_limits.py b/src/sentry/incidents/utils/subscription_limits.py index 92332aa5d6ab..fc4a4edae41a 100644 --- a/src/sentry/incidents/utils/subscription_limits.py +++ b/src/sentry/incidents/utils/subscription_limits.py @@ -1,7 +1,35 @@ +from __future__ import annotations + from django.conf import settings -from sentry import options +from sentry import features, options from sentry.models.organization import Organization +from sentry.snuba.dataset import Dataset + + +def is_metric_subscription_allowed(dataset: str, organization: Organization) -> bool: + """ + Check whether the given organization is allowed to have a metric alert + subscription for the given dataset. + + Returns True if allowed, False if the organization lacks the required features + (e.g. after a plan downgrade). + """ + has_incidents = features.has("organizations:incidents", organization) + if dataset == Dataset.Events.value: + return has_incidents + + if dataset == Dataset.Transactions.value: + return has_incidents and features.has("organizations:performance-view", organization) + + if dataset == Dataset.EventsAnalyticsPlatform.value: + return has_incidents and features.has("organizations:visibility-explore-view", organization) + + if dataset == Dataset.PerformanceMetrics.value: + return features.has("organizations:on-demand-metrics-extraction", organization) + + # Other datasets (e.g. Metrics/sessions) aren't gated here but probably should be. + return True def get_max_metric_alert_subscriptions(organization: Organization) -> int: diff --git a/src/sentry/middleware/viewer_context.py b/src/sentry/middleware/viewer_context.py index b8b6b48cfe23..144fdd7a5940 100644 --- a/src/sentry/middleware/viewer_context.py +++ b/src/sentry/middleware/viewer_context.py @@ -2,6 +2,7 @@ from collections.abc import Callable +from django.conf import settings from django.http.request import HttpRequest from django.http.response import HttpResponseBase @@ -26,6 +27,12 @@ def ViewerContextMiddleware_impl(request: HttpRequest) -> HttpResponseBase: if not enabled: return get_response(request) + # This avoids touching user session, which means we avoid + # setting `Vary: Cookie` as a response header which will + # break HTTP caching entirely. + if request.path_info.startswith(settings.ANONYMOUS_STATIC_PREFIXES): + return get_response(request) + ctx = _viewer_context_from_request(request) with viewer_context_scope(ctx): return get_response(request) diff --git a/src/sentry/models/project.py b/src/sentry/models/project.py index 612fb7aa7c75..b6885c87a30d 100644 --- a/src/sentry/models/project.py +++ b/src/sentry/models/project.py @@ -881,7 +881,9 @@ def normalize_before_relocation_import( def write_relocation_import( self, scope: ImportScope, flags: ImportFlags ) -> tuple[int, ImportKind] | None: - from sentry.receivers.project_detectors import disable_default_detector_creation + from sentry.workflow_engine.receivers.project_detectors import ( + disable_default_detector_creation, + ) with disable_default_detector_creation(): return super().write_relocation_import(scope, flags) diff --git a/src/sentry/notifications/defaults.py b/src/sentry/notifications/defaults.py index 1c3edfc865f7..ac67a5312c83 100644 --- a/src/sentry/notifications/defaults.py +++ b/src/sentry/notifications/defaults.py @@ -24,6 +24,7 @@ NotificationSettingEnum.QUOTA_PROFILE_DURATION_UI: NotificationSettingsOptionEnum.ALWAYS, NotificationSettingEnum.QUOTA_SEER_BUDGET: NotificationSettingsOptionEnum.ALWAYS, NotificationSettingEnum.QUOTA_LOG_BYTES: NotificationSettingsOptionEnum.ALWAYS, + NotificationSettingEnum.QUOTA_TRACE_METRIC_BYTES: NotificationSettingsOptionEnum.ALWAYS, NotificationSettingEnum.QUOTA_SEER_USERS: NotificationSettingsOptionEnum.ALWAYS, NotificationSettingEnum.QUOTA_SIZE_ANALYSIS: NotificationSettingsOptionEnum.ALWAYS, NotificationSettingEnum.QUOTA_WARNINGS: NotificationSettingsOptionEnum.ALWAYS, diff --git a/src/sentry/notifications/types.py b/src/sentry/notifications/types.py index a68d5c222b32..09ced27513c7 100644 --- a/src/sentry/notifications/types.py +++ b/src/sentry/notifications/types.py @@ -34,6 +34,7 @@ class NotificationSettingEnum(ValueEqualityEnum): QUOTA_SEER_BUDGET = "quotaSeerBudget" QUOTA_SPEND_ALLOCATIONS = "quotaSpendAllocations" QUOTA_LOG_BYTES = "quotaLogBytes" + QUOTA_TRACE_METRIC_BYTES = "quotaTraceMetricBytes" QUOTA_SEER_USERS = "quotaSeerUsers" QUOTA_SIZE_ANALYSIS = "quotaSizeAnalyses" SPIKE_PROTECTION = "spikeProtection" @@ -152,6 +153,10 @@ class UserOptionsSettingsKey(Enum): NotificationSettingsOptionEnum.ALWAYS, NotificationSettingsOptionEnum.NEVER, }, + NotificationSettingEnum.QUOTA_TRACE_METRIC_BYTES: { + NotificationSettingsOptionEnum.ALWAYS, + NotificationSettingsOptionEnum.NEVER, + }, NotificationSettingEnum.QUOTA_SEER_USERS: { NotificationSettingsOptionEnum.ALWAYS, NotificationSettingsOptionEnum.NEVER, diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 15b6342ed800..be1dca261c44 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -3575,18 +3575,18 @@ default=10000, flags=FLAG_AUTOMATOR_MODIFIABLE, ) -# Tuning knobs for the periodic fire-history cleanup task. +# Tuning knobs for the periodic open-period-activity cleanup task. # time_limit is a wall-clock budget checked *between* batches, so a single # batch that exceeds it will still run to completion. Setting it to 0 # prevents any batches from running. register( - "workflow_engine.fire_history_cleanup.time_limit_seconds", + "workflow_engine.open_period_activity_cleanup.time_limit_seconds", type=Float, default=5.0, flags=FLAG_AUTOMATOR_MODIFIABLE, ) register( - "workflow_engine.fire_history_cleanup.batch_size", + "workflow_engine.open_period_activity_cleanup.batch_size", type=Int, default=10000, flags=FLAG_AUTOMATOR_MODIFIABLE, @@ -4106,7 +4106,7 @@ # Set via deploy config (SENTRY_OPTIONS); requires restart to change. register( "viewer-context.enabled", - default=False, + default=True, type=Bool, flags=FLAG_NOSTORE, ) diff --git a/src/sentry/projects/project_rules/creator.py b/src/sentry/projects/project_rules/creator.py index 5d3a0e435911..b34732a87e35 100644 --- a/src/sentry/projects/project_rules/creator.py +++ b/src/sentry/projects/project_rules/creator.py @@ -9,8 +9,8 @@ from sentry.models.project import Project from sentry.models.rule import Rule, RuleSource from sentry.types.actor import Actor +from sentry.workflow_engine.defaults.detectors import ensure_default_detectors from sentry.workflow_engine.migration_helpers.issue_alert_migration import IssueAlertMigrator -from sentry.workflow_engine.processors.detector import ensure_default_detectors from sentry.workflow_engine.utils.legacy_metric_tracking import report_used_legacy_models logger = logging.getLogger(__name__) diff --git a/src/sentry/receivers/__init__.py b/src/sentry/receivers/__init__.py index a8c00c33aa6f..83b66df9be09 100644 --- a/src/sentry/receivers/__init__.py +++ b/src/sentry/receivers/__init__.py @@ -9,7 +9,6 @@ from .outbox.cell import * # noqa: F401,F403 from .outbox.control import * # noqa: F401,F403 from .owners import * # noqa: F401,F403 -from .project_detectors import * # noqa: F401,F403 from .releases import * # noqa: F401,F403 from .rule_snooze import * # noqa: F401,F403 from .rules import * # noqa: F401,F403 diff --git a/src/sentry/testutils/factories.py b/src/sentry/testutils/factories.py index fe4ba6414a57..330d276b57d6 100644 --- a/src/sentry/testutils/factories.py +++ b/src/sentry/testutils/factories.py @@ -561,7 +561,9 @@ def create_project( create_default_detectors=True, **kwargs, ) -> Project: - from sentry.receivers.project_detectors import disable_default_detector_creation + from sentry.workflow_engine.receivers.project_detectors import ( + disable_default_detector_creation, + ) if not kwargs.get("name"): kwargs["name"] = petname.generate(2, " ", letters=10).title() diff --git a/src/sentry/workflow_engine/defaults/detectors.py b/src/sentry/workflow_engine/defaults/detectors.py new file mode 100644 index 000000000000..25d40f3c3e23 --- /dev/null +++ b/src/sentry/workflow_engine/defaults/detectors.py @@ -0,0 +1,286 @@ +import logging +from collections.abc import Mapping +from datetime import timedelta +from functools import cache + +from django.db import router, transaction +from rest_framework import status + +from sentry import features +from sentry.api.exceptions import SentryAPIException +from sentry.grouping.grouptype import ErrorGroupType +from sentry.incidents.grouptype import MetricIssue +from sentry.incidents.models.alert_rule import AlertRuleDetectionType +from sentry.incidents.utils.constants import INCIDENTS_SNUBA_SUBSCRIPTION_TYPE +from sentry.incidents.utils.types import DATA_SOURCE_SNUBA_QUERY_SUBSCRIPTION +from sentry.issue_detection.performance_detection import PERFORMANCE_DETECTOR_CONFIG_MAPPINGS +from sentry.issues import grouptype +from sentry.locks import locks +from sentry.models.project import Project +from sentry.projectoptions.defaults import DEFAULT_PROJECT_PERFORMANCE_DETECTION_SETTINGS +from sentry.seer.anomaly_detection.store_data_workflow_engine import send_new_detector_data +from sentry.seer.anomaly_detection.types import ( + AnomalyDetectionSeasonality, + AnomalyDetectionSensitivity, + AnomalyDetectionThresholdType, +) +from sentry.snuba.dataset import Dataset +from sentry.snuba.models import SnubaQuery, SnubaQueryEventType +from sentry.snuba.subscriptions import create_snuba_query, create_snuba_subscription +from sentry.utils.locking import UnableToAcquireLock +from sentry.workflow_engine.models import ( + DataCondition, + DataConditionGroup, + DataSource, + DataSourceDetector, + Detector, +) +from sentry.workflow_engine.models.data_condition import Condition +from sentry.workflow_engine.types import ( + ERROR_DETECTOR_NAME, + ISSUE_STREAM_DETECTOR_NAME, + DetectorPriorityLevel, +) +from sentry.workflow_engine.typings.grouptype import IssueStreamGroupType + +VALID_DEFAULT_DETECTOR_TYPES = [ + ErrorGroupType.slug, + IssueStreamGroupType.slug, + *[m.wfe_detector_type for m in PERFORMANCE_DETECTOR_CONFIG_MAPPINGS.values()], +] + +logger = logging.getLogger(__name__) + + +@cache +def get_disabled_platforms_by_detector_type() -> Mapping[str, frozenset[str]]: + """ + Map WFE detector types to platforms where they should be disabled by default. + Derives from DEFAULT_DETECTOR_DISABLING_CONFIGS using the detection_enabled_key. + """ + from sentry.issue_detection.detectors.disable_detectors import ( + DEFAULT_DETECTOR_DISABLING_CONFIGS, + ) + + disabled_by_detector_type: dict[str, frozenset[str]] = {} + + for disable_config in DEFAULT_DETECTOR_DISABLING_CONFIGS: + detector_option_key = disable_config["detector_project_option"] + languages_to_disable = disable_config["languages_to_disable"] + + # Find matching WFE detector via detection_enabled_key + for mapping in PERFORMANCE_DETECTOR_CONFIG_MAPPINGS.values(): + if mapping.detection_enabled_key == detector_option_key: + disabled_by_detector_type[mapping.wfe_detector_type] = frozenset( + languages_to_disable + ) + break + + return disabled_by_detector_type + + +class UnableToAcquireLockApiError(SentryAPIException): + status_code = status.HTTP_400_BAD_REQUEST + code = "unable_to_acquire_lock" + message = "Unable to acquire lock for issue alert migration." + + +def _ensure_detector(project: Project, type: str, default_enabled: bool = True) -> Detector: + """ + Ensure that a detector of a given type exists for a project. + If the Detector doesn't already exist, we try to acquire a lock to avoid double-creating, + and UnableToAcquireLockApiError if that fails. + """ + group_type = grouptype.registry.get_by_slug(type) + if not group_type: + raise ValueError(f"Group type {type} not registered") + slug = group_type.slug + if slug not in VALID_DEFAULT_DETECTOR_TYPES: + raise ValueError(f"Invalid default detector type: {slug}") + + # If it already exists, life is simple and we can return immediately. + # If there happen to be duplicates, we prefer the oldest. + existing = Detector.objects.filter(type=slug, project=project).order_by("id").first() + if existing: + return existing + + # If we may need to create it, we acquire a lock to avoid double-creating. + # There isn't a unique constraint on the detector, so we can't rely on get_or_create + # to avoid duplicates. + # However, by only locking during the one-time creation, the window for a race condition is small. + lock = locks.get( + f"workflow-engine-project-{slug}-detector:{project.id}", + duration=2, + name=f"workflow_engine_default_{slug}_detector", + ) + try: + with ( + # Creation should be fast, so it's worth blocking a little rather + # than failing a request. + lock.blocking_acquire(initial_delay=0.1, timeout=3), + transaction.atomic(router.db_for_write(Detector)), + ): + detector, _ = Detector.objects.get_or_create( + type=slug, + project=project, + defaults={ + "config": {}, + "name": ( + ERROR_DETECTOR_NAME + if slug == ErrorGroupType.slug + else ISSUE_STREAM_DETECTOR_NAME + if slug == IssueStreamGroupType.slug + else group_type.description + ), + "enabled": default_enabled, + }, + ) + return detector + except UnableToAcquireLock: + raise UnableToAcquireLockApiError + + +def ensure_default_anomaly_detector( + project: Project, owner_team_id: int | None = None, enabled: bool = True +) -> Detector | None: + """ + Ensure that a default anomaly detection metric monitor exists for a project. + If the Detector doesn't already exist, we try to acquire a lock to avoid double-creating. + """ + # If it already exists, return immediately. Prefer the oldest if duplicates exist. + existing = ( + Detector.objects.filter(type=MetricIssue.slug, project=project).order_by("id").first() + ) + if existing: + logger.info( + "create_default_anomaly_detector.already_exists", + extra={"project_id": project.id, "detector_id": existing.id}, + ) + return existing + + lock = locks.get( + f"workflow-engine-project-{MetricIssue.slug}-detector:{project.id}", + duration=2, + name=f"workflow_engine_default_{MetricIssue.slug}_detector", + ) + try: + with ( + lock.blocking_acquire(initial_delay=0.1, timeout=3), + transaction.atomic(router.db_for_write(Detector)), + ): + # Double-check after acquiring lock in case another process created it + existing = ( + Detector.objects.filter(type=MetricIssue.slug, project=project) + .order_by("id") + .first() + ) + if existing: + return existing + + try: + condition_group = DataConditionGroup.objects.create( + logic_type=DataConditionGroup.Type.ANY, + organization_id=project.organization_id, + ) + + DataCondition.objects.create( + comparison={ + "sensitivity": AnomalyDetectionSensitivity.LOW, + "seasonality": AnomalyDetectionSeasonality.AUTO, + "threshold_type": AnomalyDetectionThresholdType.ABOVE, + }, + condition_result=DetectorPriorityLevel.HIGH, + type=Condition.ANOMALY_DETECTION, + condition_group=condition_group, + ) + + detector = Detector.objects.create( + project=project, + name="High Error Count (Default)", + description="Automatically monitors for anomalous spikes in error count", + workflow_condition_group=condition_group, + type=MetricIssue.slug, + config={ + "detection_type": AlertRuleDetectionType.DYNAMIC.value, + "comparison_delta": None, + }, + owner_team_id=owner_team_id, + enabled=enabled, + ) + + snuba_query = create_snuba_query( + query_type=SnubaQuery.Type.ERROR, + dataset=Dataset.Events, + query="", + aggregate="count()", + time_window=timedelta(minutes=15), + resolution=timedelta(minutes=15), + environment=None, + event_types=[SnubaQueryEventType.EventType.ERROR], + ) + + query_subscription = create_snuba_subscription( + project=project, + subscription_type=INCIDENTS_SNUBA_SUBSCRIPTION_TYPE, + snuba_query=snuba_query, + ) + + data_source = DataSource.objects.create( + organization_id=project.organization_id, + source_id=str(query_subscription.id), + type=DATA_SOURCE_SNUBA_QUERY_SUBSCRIPTION, + ) + + DataSourceDetector.objects.create( + data_source=data_source, + detector=detector, + ) + except Exception: + logger.exception( + "create_default_anomaly_detector.create_models_failed", + extra={"project_id": project.id, "organization_id": project.organization_id}, + ) + raise + + try: + send_new_detector_data(detector) + except Exception: + logger.exception( + "create_default_anomaly_detector.send_to_seer_failed", + extra={"project_id": project.id, "organization_id": project.organization_id}, + ) + raise + + return detector + except UnableToAcquireLock: + raise UnableToAcquireLockApiError + + +def ensure_performance_detectors(project: Project) -> dict[str, Detector]: + if not features.has("projects:workflow-engine-performance-detectors", project): + return {} + + disabled_platforms_map = get_disabled_platforms_by_detector_type() + + detectors = {} + for mapping in PERFORMANCE_DETECTOR_CONFIG_MAPPINGS.values(): + detector_type = mapping.wfe_detector_type + + # Determine initial enabled state based on platform and default settings + disabled_platforms = disabled_platforms_map.get(detector_type, frozenset()) + default_enabled = DEFAULT_PROJECT_PERFORMANCE_DETECTION_SETTINGS[ + mapping.detection_enabled_key + ] + enabled = (project.platform not in disabled_platforms) and default_enabled + + detectors[detector_type] = _ensure_detector(project, detector_type, default_enabled=enabled) + + return detectors + + +def ensure_default_detectors(project: Project) -> dict[str, Detector]: + detectors: dict[str, Detector] = {} + detectors[ErrorGroupType.slug] = _ensure_detector(project, ErrorGroupType.slug) + detectors[IssueStreamGroupType.slug] = _ensure_detector(project, IssueStreamGroupType.slug) + detectors.update(ensure_performance_detectors(project)) + return detectors diff --git a/src/sentry/workflow_engine/processors/detector.py b/src/sentry/workflow_engine/processors/detector.py index 3e89c592b311..fcd67574b72e 100644 --- a/src/sentry/workflow_engine/processors/detector.py +++ b/src/sentry/workflow_engine/processors/detector.py @@ -1,300 +1,35 @@ from __future__ import annotations import logging -from collections.abc import Mapping from dataclasses import dataclass -from datetime import timedelta -from functools import cache import sentry_sdk -from django.db import router, transaction -from rest_framework import status from sentry import features, options -from sentry.api.exceptions import SentryAPIException from sentry.grouping.grouptype import ErrorGroupType from sentry.incidents.grouptype import MetricIssue -from sentry.incidents.models.alert_rule import AlertRuleDetectionType -from sentry.incidents.utils.constants import INCIDENTS_SNUBA_SUBSCRIPTION_TYPE -from sentry.incidents.utils.types import DATA_SOURCE_SNUBA_QUERY_SUBSCRIPTION -from sentry.issue_detection.performance_detection import PERFORMANCE_DETECTOR_CONFIG_MAPPINGS -from sentry.issues import grouptype from sentry.issues.issue_occurrence import IssueOccurrence from sentry.issues.producer import PayloadType, produce_occurrence_to_kafka -from sentry.locks import locks from sentry.models.activity import Activity from sentry.models.group import Group -from sentry.models.project import Project -from sentry.projectoptions.defaults import DEFAULT_PROJECT_PERFORMANCE_DETECTION_SETTINGS -from sentry.seer.anomaly_detection.store_data_workflow_engine import send_new_detector_data -from sentry.seer.anomaly_detection.types import ( - AnomalyDetectionSeasonality, - AnomalyDetectionSensitivity, - AnomalyDetectionThresholdType, -) from sentry.services.eventstore.models import GroupEvent -from sentry.snuba.dataset import Dataset -from sentry.snuba.models import SnubaQuery, SnubaQueryEventType -from sentry.snuba.subscriptions import create_snuba_query, create_snuba_subscription from sentry.utils import metrics -from sentry.utils.locking import UnableToAcquireLock -from sentry.workflow_engine.models import DataPacket, DataSource, Detector -from sentry.workflow_engine.models.data_condition import Condition, DataCondition -from sentry.workflow_engine.models.data_condition_group import DataConditionGroup -from sentry.workflow_engine.models.data_source_detector import DataSourceDetector + +# TODO - remove this import once getsentry can be updated +from sentry.workflow_engine.defaults.detectors import ( + ensure_default_detectors as ensure_default_detectors, +) +from sentry.workflow_engine.models import DataPacket, Detector from sentry.workflow_engine.models.detector_group import DetectorGroup from sentry.workflow_engine.types import ( - ERROR_DETECTOR_NAME, - ISSUE_STREAM_DETECTOR_NAME, DetectorEvaluationResult, DetectorGroupKey, - DetectorPriorityLevel, WorkflowEventData, ) from sentry.workflow_engine.typings.grouptype import IssueStreamGroupType logger = logging.getLogger(__name__) -VALID_DEFAULT_DETECTOR_TYPES = [ - ErrorGroupType.slug, - IssueStreamGroupType.slug, - *[m.wfe_detector_type for m in PERFORMANCE_DETECTOR_CONFIG_MAPPINGS.values()], -] - - -@cache -def get_disabled_platforms_by_detector_type() -> Mapping[str, frozenset[str]]: - """ - Map WFE detector types to platforms where they should be disabled by default. - Derives from DEFAULT_DETECTOR_DISABLING_CONFIGS using the detection_enabled_key. - """ - from sentry.issue_detection.detectors.disable_detectors import ( - DEFAULT_DETECTOR_DISABLING_CONFIGS, - ) - - disabled_by_detector_type: dict[str, frozenset[str]] = {} - - for disable_config in DEFAULT_DETECTOR_DISABLING_CONFIGS: - detector_option_key = disable_config["detector_project_option"] - languages_to_disable = disable_config["languages_to_disable"] - - # Find matching WFE detector via detection_enabled_key - for mapping in PERFORMANCE_DETECTOR_CONFIG_MAPPINGS.values(): - if mapping.detection_enabled_key == detector_option_key: - disabled_by_detector_type[mapping.wfe_detector_type] = frozenset( - languages_to_disable - ) - break - - return disabled_by_detector_type - - -class UnableToAcquireLockApiError(SentryAPIException): - status_code = status.HTTP_400_BAD_REQUEST - code = "unable_to_acquire_lock" - message = "Unable to acquire lock for issue alert migration." - - -def _ensure_detector(project: Project, type: str, default_enabled: bool = True) -> Detector: - """ - Ensure that a detector of a given type exists for a project. - If the Detector doesn't already exist, we try to acquire a lock to avoid double-creating, - and UnableToAcquireLockApiError if that fails. - """ - group_type = grouptype.registry.get_by_slug(type) - if not group_type: - raise ValueError(f"Group type {type} not registered") - slug = group_type.slug - if slug not in VALID_DEFAULT_DETECTOR_TYPES: - raise ValueError(f"Invalid default detector type: {slug}") - - # If it already exists, life is simple and we can return immediately. - # If there happen to be duplicates, we prefer the oldest. - existing = Detector.objects.filter(type=slug, project=project).order_by("id").first() - if existing: - return existing - - # If we may need to create it, we acquire a lock to avoid double-creating. - # There isn't a unique constraint on the detector, so we can't rely on get_or_create - # to avoid duplicates. - # However, by only locking during the one-time creation, the window for a race condition is small. - lock = locks.get( - f"workflow-engine-project-{slug}-detector:{project.id}", - duration=2, - name=f"workflow_engine_default_{slug}_detector", - ) - try: - with ( - # Creation should be fast, so it's worth blocking a little rather - # than failing a request. - lock.blocking_acquire(initial_delay=0.1, timeout=3), - transaction.atomic(router.db_for_write(Detector)), - ): - detector, _ = Detector.objects.get_or_create( - type=slug, - project=project, - defaults={ - "config": {}, - "name": ( - ERROR_DETECTOR_NAME - if slug == ErrorGroupType.slug - else ISSUE_STREAM_DETECTOR_NAME - if slug == IssueStreamGroupType.slug - else group_type.description - ), - "enabled": default_enabled, - }, - ) - return detector - except UnableToAcquireLock: - raise UnableToAcquireLockApiError - - -def ensure_default_anomaly_detector( - project: Project, owner_team_id: int | None = None, enabled: bool = True -) -> Detector | None: - """ - Ensure that a default anomaly detection metric monitor exists for a project. - If the Detector doesn't already exist, we try to acquire a lock to avoid double-creating. - """ - # If it already exists, return immediately. Prefer the oldest if duplicates exist. - existing = ( - Detector.objects.filter(type=MetricIssue.slug, project=project).order_by("id").first() - ) - if existing: - logger.info( - "create_default_anomaly_detector.already_exists", - extra={"project_id": project.id, "detector_id": existing.id}, - ) - return existing - - lock = locks.get( - f"workflow-engine-project-{MetricIssue.slug}-detector:{project.id}", - duration=2, - name=f"workflow_engine_default_{MetricIssue.slug}_detector", - ) - try: - with ( - lock.blocking_acquire(initial_delay=0.1, timeout=3), - transaction.atomic(router.db_for_write(Detector)), - ): - # Double-check after acquiring lock in case another process created it - existing = ( - Detector.objects.filter(type=MetricIssue.slug, project=project) - .order_by("id") - .first() - ) - if existing: - return existing - - try: - condition_group = DataConditionGroup.objects.create( - logic_type=DataConditionGroup.Type.ANY, - organization_id=project.organization_id, - ) - - DataCondition.objects.create( - comparison={ - "sensitivity": AnomalyDetectionSensitivity.LOW, - "seasonality": AnomalyDetectionSeasonality.AUTO, - "threshold_type": AnomalyDetectionThresholdType.ABOVE, - }, - condition_result=DetectorPriorityLevel.HIGH, - type=Condition.ANOMALY_DETECTION, - condition_group=condition_group, - ) - - detector = Detector.objects.create( - project=project, - name="High Error Count (Default)", - description="Automatically monitors for anomalous spikes in error count", - workflow_condition_group=condition_group, - type=MetricIssue.slug, - config={ - "detection_type": AlertRuleDetectionType.DYNAMIC.value, - "comparison_delta": None, - }, - owner_team_id=owner_team_id, - enabled=enabled, - ) - - snuba_query = create_snuba_query( - query_type=SnubaQuery.Type.ERROR, - dataset=Dataset.Events, - query="", - aggregate="count()", - time_window=timedelta(minutes=15), - resolution=timedelta(minutes=15), - environment=None, - event_types=[SnubaQueryEventType.EventType.ERROR], - ) - - query_subscription = create_snuba_subscription( - project=project, - subscription_type=INCIDENTS_SNUBA_SUBSCRIPTION_TYPE, - snuba_query=snuba_query, - ) - - data_source = DataSource.objects.create( - organization_id=project.organization_id, - source_id=str(query_subscription.id), - type=DATA_SOURCE_SNUBA_QUERY_SUBSCRIPTION, - ) - - DataSourceDetector.objects.create( - data_source=data_source, - detector=detector, - ) - except Exception: - logger.exception( - "create_default_anomaly_detector.create_models_failed", - extra={"project_id": project.id, "organization_id": project.organization_id}, - ) - raise - - try: - send_new_detector_data(detector) - except Exception: - logger.exception( - "create_default_anomaly_detector.send_to_seer_failed", - extra={"project_id": project.id, "organization_id": project.organization_id}, - ) - raise - - return detector - except UnableToAcquireLock: - raise UnableToAcquireLockApiError - - -def ensure_performance_detectors(project: Project) -> dict[str, Detector]: - if not features.has("projects:workflow-engine-performance-detectors", project): - return {} - - disabled_platforms_map = get_disabled_platforms_by_detector_type() - - detectors = {} - for mapping in PERFORMANCE_DETECTOR_CONFIG_MAPPINGS.values(): - detector_type = mapping.wfe_detector_type - - # Determine initial enabled state based on platform and default settings - disabled_platforms = disabled_platforms_map.get(detector_type, frozenset()) - default_enabled = DEFAULT_PROJECT_PERFORMANCE_DETECTION_SETTINGS[ - mapping.detection_enabled_key - ] - enabled = (project.platform not in disabled_platforms) and default_enabled - - detectors[detector_type] = _ensure_detector(project, detector_type, default_enabled=enabled) - - return detectors - - -def ensure_default_detectors(project: Project) -> dict[str, Detector]: - detectors: dict[str, Detector] = {} - detectors[ErrorGroupType.slug] = _ensure_detector(project, ErrorGroupType.slug) - detectors[IssueStreamGroupType.slug] = _ensure_detector(project, IssueStreamGroupType.slug) - detectors.update(ensure_performance_detectors(project)) - return detectors - @dataclass(frozen=True) class EventDetectors: @@ -328,6 +63,7 @@ def detectors(self) -> set[Detector]: return {d for d in [self.issue_stream_detector, self.event_detector] if d is not None} +# TODO - Delete this once the issue stream is fully rolled out. def _is_issue_stream_detector_enabled(event_data: WorkflowEventData) -> bool: """ Check if the issue stream detector should be enabled for this event's group type. @@ -550,6 +286,7 @@ def process_detectors[T]( return results +# TODO - move to another file / location def associate_new_group_with_detector(group: Group, detector_id: int | None = None) -> bool: """ Associate a new Group with it's Detector in the database. @@ -627,6 +364,7 @@ def associate_new_group_with_detector(group: Group, detector_id: int | None = No return True +# TODO - move to another file / location def ensure_association_with_detector(group: Group, detector_id: int | None = None) -> bool: """ Ensure a Group has a DetectorGroup association, creating it if missing. diff --git a/src/sentry/workflow_engine/receivers/__init__.py b/src/sentry/workflow_engine/receivers/__init__.py index fc1f322310e9..0434ce88dd83 100644 --- a/src/sentry/workflow_engine/receivers/__init__.py +++ b/src/sentry/workflow_engine/receivers/__init__.py @@ -5,5 +5,6 @@ from .data_source_detector import * # NOQA from .detector import * # NOQA from .detector_workflow import * # NOQA +from .project_detectors import * # noqa: F401,F403 from .workflow import * # NOQA from .workflow_data_condition_group import * # NOQA diff --git a/src/sentry/receivers/project_detectors.py b/src/sentry/workflow_engine/receivers/project_detectors.py similarity index 71% rename from src/sentry/receivers/project_detectors.py rename to src/sentry/workflow_engine/receivers/project_detectors.py index 0725399df098..3618abe5c429 100644 --- a/src/sentry/receivers/project_detectors.py +++ b/src/sentry/workflow_engine/receivers/project_detectors.py @@ -1,5 +1,6 @@ import logging from contextlib import contextmanager +from typing import Any, Iterator import sentry_sdk from django.db.models.signals import post_save @@ -7,7 +8,8 @@ from sentry import features from sentry.models.project import Project from sentry.signals import project_created -from sentry.workflow_engine.processors.detector import ( +from sentry.users.models.user import User +from sentry.workflow_engine.defaults.detectors import ( UnableToAcquireLockApiError, ensure_default_anomaly_detector, ensure_default_detectors, @@ -17,7 +19,7 @@ @contextmanager -def disable_default_detector_creation(): +def disable_default_detector_creation() -> Iterator[None]: """ Context manager that temporarily disconnects the signal handlers that create default detectors, preventing them from being created when a project is saved. @@ -46,7 +48,11 @@ def disable_default_detector_creation(): ) -def create_project_detectors(instance, created, **kwargs): +def create_project_detectors( + instance: Project, + created: bool, + **kwargs: Any, +) -> None: if created: try: ensure_default_detectors(instance) @@ -54,7 +60,11 @@ def create_project_detectors(instance, created, **kwargs): sentry_sdk.capture_exception(e) -def create_default_anomaly_detector(project: Project, user=None, user_id=None, **kwargs): +def create_default_anomaly_detector( + project: Project, + user: User | None = None, + **kwargs: Any, +) -> None: """ Creates default anomaly detector when project is created, with the team as owner. This listens to project_created signal which provides user information. @@ -76,28 +86,43 @@ def create_default_anomaly_detector(project: Project, user=None, user_id=None, * "organizations:anomaly-detection-alerts", project.organization, actor=user ) detector = ensure_default_anomaly_detector( - project, owner_team_id=owner_team.id if owner_team else None, enabled=enabled + project, + owner_team_id=owner_team.id if owner_team else None, + enabled=enabled, ) if detector: logger.info( "create_default_anomaly_detector.created", - extra={"project_id": project.id, "detector_id": detector.id, "enabled": enabled}, + extra={ + "project_id": project.id, + "detector_id": detector.id, + "enabled": enabled, + }, ) except UnableToAcquireLockApiError as e: logger.warning( "create_default_anomaly_detector.lock_failed", - extra={"project_id": project.id, "organization_id": project.organization_id}, + extra={ + "project_id": project.id, + "organization_id": project.organization_id, + }, ) sentry_sdk.capture_exception(e) except Exception: logger.exception( "create_default_anomaly_detector.failed", - extra={"project_id": project.id, "organization_id": project.organization_id}, + extra={ + "project_id": project.id, + "organization_id": project.organization_id, + }, ) post_save.connect( - create_project_detectors, sender=Project, dispatch_uid="create_project_detectors", weak=False + create_project_detectors, + sender=Project, + dispatch_uid="create_project_detectors", + weak=False, ) project_created.connect( create_default_anomaly_detector, diff --git a/src/sentry/workflow_engine/tasks/__init__.py b/src/sentry/workflow_engine/tasks/__init__.py index a7a1abaaff65..f227bc4a0860 100644 --- a/src/sentry/workflow_engine/tasks/__init__.py +++ b/src/sentry/workflow_engine/tasks/__init__.py @@ -2,9 +2,9 @@ "process_delayed_workflows", "process_workflow_activity", "process_workflows_event", - "prune_old_fire_history", + "prune_old_open_period_activity", ] -from .cleanup import prune_old_fire_history +from .cleanup import prune_old_open_period_activity from .delayed_workflows import process_delayed_workflows from .workflows import process_workflow_activity, process_workflows_event diff --git a/src/sentry/workflow_engine/tasks/cleanup.py b/src/sentry/workflow_engine/tasks/cleanup.py index 437a4b532c59..3863ec6a9503 100644 --- a/src/sentry/workflow_engine/tasks/cleanup.py +++ b/src/sentry/workflow_engine/tasks/cleanup.py @@ -15,28 +15,30 @@ logger = logging.getLogger(__name__) -FIRE_HISTORY_RETENTION_DAYS = 90 +OPEN_PERIOD_ACTIVITY_RETENTION_DAYS = 90 @instrumented_task( - name="sentry.workflow_engine.tasks.cleanup.prune_old_fire_history", + name="sentry.workflow_engine.tasks.cleanup.prune_old_open_period_activity", namespace=workflow_engine_tasks, processing_deadline_duration=15, silo_mode=SiloMode.CELL, ) -def prune_old_fire_history() -> None: - from sentry.workflow_engine.models import WorkflowFireHistory +def prune_old_open_period_activity() -> None: + from sentry.models.groupopenperiodactivity import GroupOpenPeriodActivity - time_limit: float = options.get("workflow_engine.fire_history_cleanup.time_limit_seconds") - batch_size: int = options.get("workflow_engine.fire_history_cleanup.batch_size") + time_limit: float = options.get( + "workflow_engine.open_period_activity_cleanup.time_limit_seconds" + ) + batch_size: int = options.get("workflow_engine.open_period_activity_cleanup.batch_size") - cutoff = timezone.now() - timedelta(days=FIRE_HISTORY_RETENTION_DAYS) + cutoff = timezone.now() - timedelta(days=OPEN_PERIOD_ACTIVITY_RETENTION_DAYS) start = time.time() batches_deleted = 0 while (time.time() - start) < time_limit: has_more = bulk_delete_objects( - WorkflowFireHistory, + GroupOpenPeriodActivity, limit=batch_size, logger=logger, date_added__lte=cutoff, @@ -46,7 +48,7 @@ def prune_old_fire_history() -> None: batches_deleted += 1 metrics.incr( - "workflow_engine.tasks.prune_old_fire_history.batches_deleted", + "workflow_engine.tasks.prune_old_open_period_activity.batches_deleted", amount=batches_deleted, sample_rate=1.0, ) diff --git a/static/app/components/commandPalette/useGlobalCommandPaletteActions.tsx b/static/app/components/commandPalette/useGlobalCommandPaletteActions.tsx index 838375531c0c..74a2e864ff3b 100644 --- a/static/app/components/commandPalette/useGlobalCommandPaletteActions.tsx +++ b/static/app/components/commandPalette/useGlobalCommandPaletteActions.tsx @@ -6,6 +6,7 @@ import {ProjectAvatar} from '@sentry/scraps/avatar'; import {addLoadingMessage, addSuccessMessage} from 'sentry/actionCreators/indicator'; import {openInviteMembersModal} from 'sentry/actionCreators/modal'; +import {openSudo} from 'sentry/actionCreators/sudoModal'; import {useCommandPaletteActionsRegister} from 'sentry/components/commandPalette/context'; import type { CMDKQueryOptions, @@ -350,6 +351,7 @@ export function useGlobalCommandPaletteActions() { label: t('Open _admin'), icon: , }, + keywords: [t('superuser')], onAction: () => window.open('/_admin/', '_blank', 'noreferrer'), }, { @@ -357,13 +359,30 @@ export function useGlobalCommandPaletteActions() { label: t('Open %s in _admin', organization.name), icon: , }, + keywords: [t('superuser')], onAction: () => window.open( - `/_admin/organizations/${organization.slug}/`, + `/_admin/customers/${organization.slug}/`, '_blank', 'noreferrer' ), }, + ...(isActiveSuperuser() + ? [] + : [ + { + display: { + label: t('Open Superuser Modal'), + icon: , + }, + keywords: [t('superuser')], + onAction: () => + openSudo({ + isSuperuser: true, + needsReload: true, + }), + }, + ]), ...(isActiveSuperuser() ? [ { @@ -371,6 +390,7 @@ export function useGlobalCommandPaletteActions() { label: t('Exit Superuser'), icon: , }, + keywords: [t('superuser')], onAction: () => exitSuperuser(), }, ] diff --git a/static/app/components/core/textarea/textarea.tsx b/static/app/components/core/textarea/textarea.tsx index 7da8b105989f..32c621500af0 100644 --- a/static/app/components/core/textarea/textarea.tsx +++ b/static/app/components/core/textarea/textarea.tsx @@ -34,7 +34,7 @@ function TextAreaControl({ ...p }: TextAreaProps) { return autosize ? ( - + ) : (