feat: ensure the instrumentation pattern at compile/CI time

.github/workflows/ci.yaml

@@ -1352,6 +1352,9 @@ jobs:
       - name: Check workspace deps
         run: cargo xtask check-workspace-deps
 
+      - name: Check metric docs
+        run: cargo xtask check-metric-docs
+
       - name: Check licenses
         run: cargo make --no-workspace check-licenses
 

Makefile.toml

@@ -579,6 +579,7 @@ dependencies = [
   "carbide-lints",
   "check-format-nightly",
   "check-workspace-deps",
+  "check-metric-docs",
   "check-licenses",
   "check-bans",
 ]
@@ -818,6 +819,11 @@ script = [
   "cargo --quiet xtask check-workspace-deps || (RC=$?; echo 'To fix automatically, run `cargo xtask check-workspace-deps --fix`'; exit $RC)",
 ]
 
+[tasks.check-metric-docs]
+workspace = false
+description = "Fail if a #[derive(Event)] counter/histogram is missing a row in docs/observability/core_metrics.md"
+script = ["cargo --quiet xtask check-metric-docs"]
+
 [tasks.check-isolated-package-builds]
 workspace = false
 description = "Check that workspace packages build independently with default features"

STYLE_GUIDE.md

@@ -157,6 +157,29 @@ Networking technologies should be integrated using the workflows described in [N
 When designing metrics, be careful with cardinality. Do not attach highly unique labels that explode time-series
 count, like per-machine or per-instance attributes.
 
+## Instrumentation
+
+A significant event -- a count, a rate, or a duration -- is declared with the instrumentation framework
+(`carbide-instrument`) rather than by hand-rolling an OpenTelemetry instrument. One `#[derive(Event)]` declaration
+produces a structured log line, a Prometheus metric, or both from a single `emit()`, with metric cardinality bounded by
+the type system. See [Instrumentation](docs/observability/instrumentation.md) for the full model and when to use it.
+
+The canonical pattern, checked at compile time by the derive:
+
+- **`carbide_` prefix** on every metric name. The name in the attribute is the exposed name, verbatim, so a dashboard
+  greps straight back to the source line.
+- **`_total` suffix for counters** (never a doubled `_total_total` -- the Prometheus exporter appends the suffix), and a
+  **unit suffix for histograms** (`_seconds`, `_milliseconds`, `_microseconds`, `_bytes`).
+- **A counter's `describe` opens with "Number of ..."**. That HELP text is the row the
+  [core metrics catalogue](docs/observability/core_metrics.md) records, and every framework counter and histogram --
+  apart from a `name_unchecked` one, whose exposed name is an OTel-sanitized transform -- must appear there, enforced by
+  `cargo xtask check-metric-docs`.
+- **Bounded `#[label]` fields; high-cardinality detail in `#[context]`**. Labels come from `LabelValue` enums; machine
+  IDs, IPs, and error text stay on the log line only.
+
+`name_unchecked` and `describe_unchecked` are the greppable escape hatches for grandfathered metrics; new metrics use
+the standard form.
+
 ## Logging
 
 All services should emit logs in "logfmt" syntax. This structured logging format allows administrators to efficiently

crates/instrument-macros/src/lib.rs

@@ -93,7 +93,10 @@ fn expand_label_value(input: DeriveInput) -> syn::Result<TokenStream> {
 /// (enum via `LabelValue`; goes to both the log line and the metric),
 /// `#[context]` (any `Display`; log-only), or `#[observation]` (the histogram
 /// value). The metric name is validated at compile time: `carbide_` prefix,
-/// `_total` for counters, a unit suffix for histograms.
+/// `_total` for counters (never a doubled `_total_total`), a unit suffix for
+/// histograms. A counter's `describe` is checked too -- present and opening
+/// with "Number of ..." -- with `describe_unchecked` as the escape hatch for
+/// grandfathered text, mirroring `name_unchecked` for names.
 #[proc_macro_derive(Event, attributes(event, label, context, observation))]
 pub fn derive_event(input: TokenStream) -> TokenStream {
     let input = syn::parse_macro_input!(input as DeriveInput);
@@ -130,6 +133,7 @@ struct EventArgs {
     log: LogSpec,
     metric: MetricSpec,
     name_unchecked: bool,
+    describe_unchecked: bool,
 }
 
 fn parse_event_args(input: &DeriveInput) -> syn::Result<EventArgs> {
@@ -142,6 +146,7 @@ fn parse_event_args(input: &DeriveInput) -> syn::Result<EventArgs> {
         log: LogSpec::Info,
         metric: MetricSpec::None,
         name_unchecked: false,
+        describe_unchecked: false,
     };
     let mut saw_attr = false;
 
@@ -163,6 +168,8 @@ fn parse_event_args(input: &DeriveInput) -> syn::Result<EventArgs> {
                 args.unit = Some(meta.value()?.parse()?);
             } else if meta.path.is_ident("name_unchecked") {
                 args.name_unchecked = true;
+            } else if meta.path.is_ident("describe_unchecked") {
+                args.describe_unchecked = true;
             } else if meta.path.is_ident("log") {
                 let ident: Ident = meta.value()?.parse()?;
                 args.log = match ident.to_string().as_str() {
@@ -195,7 +202,7 @@ fn parse_event_args(input: &DeriveInput) -> syn::Result<EventArgs> {
             } else {
                 return Err(meta.error(
                     "unknown `event` key; expected name, component, message, describe, log, \
-                     metric, unit, or name_unchecked",
+                     metric, unit, name_unchecked, or describe_unchecked",
                 ));
             }
             Ok(())
@@ -282,11 +289,28 @@ fn expand_event(input: DeriveInput) -> syn::Result<TokenStream> {
             ));
         }
         match args.metric {
-            MetricSpec::Counter if !name_value.ends_with("_total") => {
-                return Err(syn::Error::new_spanned(
-                    name,
-                    "counter names end in `_total` (Prometheus convention)",
-                ));
+            MetricSpec::Counter => {
+                if !name_value.ends_with("_total") {
+                    return Err(syn::Error::new_spanned(
+                        name,
+                        "counter names end in `_total` (Prometheus convention)",
+                    ));
+                }
+                // The OpenTelemetry instrument name must not carry `_total`
+                // itself: the Prometheus exporter appends it, so a name that
+                // still ends in `_total` after one is stripped ships a doubled
+                // `_total_total` series (the #3431 footgun).
+                if name_value
+                    .strip_suffix("_total")
+                    .is_some_and(|base| base.ends_with("_total"))
+                {
+                    return Err(syn::Error::new_spanned(
+                        name,
+                        "counter name ends in `_total_total`: the Prometheus exporter appends the \
+                         `_total` suffix, so the instrument name must carry only one. Drop a \
+                         `_total` (use name_unchecked only to keep a grandfathered doubled name)",
+                    ));
+                }
             }
             MetricSpec::Histogram if histogram_unit.is_none() => {
                 return Err(syn::Error::new_spanned(
@@ -322,6 +346,32 @@ fn expand_event(input: DeriveInput) -> syn::Result<TokenStream> {
              metric = none",
         ));
     }
+    // A counter's `describe` is its Prometheus HELP text and the row the
+    // `core_metrics.md` catalogue records, so a counter must document itself,
+    // and the tech-writer house rule is that the text opens with "Number of ".
+    // `describe_unchecked` is the escape hatch for a grandfathered describe --
+    // legacy phrasings, or the "Total number of ..." on a name_unchecked
+    // counter -- mirroring `name_unchecked` for names.
+    if args.metric == MetricSpec::Counter && !args.describe_unchecked {
+        match &args.describe {
+            None => {
+                return Err(syn::Error::new_spanned(
+                    &input.ident,
+                    "a counter must document itself: add describe = \"Number of ...\" (its \
+                     Prometheus HELP text, and the core_metrics.md catalogue row). Use \
+                     describe_unchecked to keep a grandfathered counter's describe",
+                ));
+            }
+            Some(describe) if !describe.value().starts_with("Number of ") => {
+                return Err(syn::Error::new_spanned(
+                    describe,
+                    "a counter's describe opens with \"Number of ...\" (the tech-writer house \
+                     rule). Use describe_unchecked to keep a grandfathered describe",
+                ));
+            }
+            Some(_) => {}
+        }
+    }
     let unit_value: String = match (&args.unit, histogram_unit) {
         (Some(explicit), _) => explicit.value(),
         (None, Some(from_suffix)) => from_suffix.to_string(),

crates/instrument/src/lib.rs

@@ -37,6 +37,7 @@
 //!     component = "component_manager",
 //!     log       = warn,                          // error|warn|info|debug|trace|off
 //!     metric    = counter,                       // counter | histogram | none
+//!     describe  = "Number of power control operations that failed", // counter HELP text
 //!     message   = "power control failed",
 //! )]
 //! struct PowerControlFailed {
@@ -69,7 +70,8 @@
 //!
 //! ```compile_fail
 //! #[derive(carbide_instrument::Event)]
-//! #[event(name = "carbide_demo_total", component = "demo", log = off, metric = counter)]
+//! #[event(name = "carbide_demo_total", component = "demo", log = off, metric = counter,
+//!         describe = "Number of demo events")]
 //! struct Demo {
 //!     #[label]
 //!     machine_id: String, // ERROR: String is not a LabelValue
@@ -116,6 +118,45 @@
 //! #[event(name = "carbide_demo", component = "demo", message = "demo", describe = "demo")]
 //! struct Demo {} // ERROR: `describe` documents a metric; this event has metric = none
 //! ```
+//!
+//! A counter documents itself: `describe` is required and opens with
+//! "Number of ..." (the tech-writer house rule, so the `core_metrics.md`
+//! catalogue reads consistently):
+//!
+//! ```compile_fail
+//! #[derive(carbide_instrument::Event)]
+//! #[event(name = "carbide_demo_total", component = "demo", log = off, metric = counter)]
+//! struct Demo {} // ERROR: a counter must document itself with describe = "Number of ..."
+//! ```
+//!
+//! ```compile_fail
+//! #[derive(carbide_instrument::Event)]
+//! #[event(name = "carbide_demo_total", component = "demo", log = off, metric = counter,
+//!         describe = "Total number of demos")]
+//! struct Demo {} // ERROR: a counter's describe opens with "Number of ..."
+//! ```
+//!
+//! A counter name ends in `_total` (Prometheus convention) but not
+//! `_total_total` -- the framework strips one `_total` before registering and
+//! the exporter appends it back, so a doubled suffix ships a `_total_total`
+//! series:
+//!
+//! ```compile_fail
+//! #[derive(carbide_instrument::Event)]
+//! #[event(name = "carbide_demo_total_total", component = "demo", log = off, metric = counter,
+//!         describe = "Number of demos")]
+//! struct Demo {} // ERROR: counter name ends in `_total_total`
+//! ```
+//!
+//! Both checks have a greppable escape hatch for grandfathered metrics --
+//! `describe_unchecked` for the text, `name_unchecked` for the name:
+//!
+//! ```
+//! #[derive(carbide_instrument::Event)]
+//! #[event(name = "carbide_demo_total_total", component = "demo", log = off, metric = counter,
+//!         name_unchecked, describe = "Total number of demos", describe_unchecked)]
+//! struct Demo {}
+//! ```
 
 use std::time::Duration;
 

crates/instrument/src/testing.rs

@@ -25,7 +25,8 @@
 //!
 //! #[derive(Event)]
 //! #[event(name = "carbide_doc_demo_total", component = "demo",
-//!         log = warn, metric = counter, message = "demo fired")]
+//!         log = warn, metric = counter, message = "demo fired",
+//!         describe = "Number of demo events fired")]
 //! struct Demo {}
 //!
 //! let metrics = MetricsCapture::start();

crates/instrument/tests/matrix.rs

@@ -46,6 +46,7 @@ fn both_sides_from_one_emit() {
         component = "matrix-test",
         log = warn,
         metric = counter,
+        describe_unchecked,
         message = "matrix both fired"
     )]
     struct BothSides {
@@ -92,7 +93,8 @@ fn metric_only_writes_no_log() {
         name = "carbide_test_matrix_quiet_total",
         component = "matrix-test",
         log = off,
-        metric = counter
+        metric = counter,
+        describe_unchecked
     )]
     struct Quiet {
         #[label]
@@ -208,7 +210,8 @@ fn unit_struct_and_declared_knobs() {
         name = "carbide_test_matrix_unit_total",
         component = "matrix-test",
         log = off,
-        metric = counter
+        metric = counter,
+        describe_unchecked
     )]
     struct Tick;
 
@@ -235,6 +238,7 @@ fn per_instance_log_at_override() {
         component = "matrix-test",
         log = dynamic,
         metric = counter,
+        describe_unchecked,
         message = "call finished"
     )]
     struct CallFinished {

crates/xtask/Cargo.toml

@@ -29,6 +29,8 @@ clap = { workspace = true }
 cargo_metadata = { workspace = true }
 similar = { workspace = true }
 version-compare = { workspace = true }
+syn = { workspace = true, features = ["full", "parsing"] }
+proc-macro2 = { workspace = true, features = ["proc-macro"] }
 
 chrono = { workspace = true }
 tracing = { workspace = true }

crates/xtask/src/main.rs

@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 mod isolated_package_builds;
+mod metric_docs;
 mod squash_migrations;
 mod workspace_deps;
 
@@ -33,6 +34,11 @@ enum Xtask {
         about = "Check that each workspace package builds independently with its default features"
     )]
     IsolatedPackageBuilds,
+    #[clap(
+        name = "check-metric-docs",
+        about = "Check that every #[derive(Event)] counter/histogram has a row in docs/observability/core_metrics.md"
+    )]
+    CheckMetricDocs,
     #[clap(
         name = "squash-migrations",
         about = "Create a single squashed migration from all existing migrations in crates/api-db/migrations"
@@ -57,6 +63,7 @@ async fn main() -> eyre::Result<()> {
             workspace_deps::check(fix)?.report_and_exit()
         }
         Xtask::IsolatedPackageBuilds => isolated_package_builds::check()?,
+        Xtask::CheckMetricDocs => metric_docs::check()?,
         Xtask::SquashMigrations(args) => squash_migrations::run(args).await?,
     }
     Ok(())