From ae68e294da46705cb22c7a239ddcd258bedc2a67 Mon Sep 17 00:00:00 2001 From: rosstaco Date: Fri, 24 Jul 2026 07:49:57 +0000 Subject: [PATCH] chore: remove copilot-metrics-lgtm feature from main copilot-metrics-lgtm is not ready for human consumption, so remove it from main to keep the release workflow from publishing it. The feature is preserved on the feat/copilot-metrics-lgtm branch for continued work. - Delete src/copilot-metrics-lgtm and test/copilot-metrics-lgtm - Drop copilot-metrics-lgtm from both CI test matrices in test.yaml Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9a35c641-c703-4ecf-b6a0-b7775b16551f --- .github/workflows/test.yaml | 4 - src/copilot-metrics-lgtm/README.md | 109 - .../dashboards/github-copilot.json | 2322 ----------------- .../devcontainer-feature.json | 57 - src/copilot-metrics-lgtm/docker-compose.yml | 19 - src/copilot-metrics-lgtm/install.sh | 52 - src/copilot-metrics-lgtm/post-start.sh | 111 - test/copilot-metrics-lgtm/debian.sh | 39 - test/copilot-metrics-lgtm/default.sh | 39 - test/copilot-metrics-lgtm/noble.sh | 39 - test/copilot-metrics-lgtm/scenarios.json | 23 - test/copilot-metrics-lgtm/test.sh | 39 - 12 files changed, 2853 deletions(-) delete mode 100644 src/copilot-metrics-lgtm/README.md delete mode 100644 src/copilot-metrics-lgtm/dashboards/github-copilot.json delete mode 100644 src/copilot-metrics-lgtm/devcontainer-feature.json delete mode 100644 src/copilot-metrics-lgtm/docker-compose.yml delete mode 100644 src/copilot-metrics-lgtm/install.sh delete mode 100644 src/copilot-metrics-lgtm/post-start.sh delete mode 100644 test/copilot-metrics-lgtm/debian.sh delete mode 100644 test/copilot-metrics-lgtm/default.sh delete mode 100644 test/copilot-metrics-lgtm/noble.sh delete mode 100644 test/copilot-metrics-lgtm/scenarios.json delete mode 100644 test/copilot-metrics-lgtm/test.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5920988..518e2ef 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,9 +18,6 @@ jobs: - prompty-dumpty - copilot-persistence - gitlab-ci-local - # copilot-metrics-lgtm is intentionally excluded here: it requires Docker - # (docker-in-docker) which the autogenerated test image does not provide. - # It is exercised in the test-scenarios job instead. - atlassian-jira-confluence-cli - package-source-overrides baseImage: @@ -47,7 +44,6 @@ jobs: - prompty-dumpty - copilot-persistence - gitlab-ci-local - - copilot-metrics-lgtm - atlassian-jira-confluence-cli - package-source-overrides steps: diff --git a/src/copilot-metrics-lgtm/README.md b/src/copilot-metrics-lgtm/README.md deleted file mode 100644 index c9886f2..0000000 --- a/src/copilot-metrics-lgtm/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# Copilot Metrics (LGTM) - -Captures OpenTelemetry traces emitted by **GitHub Copilot** (VS Code Copilot -Chat _and_ the Copilot CLI) and visualizes them in a local **Grafana LGTM -stack** (Loki / Grafana / Tempo / Mimir, via the `grafana/otel-lgtm` image). - -## Usage - -Add this Feature to your `devcontainer.json`: - -```json -{ - "features": { - "ghcr.io/rosstaco/devcontainer-features/copilot-metrics-lgtm:1": {} - }, - "forwardPorts": [3000] -} -``` - -Open Grafana at **http://localhost:3000** after the container starts. - -## Prerequisites - -This Feature requires a Docker daemon inside the devcontainer. Add **one** of -the following to your `features` block: - -| Feature | When to use | -|---------|-------------| -| `ghcr.io/devcontainers/features/docker-in-docker` | Default / recommended. Simplest. | -| `ghcr.io/devcontainers/features/docker-outside-of-docker` | When `privileged: true` is not allowed (restricted Codespaces, corporate policy). | - -> **⚠️ `forwardPorts` required** — Features cannot contribute `forwardPorts`. -> Add `"forwardPorts": [3000]` to your `devcontainer.json` to access Grafana -> in your browser. VS Code's auto port forwarding usually catches it, but the -> explicit declaration ensures it. - -## Options - -| Option | Type | Default | Description | -|--------------------|---------|----------------------------|-------------------------------------------| -| `lgtmImage` | string | `grafana/otel-lgtm:latest` | LGTM image tag to use. | -| `installDashboard` | boolean | `true` | POST the bundled dashboard on first start.| -| `dockerMode` | string | `auto` | `auto`, `dind`, or `dood`. | -| `grafanaPort` | string | `3000` | Grafana web UI port (change if 3000 conflicts).| -| `captureContent` | boolean | `true` | Capture full prompt/response content in traces. | - -## Data flow - -``` -VS Code Copilot Chat ─┐ ┌─ Tempo (traces) ─┐ - ├─ OTLP/HTTP :4318 ──► LGTM ┤ ├─► Grafana :3000 -Copilot CLI ─┘ └─ Prometheus etc. ─┘ -``` - -Both exporters target `http://localhost:4318` inside the devcontainer. - -## Privacy notice - -> **This Feature captures full prompt and response content locally when -> `captureContent` is `true` (the default).** - -- **VS Code Copilot Chat** sends `gen_ai.input.messages` and - `gen_ai.output.messages` to the local Tempo instance when - `github.copilot.chat.otel.captureContent` is `true` (the default). -- **Copilot CLI** captures `gen_ai.input.messages` when - `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` is `true`. - -Both are controlled by the `captureContent` option. Set to `false` to capture -only span metadata (timing, token counts) without message bodies. - -All data stays local (inside the devcontainer's LGTM stack). To wipe all -stored data: - -```bash -docker volume rm copilot-lgtm-data -``` - -## Docker modes - -### Docker-in-Docker (default) - -LGTM runs in the devcontainer's inner Docker daemon. `localhost:4318` works -transparently. Requires `privileged: true` (provided by the DinD feature). - -### Docker-outside-of-Docker - -For environments that can't use DinD, the Feature supports DooD. LGTM runs -on the host daemon but joins the devcontainer's network namespace -(`--network=container:$HOSTNAME`), so `localhost:4318` still works. - -Set `"dockerMode": "dood"` explicitly, or leave `"auto"` to detect -automatically. - -## First start - -The first `docker compose up` pulls **~600 MB** (`grafana/otel-lgtm`). -Subsequent starts are fast thanks to the `copilot-lgtm-data` named volume -that persists data across container rebuilds. - -## State reset - -```bash -docker volume ls | grep copilot-lgtm-data -# Then: docker volume rm -``` - -The volume name is `${devcontainerId}-copilot-lgtm-data` — check `docker volume ls` -to find the exact name. This removes all Grafana dashboards (including manual -UI edits), traces, metrics, and logs. diff --git a/src/copilot-metrics-lgtm/dashboards/github-copilot.json b/src/copilot-metrics-lgtm/dashboards/github-copilot.json deleted file mode 100644 index 964b1ab..0000000 --- a/src/copilot-metrics-lgtm/dashboards/github-copilot.json +++ /dev/null @@ -1,2322 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Monitors GitHub Copilot usage and performance via Application Insights OpenTelemetry data. Tracks agent operations, LLM token consumption, model latency (response duration and time-to-first-token), chat sessions, tool calls, feature adoption, and errors. Includes trace drill-down for debugging individual Copilot interactions.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [ - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": false, - "tags": [], - "targetBlank": true, - "title": "Tempo", - "tooltip": "Open Tempo Explore", - "type": "link", - "url": "/explore?left=%7B%22datasource%22:%22tempo%22%7D" - } - ], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "panels": [], - "title": "Summary", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Key performance indicators for agent operations including total operations executed, token consumption (input and output), and average response time across all agents.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total Operations" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Input Tokens" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - }, - { - "id": "unit", - "value": "short" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Output Tokens" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Avg Response Time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - }, - { - "id": "unit", - "value": "s" - }, - { - "id": "decimals", - "value": 2 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && name=~\".*invoke_agent.*\" } | count_over_time()", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && name=~\".*invoke_agent.*\" } | sum_over_time(span.gen_ai.usage.input_tokens)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "C", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && name=~\".*invoke_agent.*\" } | sum_over_time(span.gen_ai.usage.output_tokens)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "D", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && name=~\".*invoke_agent.*\" } | avg_over_time(span:duration)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Agent Summary Statistics", - "type": "stat" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Key metrics across all LLM chat completions: total calls, total tokens consumed, and average time-to-first-token (TTFT).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "LLM Calls" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Chat Sessions" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Avg TTFT" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-orange", - "mode": "fixed" - } - }, - { - "id": "unit", - "value": "ms" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Tool Calls" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 13, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | count_over_time()", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.copilot_chat.chat_session_id != nil } | count_over_time() by (span.copilot_chat.chat_session_id)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "C", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.operation.name = \"execute_tool\" } | count_over_time()", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "D", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.copilot_chat.time_to_first_token > 0 } | avg_over_time(span.copilot_chat.time_to_first_token)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Chat & Tool Summary", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 101, - "panels": [], - "title": "Usage Over Time", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Number of LLM API calls over time, grouped by operation type (chat completions, agent invocations, tool executions).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "maxDataPoints": 60, - "options": { - "legend": { - "calcs": [ - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": true, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.operation.name != nil } | rate() by (span.gen_ai.operation.name)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Operations Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Total token consumption (input + output) over time, broken down by model.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Tokens", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 2, - "maxDataPoints": 60, - "options": { - "legend": { - "calcs": [ - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": true, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | sum_over_time(span.gen_ai.usage.input_tokens) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | sum_over_time(span.gen_ai.usage.output_tokens) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Token Consumption Over Time by Model", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 102, - "panels": [], - "title": "Model Performance", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Breakdown of LLM calls by model. Shows which AI models are being used and how frequently.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 3, - "options": { - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value", - "percent" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "sort": "desc", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | count_over_time() by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Model Usage Distribution", - "type": "piechart", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "exclude": { - "pattern": "^(Time|Value|traceId|span\\..*)$" - } - } - } - ] - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Average and P90 response duration per model. Helps identify which models are slower.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 4, - "options": { - "barRadius": 0.05, - "barWidth": 0.7, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | avg_over_time(span:duration) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | quantile_over_time(span:duration, 0.9) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Response Duration by Model (Avg / P90)", - "type": "barchart", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "exclude": { - "pattern": "^(Time|Value|traceId|span\\..*)$" - } - } - }, - { - "id": "reduce", - "options": { - "reducers": [ - "mean" - ], - "mode": "seriesToRows", - "includeTimeField": false - } - } - ] - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Time to First Token (TTFT) percentiles per model. TTFT measures how quickly the user starts seeing a response stream. Lower is better for user experience.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 5, - "options": { - "barRadius": 0.05, - "barWidth": 0.7, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.copilot_chat.time_to_first_token > 0 } | quantile_over_time(span.copilot_chat.time_to_first_token, 0.5) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.copilot_chat.time_to_first_token > 0 } | quantile_over_time(span.copilot_chat.time_to_first_token, 0.9) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Time to First Token by Model (P50 / P90)", - "type": "barchart", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "exclude": { - "pattern": "^(Time|Value|traceId|span\\..*)$" - } - } - }, - { - "id": "reduce", - "options": { - "reducers": [ - "mean" - ], - "mode": "seriesToRows", - "includeTimeField": false - } - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 103, - "panels": [], - "title": "Latency Trends", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Average LLM response duration over time, split by model.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 6, - "maxDataPoints": 60, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": true, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | avg_over_time(span:duration) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Avg Response Duration Over Time by Model", - "type": "timeseries" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Time to First Token (TTFT) P50 and P90 over time. TTFT is the delay before the user sees the first response token streaming in. Critical for perceived responsiveness.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 7, - "maxDataPoints": 60, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": true, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.copilot_chat.time_to_first_token > 0 } | quantile_over_time(span.copilot_chat.time_to_first_token, 0.5)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.copilot_chat.time_to_first_token > 0 } | quantile_over_time(span.copilot_chat.time_to_first_token, 0.9)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Time to First Token (P50 / P90) Over Time", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 104, - "panels": [], - "title": "Agent & Feature Activity", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Breakdown of Copilot feature/agent usage. Shows which Copilot features (Chat, Edit Agent, Commit Generator, Explore, etc.) are being used most.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 35 - }, - "id": 8, - "options": { - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value", - "percent" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.agent.name != nil } | count_over_time() by (span.gen_ai.agent.name)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Copilot Feature Usage", - "type": "piechart", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "exclude": { - "pattern": "^(Time|Value|traceId|span\\..*)$" - } - } - } - ] - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Top tools invoked by Copilot during agent interactions. Includes file operations, search, MCP tool calls, and terminal commands.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 35 - }, - "id": 9, - "options": { - "barRadius": 0.05, - "barWidth": 0.7, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.operation.name = \"execute_tool\" && span.gen_ai.tool.name != nil } | count_over_time() by (span.gen_ai.tool.name)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Top Tool Calls", - "type": "barchart", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "exclude": { - "pattern": "^(Time|Value|traceId|span\\..*)$" - } - } - }, - { - "id": "reduce", - "options": { - "reducers": [ - "sum" - ], - "mode": "seriesToRows", - "includeTimeField": false - } - } - ] - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Copilot activity over time: chat sessions started, agent conversation turns, and tool calls executed.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 35 - }, - "id": 10, - "maxDataPoints": 60, - "options": { - "legend": { - "calcs": [ - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": true, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.copilot_chat.chat_session_id != nil } | count_over_time() by (span.copilot_chat.chat_session_id)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.operation.name = \"chat\" } | rate()", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "C", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.operation.name = \"execute_tool\" } | rate()", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Sessions, Turns & Tool Calls Over Time", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 105, - "panels": [], - "title": "Token Efficiency & Cost", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Detailed breakdown of token usage per model: total input tokens, output tokens, average tokens per call, and call count.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "TotalInputTokens" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-blues" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "TotalOutputTokens" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-greens" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 14, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": false, - "fields": [], - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "TotalInputTokens" - } - ] - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | count_over_time() by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | sum_over_time(span.gen_ai.usage.input_tokens) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "C", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | sum_over_time(span.gen_ai.usage.output_tokens) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "D", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | avg_over_time(span.gen_ai.usage.input_tokens) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "E", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.response.model != nil } | avg_over_time(span.gen_ai.usage.output_tokens) by (span.gen_ai.response.model)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Token Usage by Model", - "type": "table" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Input vs Output token ratio over time. A high ratio indicates large prompts with relatively small responses, which is typical for code context.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Tokens", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "mean", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": true, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.usage.input_tokens > 0 } | sum_over_time(span.gen_ai.usage.input_tokens)", - "filters": [], - "limit": 20, - "tableType": "spans" - }, - { - "refId": "B", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && span.gen_ai.usage.output_tokens > 0 } | sum_over_time(span.gen_ai.usage.output_tokens)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Input vs Output Tokens Over Time", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 52 - }, - "id": 106, - "panels": [], - "title": "Errors & Reliability", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Exceptions thrown by Copilot Chat over time. Includes canceled operations, MCP server errors, and other failures.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 11, - "maxDataPoints": 60, - "options": { - "legend": { - "calcs": [ - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": true, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && event:name = \"exception\" } | rate() by (event.exception.type)", - "filters": [], - "limit": 20, - "tableType": "spans" - } - ], - "title": "Exceptions Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Detailed exception log showing each error with timestamp, type, and message. Useful for debugging Copilot issues like MCP server failures or canceled requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "ExceptionType" - }, - "properties": [ - { - "id": "custom.width", - "value": 120 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 180 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 16, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": true, - "enablePagination": true, - "fields": [], - "reducer": [ - "count" - ], - "show": true - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "timestamp" - } - ] - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && event:name = \"exception\" } | select(event.exception.type, event.exception.message)", - "limit": 100, - "tableType": "traces", - "filters": [] - } - ], - "title": "Exception Details", - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 107, - "panels": [], - "title": "Select a trace", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Interactive table listing recent Copilot Chat operations. Click any row to drill down into detailed trace information for that specific operation.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "links": [ - { - "title": "Open trace \"${__data.fields.traceID}\"", - "url": "/d/${__dashboard.uid}/${__dashboard.slug}?var-copilotTraceId=${__data.fields.traceID}", - "targetBlank": false - } - ], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "traceDuration" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "startTime" - }, - "properties": [ - { - "id": "custom.width", - "value": 200 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 62 - }, - "id": 108, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "timestamp" - } - ] - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name=~\"${source:regex}\" && name=~\".*invoke_agent.*\" } | select(span.gen_ai.operation.name, span.gen_ai.response.model, span.gen_ai.usage.input_tokens, span.gen_ai.usage.output_tokens, resource.service.name)", - "limit": 100, - "tableType": "traces", - "filters": [] - } - ], - "title": "Recent Operations", - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 109, - "panels": [], - "title": "Trace detail", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "description": "Detailed trace visualization showing the complete execution flow of a selected Copilot Chat operation, including all dependencies, timing, and hierarchical span relationships.", - "gridPos": { - "h": 27, - "w": 24, - "x": 0, - "y": 71 - }, - "id": 110, - "options": { - "spanFilters": { - "criticalPathOnly": false, - "matchesOnly": false, - "serviceNameOperator": "=", - "spanNameOperator": "=", - "tags": [ - { - "id": "copilot-trace-001", - "operator": "=" - } - ] - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "${copilotTraceId}", - "filters": [] - } - ], - "title": "Trace Visualization", - "type": "traces" - } - ], - "preload": false, - "refresh": "30m", - "schemaVersion": 42, - "tags": [ - "ai", - "copilot", - "github-copilot", - "grafana-cloud-otel-lgtm", - "lgtm", - "llm", - "opentelemetry", - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "text": "Tempo", - "value": "tempo" - }, - "label": "Tempo datasource", - "name": "tempo_ds", - "options": [], - "query": "tempo", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": [ - "copilot-chat", - "github-copilot" - ], - "value": [ - "copilot-chat", - "github-copilot" - ] - }, - "description": "Filter by sender: VS Code Copilot Chat (copilot-chat), Copilot CLI (github-copilot), or both", - "includeAll": false, - "label": "Source", - "multi": true, - "name": "source", - "options": [ - { - "selected": true, - "text": "VS Code Copilot", - "value": "copilot-chat" - }, - { - "selected": true, - "text": "Copilot CLI", - "value": "github-copilot" - } - ], - "query": "VS Code Copilot : copilot-chat, Copilot CLI : github-copilot", - "type": "custom" - }, - { - "current": { - "text": "", - "value": "" - }, - "description": "Trace Id of the Copilot Chat operations (set via data link from Recent Operations row)", - "hide": 2, - "label": "Trace Id", - "name": "copilotTraceId", - "options": [], - "query": "", - "type": "textbox" - } - ] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "GitHub Copilot", - "uid": "GitHubCopilot", - "version": null, - "weekStart": "" - }, - "overwrite": true, - "message": "Convert to LGTM stack" -} diff --git a/src/copilot-metrics-lgtm/devcontainer-feature.json b/src/copilot-metrics-lgtm/devcontainer-feature.json deleted file mode 100644 index 77b7c8c..0000000 --- a/src/copilot-metrics-lgtm/devcontainer-feature.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "id": "copilot-metrics-lgtm", - "version": "1.0.0", - "name": "Copilot Metrics (LGTM)", - "description": "Captures GitHub Copilot OTel traces and visualizes them in a local Grafana LGTM stack (Loki / Grafana / Tempo / Mimir).", - "documentationURL": "https://github.com/rosstaco/devcontainer-features/tree/main/src/copilot-metrics-lgtm", - "options": { - "lgtmImage": { - "type": "string", - "default": "grafana/otel-lgtm:latest", - "description": "LGTM image tag to use." - }, - "installDashboard": { - "type": "boolean", - "default": true, - "description": "POST the bundled Grafana dashboard on first start." - }, - "dockerMode": { - "type": "string", - "default": "auto", - "enum": ["auto", "dind", "dood"], - "description": "Docker mode: auto-detect, docker-in-docker, or docker-outside-of-docker." - }, - "grafanaPort": { - "type": "string", - "default": "3000", - "description": "Port for the Grafana web UI (change if 3000 conflicts with your app)." - }, - "captureContent": { - "type": "boolean", - "default": true, - "description": "Capture full prompt/response content in traces. Sets OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT for the CLI and github.copilot.chat.otel.captureContent for Chat." - } - }, - "mounts": [ - { - "source": "${devcontainerId}-copilot-lgtm-data", - "target": "/lgtm-data", - "type": "volume" - } - ], - "customizations": { - "vscode": { - "settings": { - "github.copilot.chat.otel.enabled": true, - "github.copilot.chat.otel.exporterType": "otlp-http", - "github.copilot.chat.otel.otlpEndpoint": "http://localhost:4318", - "github.copilot.chat.otel.captureContent": true - } - } - }, - "postStartCommand": "bash /usr/local/share/copilot-metrics-lgtm/post-start.sh", - "installsAfter": [ - "ghcr.io/devcontainers/features/docker-in-docker", - "ghcr.io/devcontainers/features/docker-outside-of-docker" - ] -} diff --git a/src/copilot-metrics-lgtm/docker-compose.yml b/src/copilot-metrics-lgtm/docker-compose.yml deleted file mode 100644 index f71c527..0000000 --- a/src/copilot-metrics-lgtm/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -services: - lgtm: - image: ${LGTM_IMAGE:-grafana/otel-lgtm:latest} - restart: unless-stopped - # DinD: "host" — binds to devcontainer's localhost directly. - # DooD: "container:" — joins devcontainer's network namespace. - # Either way localhost:4318 and :3000 work from inside the devcontainer. - network_mode: ${LGTM_NETWORK_MODE:-host} - environment: - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" - GF_AUTH_DISABLE_LOGIN_FORM: "true" - GF_AUTH_DISABLE_SIGNOUT_MENU: "true" - GF_AUTH_BASIC_ENABLED: "false" - GF_USERS_ALLOW_SIGN_UP: "false" - GF_SECURITY_ALLOW_EMBEDDING: "true" - GF_SERVER_HTTP_PORT: ${GRAFANA_PORT:-3000} - volumes: - - /lgtm-data:/data diff --git a/src/copilot-metrics-lgtm/install.sh b/src/copilot-metrics-lgtm/install.sh deleted file mode 100644 index 602e1bd..0000000 --- a/src/copilot-metrics-lgtm/install.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Installs the copilot-metrics-lgtm Feature: stages runtime files into -# /usr/local/share/copilot-metrics-lgtm/ and writes a .env with resolved options. -set -euo pipefail - -FEATURE_DIR="/usr/local/share/copilot-metrics-lgtm" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -mkdir -p "${FEATURE_DIR}/dashboards" - -# Copy runtime files -cp "${SCRIPT_DIR}/docker-compose.yml" "${FEATURE_DIR}/" -cp "${SCRIPT_DIR}/post-start.sh" "${FEATURE_DIR}/" -cp "${SCRIPT_DIR}/dashboards/github-copilot.json" "${FEATURE_DIR}/dashboards/" - -# Resolve options (env vars set by devcontainer CLI per Feature spec) -LGTM_IMAGE="${LGTMIMAGE:-grafana/otel-lgtm:latest}" -INSTALL_DASHBOARD="${INSTALLDASHBOARD:-true}" -DOCKER_MODE="${DOCKERMODE:-auto}" -CAPTURE_CONTENT="${CAPTURECONTENT:-true}" - -# Validate grafanaPort -GRAFANA_PORT="${GRAFANAPORT:-3000}" -if [[ ! "${GRAFANA_PORT}" =~ ^[0-9]+$ ]]; then - echo "WARNING: grafanaPort '${GRAFANA_PORT}' is not a number; using 3000" >&2 - GRAFANA_PORT="3000" -fi - -# Basic validation of lgtmImage -if [[ ! "${LGTM_IMAGE}" =~ ^[a-zA-Z0-9._/:@-]+$ ]]; then - echo "WARNING: lgtmImage '${LGTM_IMAGE}' looks invalid; using default" >&2 - LGTM_IMAGE="grafana/otel-lgtm:latest" -fi - -# Write config for post-start.sh and docker-compose variable substitution. -# Values are read back with grep/cut, not sourced, to avoid injection. -cat > "${FEATURE_DIR}/.env" < /etc/profile.d/copilot-metrics-lgtm.sh </dev/null | head -1 | cut -d= -f2-)" - echo "${val:-$2}" -} - -if [ "$(id -u)" -eq 0 ]; then SUDO=""; else SUDO="sudo"; fi - -# --------------------------------------------------------------------------- -# Preflight -# --------------------------------------------------------------------------- - -if ! command -v docker &>/dev/null; then - echo "copilot-metrics-lgtm: ERROR — 'docker' not found." >&2 - echo " Install ghcr.io/devcontainers/features/docker-in-docker" >&2 - echo " or ghcr.io/devcontainers/features/docker-outside-of-docker." >&2 - exit 1 -fi - -if ! docker compose version &>/dev/null; then - echo "copilot-metrics-lgtm: ERROR — 'docker compose' not available." >&2 - exit 1 -fi - -# --------------------------------------------------------------------------- -# Read options -# --------------------------------------------------------------------------- - -DOCKER_MODE="$(get_config DOCKER_MODE auto)" -INSTALL_DASHBOARD="$(get_config INSTALL_DASHBOARD true)" -GRAFANA_PORT="$(get_config GRAFANA_PORT 3000)" - -# --------------------------------------------------------------------------- -# Auto-detect Docker mode -# --------------------------------------------------------------------------- - -if [[ "${DOCKER_MODE}" == "auto" ]]; then - if docker inspect "$HOSTNAME" >/dev/null 2>&1; then - DOCKER_MODE=dood - else - DOCKER_MODE=dind - fi - echo "copilot-metrics-lgtm: auto-detected docker mode: ${DOCKER_MODE}" -fi - -# --------------------------------------------------------------------------- -# Prepare data volume mount point -# --------------------------------------------------------------------------- - -$SUDO mkdir -p /lgtm-data -$SUDO chmod 0777 /lgtm-data - -# --------------------------------------------------------------------------- -# Start LGTM stack -# --------------------------------------------------------------------------- - -cd "${FEATURE_DIR}" - -if [[ "${DOCKER_MODE}" == "dood" ]]; then - export LGTM_NETWORK_MODE="container:${HOSTNAME}" -fi - -docker compose up -d - -# --------------------------------------------------------------------------- -# Wait for Grafana -# --------------------------------------------------------------------------- - -GRAFANA_URL="http://localhost:${GRAFANA_PORT}" -echo "copilot-metrics-lgtm: waiting for Grafana..." -for i in $(seq 1 30); do - if curl -fsS "${GRAFANA_URL}/api/health" >/dev/null 2>&1; then - echo "copilot-metrics-lgtm: Grafana is ready" - break - fi - sleep 2 -done - -# --------------------------------------------------------------------------- -# Provision dashboard -# --------------------------------------------------------------------------- - -if [[ "${INSTALL_DASHBOARD}" == "true" ]]; then - DASHBOARD_FILE="${FEATURE_DIR}/dashboards/github-copilot.json" - DASHBOARD_UID="GitHubCopilot" - if [[ -f "${DASHBOARD_FILE}" ]]; then - if curl -fsS "${GRAFANA_URL}/api/dashboards/uid/${DASHBOARD_UID}" >/dev/null 2>&1; then - echo "copilot-metrics-lgtm: dashboard ${DASHBOARD_UID} already present; leaving as-is." - else - echo "copilot-metrics-lgtm: provisioning dashboard..." - curl -fsS -X POST -H "Content-Type: application/json" \ - --data @"${DASHBOARD_FILE}" \ - "${GRAFANA_URL}/api/dashboards/db" \ - >/dev/null && echo "copilot-metrics-lgtm: dashboard provisioned" \ - || echo "copilot-metrics-lgtm: dashboard provisioning failed (Grafana may still be starting)" - fi - fi -fi diff --git a/test/copilot-metrics-lgtm/debian.sh b/test/copilot-metrics-lgtm/debian.sh deleted file mode 100644 index 71249e3..0000000 --- a/test/copilot-metrics-lgtm/debian.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# This test file will be executed against an auto-generated devcontainer.json that -# includes the 'copilot-metrics-lgtm' Feature with no options. -# -# Eg: -# { -# "image": "<..some-base-image...>", -# "features": { -# "copilot-metrics-lgtm": {} -# } -# } - -set -e - -source dev-container-features-test-lib - -FEATURE_DIR="/usr/local/share/copilot-metrics-lgtm" - -# Core runtime files -check "post-start.sh exists" test -f "${FEATURE_DIR}/post-start.sh" -check "post-start.sh is executable" test -x "${FEATURE_DIR}/post-start.sh" -check "docker-compose.yml exists" test -f "${FEATURE_DIR}/docker-compose.yml" -check "dashboard JSON exists" test -f "${FEATURE_DIR}/dashboards/github-copilot.json" - -# Config file written by install.sh -check ".env exists" test -f "${FEATURE_DIR}/.env" -check ".env has LGTM_IMAGE" grep -q "^LGTM_IMAGE=" "${FEATURE_DIR}/.env" -check ".env has INSTALL_DASHBOARD" grep -q "^INSTALL_DASHBOARD=" "${FEATURE_DIR}/.env" -check ".env has DOCKER_MODE" grep -q "^DOCKER_MODE=" "${FEATURE_DIR}/.env" -check ".env has GRAFANA_PORT" grep -q "^GRAFANA_PORT=" "${FEATURE_DIR}/.env" -check ".env has CAPTURE_CONTENT" grep -q "^CAPTURE_CONTENT=" "${FEATURE_DIR}/.env" -check "profile.d env var exists" test -f /etc/profile.d/copilot-metrics-lgtm.sh - -# Syntax checks -check "post-start.sh valid bash" bash -n "${FEATURE_DIR}/post-start.sh" -check "dashboard is valid JSON" jq empty "${FEATURE_DIR}/dashboards/github-copilot.json" - -reportResults diff --git a/test/copilot-metrics-lgtm/default.sh b/test/copilot-metrics-lgtm/default.sh deleted file mode 100644 index 71249e3..0000000 --- a/test/copilot-metrics-lgtm/default.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# This test file will be executed against an auto-generated devcontainer.json that -# includes the 'copilot-metrics-lgtm' Feature with no options. -# -# Eg: -# { -# "image": "<..some-base-image...>", -# "features": { -# "copilot-metrics-lgtm": {} -# } -# } - -set -e - -source dev-container-features-test-lib - -FEATURE_DIR="/usr/local/share/copilot-metrics-lgtm" - -# Core runtime files -check "post-start.sh exists" test -f "${FEATURE_DIR}/post-start.sh" -check "post-start.sh is executable" test -x "${FEATURE_DIR}/post-start.sh" -check "docker-compose.yml exists" test -f "${FEATURE_DIR}/docker-compose.yml" -check "dashboard JSON exists" test -f "${FEATURE_DIR}/dashboards/github-copilot.json" - -# Config file written by install.sh -check ".env exists" test -f "${FEATURE_DIR}/.env" -check ".env has LGTM_IMAGE" grep -q "^LGTM_IMAGE=" "${FEATURE_DIR}/.env" -check ".env has INSTALL_DASHBOARD" grep -q "^INSTALL_DASHBOARD=" "${FEATURE_DIR}/.env" -check ".env has DOCKER_MODE" grep -q "^DOCKER_MODE=" "${FEATURE_DIR}/.env" -check ".env has GRAFANA_PORT" grep -q "^GRAFANA_PORT=" "${FEATURE_DIR}/.env" -check ".env has CAPTURE_CONTENT" grep -q "^CAPTURE_CONTENT=" "${FEATURE_DIR}/.env" -check "profile.d env var exists" test -f /etc/profile.d/copilot-metrics-lgtm.sh - -# Syntax checks -check "post-start.sh valid bash" bash -n "${FEATURE_DIR}/post-start.sh" -check "dashboard is valid JSON" jq empty "${FEATURE_DIR}/dashboards/github-copilot.json" - -reportResults diff --git a/test/copilot-metrics-lgtm/noble.sh b/test/copilot-metrics-lgtm/noble.sh deleted file mode 100644 index 71249e3..0000000 --- a/test/copilot-metrics-lgtm/noble.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# This test file will be executed against an auto-generated devcontainer.json that -# includes the 'copilot-metrics-lgtm' Feature with no options. -# -# Eg: -# { -# "image": "<..some-base-image...>", -# "features": { -# "copilot-metrics-lgtm": {} -# } -# } - -set -e - -source dev-container-features-test-lib - -FEATURE_DIR="/usr/local/share/copilot-metrics-lgtm" - -# Core runtime files -check "post-start.sh exists" test -f "${FEATURE_DIR}/post-start.sh" -check "post-start.sh is executable" test -x "${FEATURE_DIR}/post-start.sh" -check "docker-compose.yml exists" test -f "${FEATURE_DIR}/docker-compose.yml" -check "dashboard JSON exists" test -f "${FEATURE_DIR}/dashboards/github-copilot.json" - -# Config file written by install.sh -check ".env exists" test -f "${FEATURE_DIR}/.env" -check ".env has LGTM_IMAGE" grep -q "^LGTM_IMAGE=" "${FEATURE_DIR}/.env" -check ".env has INSTALL_DASHBOARD" grep -q "^INSTALL_DASHBOARD=" "${FEATURE_DIR}/.env" -check ".env has DOCKER_MODE" grep -q "^DOCKER_MODE=" "${FEATURE_DIR}/.env" -check ".env has GRAFANA_PORT" grep -q "^GRAFANA_PORT=" "${FEATURE_DIR}/.env" -check ".env has CAPTURE_CONTENT" grep -q "^CAPTURE_CONTENT=" "${FEATURE_DIR}/.env" -check "profile.d env var exists" test -f /etc/profile.d/copilot-metrics-lgtm.sh - -# Syntax checks -check "post-start.sh valid bash" bash -n "${FEATURE_DIR}/post-start.sh" -check "dashboard is valid JSON" jq empty "${FEATURE_DIR}/dashboards/github-copilot.json" - -reportResults diff --git a/test/copilot-metrics-lgtm/scenarios.json b/test/copilot-metrics-lgtm/scenarios.json deleted file mode 100644 index adc5bfe..0000000 --- a/test/copilot-metrics-lgtm/scenarios.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "default": { - "image": "mcr.microsoft.com/devcontainers/base:ubuntu-22.04", - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "copilot-metrics-lgtm": {} - } - }, - "noble": { - "image": "mcr.microsoft.com/devcontainers/base:ubuntu-24.04", - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "copilot-metrics-lgtm": {} - } - }, - "debian": { - "image": "mcr.microsoft.com/devcontainers/base:debian-12", - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "copilot-metrics-lgtm": {} - } - } -} diff --git a/test/copilot-metrics-lgtm/test.sh b/test/copilot-metrics-lgtm/test.sh deleted file mode 100644 index 71249e3..0000000 --- a/test/copilot-metrics-lgtm/test.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# This test file will be executed against an auto-generated devcontainer.json that -# includes the 'copilot-metrics-lgtm' Feature with no options. -# -# Eg: -# { -# "image": "<..some-base-image...>", -# "features": { -# "copilot-metrics-lgtm": {} -# } -# } - -set -e - -source dev-container-features-test-lib - -FEATURE_DIR="/usr/local/share/copilot-metrics-lgtm" - -# Core runtime files -check "post-start.sh exists" test -f "${FEATURE_DIR}/post-start.sh" -check "post-start.sh is executable" test -x "${FEATURE_DIR}/post-start.sh" -check "docker-compose.yml exists" test -f "${FEATURE_DIR}/docker-compose.yml" -check "dashboard JSON exists" test -f "${FEATURE_DIR}/dashboards/github-copilot.json" - -# Config file written by install.sh -check ".env exists" test -f "${FEATURE_DIR}/.env" -check ".env has LGTM_IMAGE" grep -q "^LGTM_IMAGE=" "${FEATURE_DIR}/.env" -check ".env has INSTALL_DASHBOARD" grep -q "^INSTALL_DASHBOARD=" "${FEATURE_DIR}/.env" -check ".env has DOCKER_MODE" grep -q "^DOCKER_MODE=" "${FEATURE_DIR}/.env" -check ".env has GRAFANA_PORT" grep -q "^GRAFANA_PORT=" "${FEATURE_DIR}/.env" -check ".env has CAPTURE_CONTENT" grep -q "^CAPTURE_CONTENT=" "${FEATURE_DIR}/.env" -check "profile.d env var exists" test -f /etc/profile.d/copilot-metrics-lgtm.sh - -# Syntax checks -check "post-start.sh valid bash" bash -n "${FEATURE_DIR}/post-start.sh" -check "dashboard is valid JSON" jq empty "${FEATURE_DIR}/dashboards/github-copilot.json" - -reportResults