Otel sources

README.md

@@ -28,11 +28,17 @@ For detailed deployment options (bare Node.js, reverse proxy, backups), see the
 - Register services with health check endpoints and poll them on configurable intervals (5s to 1hr)
 - Exponential backoff on failures with circuit breaker protection (opens after 10 consecutive failures)
 - Custom schema mapping for non-standard health endpoints, including object-keyed formats (Spring Boot Actuator, ASP.NET Health Checks, etc.) with skipped-check support
+- **OTLP push ingestion** — receive metrics via `POST /v1/metrics` and traces via `POST /v1/traces` from OpenTelemetry collectors with team-scoped API key authentication, auto-registration of unknown services, and per-service custom metric/attribute name mappings
+- **OTLP trace-based dependency discovery** — automatically discover dependencies from CLIENT and PRODUCER spans, with full span storage for future trace timeline views, configurable retention (default 7 days), and auto-association to registered services
+- **Histogram and sum metric processing** — extract percentile latency (p50/p95/p99) from OTLP histogram metrics and request counts from sum metrics via linear interpolation
+- **Prometheus scraping** — poll Prometheus text exposition endpoints (`text/plain; version=0.0.4`) with automatic metric-to-dependency mapping and per-service custom metric/label name mappings
 - Contact info and impact overrides with 3-tier merge hierarchy (instance > canonical > polled) — resolved in API responses
 - Per-hostname concurrency limiting and request deduplication prevent polling abuse
 
 **Visualization**
 - Interactive dependency graph (React Flow) with team filtering, search, layout controls, automatic high-latency detection, and isolated tree view
+- Visual distinction for auto-discovered vs manually-configured dependencies (dashed edges with "Suggested" badge for unconfirmed trace-discovered associations)
+- Org-wide external node enrichment — add display names, descriptions, impact, and contact info to shared external dependencies (e.g., Stripe, PostgreSQL)
 - Latency charts (min/avg/max over time) and health timeline swimlanes per dependency
 - Edge selection shows per-dependency latency chart, contact info, impact, and error history
 - Node selection shows aggregate latency chart across all dependents and merged contact info
@@ -77,8 +83,8 @@ For detailed deployment options (bare Node.js, reverse proxy, backups), see the
 
 **Operations**
 - SQLite database — zero external dependencies, sessions survive restarts
-- Automatic data retention cleanup (configurable period, default 365 days)
-- Runtime-configurable admin settings (retention, polling, rate limits, alerts)
+- Automatic data retention cleanup (configurable period, default 365 days) with separate span retention (default 7 days, admin-configurable)
+- Runtime-configurable admin settings (retention, span retention, polling, rate limits, alerts)
 - Structured JSON logging in production via pino
 - Docker image with health check and volume-mounted data
 
@@ -193,8 +199,10 @@ cp server/.env.example server/.env
 | `SSL_CERT_PATH` | — | PEM certificate path (pair with `SSL_KEY_PATH`) |
 | `SSL_KEY_PATH` | — | PEM private key path (pair with `SSL_CERT_PATH`) |
 | `HTTP_PORT` | — | Plain HTTP port for health checks + redirect when `ENABLE_HTTPS=true` |
-| `RATE_LIMIT_MAX` | `100` | Max requests per IP per 15-minute window |
-| `AUTH_RATE_LIMIT_MAX` | `10` | Max auth requests per IP per minute |
+| `RATE_LIMIT_MAX` | `3000` | Max requests per IP per minute (global) |
+| `AUTH_RATE_LIMIT_MAX` | `20` | Max auth requests per IP per minute |
+| `OTLP_RATE_LIMIT_MAX` | `600` | Max OTLP requests per IP per minute (global) |
+| `OTLP_PER_KEY_RATE_LIMIT_RPM` | `150000` | Default per-API-key rate limit (requests/minute) |
 
 **Operations:**
 
@@ -308,23 +316,28 @@ All endpoints require authentication unless noted. Admin endpoints require the a
 | Users | CRUD on `/api/users` (admin), `POST` and `PUT /:id/password` (local auth) |
 | Aliases | CRUD on `/api/aliases` (admin for mutations), `GET /canonical-names` |
 | Overrides | `GET/PUT/DELETE /api/canonical-overrides/:name`, `PUT/DELETE /api/dependencies/:id/overrides` |
-| Associations | CRUD on `/api/dependencies/:id/associations` |
+| Associations | CRUD on `/api/dependencies/:id/associations`, `PUT /:assocId/confirm`, `PUT /:assocId/dismiss` |
+| Discovered Deps | `GET /api/services/:id/dependencies/discovered`, `PATCH /api/dependencies/:id/enrich` |
+| External Nodes | `GET/PUT/DELETE /api/external-nodes/:canonicalName` |
 | Graph | `GET /api/graph` with `team`, `service`, `dependency` filters |
 | History | `GET /api/latency/:id` + `/buckets`, `GET /api/errors/:id`, `GET /api/dependencies/:id/timeline`, `GET /api/services/:id/poll-history` |
-| Admin | `GET/PUT /api/admin/settings`, `GET /api/admin/audit-log` |
+| Admin | `GET/PUT /api/admin/settings`, `GET/PUT /api/admin/settings/span-retention`, `GET /api/admin/audit-log` |
 | Manifest | `GET/POST /api/teams/:id/manifests`, `GET/PUT/DELETE /:id/manifests/:configId`, `POST /:id/manifests/sync`, `POST /:id/manifests/:configId/sync`, `GET /:id/manifests/:configId/sync-history`, `POST /api/manifest/validate`, `POST /api/manifest/test-url` |
 | Drift Flags | `GET /api/teams/:id/drifts` + `/summary`, `PUT /:driftId/accept` + `/dismiss` + `/reopen`, `POST /bulk-accept` + `/bulk-dismiss` |
 | Catalog | `GET /api/catalog/external-dependencies` — canonical name registry with team usage, descriptions, and aliases |
 | Alerts | CRUD on `/api/teams/:id/alert-channels` + `/test`, `GET/PUT /:id/alert-rules`, `GET /:id/alert-history`, `GET/POST /:id/alert-mutes`, `DELETE /:id/alert-mutes/:muteId`, `GET /api/admin/alert-mutes` |
+| OTLP | `POST /v1/metrics`, `POST /v1/traces` (API key auth, OTLP JSON) |
+| API Keys | `GET/POST /api/teams/:id/api-keys`, `DELETE /:id/api-keys/:keyId` |
 
 ## Security
 
 Depsera includes defense-in-depth security:
 
 - **Security headers** via Helmet (CSP, HSTS, X-Frame-Options, X-Content-Type-Options)
+- **API key authentication** for OTLP push endpoints — team-scoped keys with SHA-256 hashing, prefix display, and last-used tracking
 - **SSRF protection** on health endpoints with private IP blocking and DNS rebinding prevention; configurable allowlist for internal networks
 - **CSRF protection** via double-submit cookie pattern
-- **Rate limiting** — global (100 req/15min) and auth-specific (10 req/min) per IP
+- **Rate limiting** — global (3000 req/min), auth-specific (20 req/min), OTLP global (600 req/min per IP), and per-API-key token bucket (150k req/min default)
 - **Session secret validation** — production startup refuses weak or missing secrets
 - **Redirect validation** prevents open redirect attacks on logout
 - **Body size limit** (100KB) on JSON payloads

client/src/App.test.tsx

@@ -31,12 +31,12 @@ describe('App', () => {
     expect(await screen.findByText('Sign in to continue')).toBeInTheDocument();
   });
 
-  it('shows the dashboard title on login page', async () => {
+  it('shows the logo on login page', async () => {
     mockFetch
       .mockResolvedValueOnce({ ok: false, status: 401, json: () => Promise.resolve({}) })
       .mockResolvedValueOnce({ ok: true, status: 200, json: () => Promise.resolve({ mode: 'oidc' }) });
 
     renderApp();
-    expect(await screen.findByRole('heading', { name: 'Depsera' })).toBeInTheDocument();
+    expect(await screen.findByAltText('Depsera')).toBeInTheDocument();
   });
 });

client/src/App.tsx

@@ -17,6 +17,7 @@ import ServiceCatalog from './components/pages/Catalog/ServiceCatalog';
 import ManifestPage from './components/pages/Manifest/ManifestPage';
 import ManifestAdmin from './components/pages/Admin/ManifestAdmin';
 import AlertMutesAdmin from './components/pages/Admin/AlertMutesAdmin';
+import OtlpAdmin from './components/pages/Admin/OtlpAdmin';
 
 function App() {
   return (
@@ -80,6 +81,14 @@ function App() {
             </ProtectedRoute>
           }
         />
+        <Route
+          path="admin/otlp"
+          element={
+            <ProtectedRoute requireAdmin>
+              <OtlpAdmin />
+            </ProtectedRoute>
+          }
+        />
       </Route>
       <Route path="*" element={<NotFound />} />
     </Routes>

client/src/api/apiKeys.ts

@@ -0,0 +1,50 @@
+import { handleResponse } from './common';
+import { withCsrfToken } from './csrf';
+
+export interface ApiKey {
+  id: string;
+  team_id: string;
+  name: string;
+  key_prefix: string;
+  last_used_at: string | null;
+  created_at: string;
+  created_by: string;
+  rate_limit_rpm: number | null;
+  rate_limit_admin_locked: number;
+}
+
+export interface ApiKeyWithRawKey extends ApiKey {
+  rawKey: string;
+}
+
+export async function listApiKeys(teamId: string): Promise<ApiKey[]> {
+  const response = await fetch(`/api/teams/${teamId}/api-keys`, {
+    credentials: 'include',
+  });
+  return handleResponse<ApiKey[]>(response);
+}
+
+export async function createApiKey(
+  teamId: string,
+  name: string
+): Promise<ApiKeyWithRawKey> {
+  const response = await fetch(`/api/teams/${teamId}/api-keys`, {
+    method: 'POST',
+    headers: withCsrfToken({ 'Content-Type': 'application/json' }),
+    body: JSON.stringify({ name }),
+    credentials: 'include',
+  });
+  return handleResponse<ApiKeyWithRawKey>(response);
+}
+
+export async function deleteApiKey(teamId: string, keyId: string): Promise<void> {
+  const response = await fetch(`/api/teams/${teamId}/api-keys/${keyId}`, {
+    method: 'DELETE',
+    headers: withCsrfToken(),
+    credentials: 'include',
+  });
+  if (!response.ok) {
+    const error = await response.json().catch(() => ({ message: 'Delete failed' }));
+    throw new Error(error.message || error.error || `HTTP error ${response.status}`);
+  }
+}

client/src/api/associations.ts

@@ -35,3 +35,25 @@ export async function deleteAssociation(
     throw new Error(error.message || `HTTP error ${response.status}`);
   }
 }
+
+export async function confirmAssociation(
+  depId: string,
+  assocId: string,
+): Promise<{ success: boolean }> {
+  const response = await fetch(
+    `/api/dependencies/${depId}/associations/${assocId}/confirm`,
+    { method: 'PUT', headers: withCsrfToken(), credentials: 'include' },
+  );
+  return handleResponse<{ success: boolean }>(response);
+}
+
+export async function dismissAssociation(
+  depId: string,
+  assocId: string,
+): Promise<{ success: boolean }> {
+  const response = await fetch(
+    `/api/dependencies/${depId}/associations/${assocId}/dismiss`,
+    { method: 'PUT', headers: withCsrfToken(), credentials: 'include' },
+  );
+  return handleResponse<{ success: boolean }>(response);
+}

client/src/api/common.test.ts

@@ -0,0 +1,71 @@
+import { handleResponse } from './common';
+
+function mockResponse(body: unknown, status = 200): Response {
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    json: () => Promise.resolve(body),
+  } as Response;
+}
+
+describe('handleResponse', () => {
+  it('returns parsed JSON on success', async () => {
+    const data = { id: '1', name: 'Test' };
+    const result = await handleResponse(mockResponse(data));
+    expect(result).toEqual(data);
+  });
+
+  it('throws error with message from response body', async () => {
+    await expect(
+      handleResponse(mockResponse({ message: 'Not found' }, 404)),
+    ).rejects.toThrow('Not found');
+  });
+
+  it('throws error with error field from response body', async () => {
+    await expect(
+      handleResponse(mockResponse({ error: 'Bad request' }, 400)),
+    ).rejects.toThrow('Bad request');
+  });
+
+  it('throws generic error when response body has no message', async () => {
+    await expect(
+      handleResponse(mockResponse({}, 500)),
+    ).rejects.toThrow('HTTP error 500');
+  });
+
+  it('throws fallback error when response body is not JSON', async () => {
+    const response = {
+      ok: false,
+      status: 500,
+      json: () => Promise.reject(new Error('not JSON')),
+    } as Response;
+
+    await expect(handleResponse(response)).rejects.toThrow('Request failed');
+  });
+
+  it('dispatches auth:expired event on 401 response', async () => {
+    const handler = jest.fn();
+    window.addEventListener('auth:expired', handler);
+
+    await expect(
+      handleResponse(mockResponse({ error: 'Not authenticated' }, 401)),
+    ).rejects.toThrow('Not authenticated');
+
+    expect(handler).toHaveBeenCalledTimes(1);
+
+    window.removeEventListener('auth:expired', handler);
+  });
+
+  it('does not dispatch auth:expired event on non-401 errors', async () => {
+    const handler = jest.fn();
+    window.addEventListener('auth:expired', handler);
+
+    await expect(
+      handleResponse(mockResponse({ message: 'Server error' }, 500)),
+    ).rejects.toThrow('Server error');
+
+    expect(handler).not.toHaveBeenCalled();
+
+    window.removeEventListener('auth:expired', handler);
+  });
+});

client/src/api/common.ts

@@ -7,6 +7,9 @@
  */
 export async function handleResponse<T>(response: Response): Promise<T> {
   if (!response.ok) {
+    if (response.status === 401) {
+      window.dispatchEvent(new Event('auth:expired'));
+    }
     const error = await response.json().catch(() => ({ message: 'Request failed' }));
     throw new Error(error.message || error.error || `HTTP error ${response.status}`);
   }

client/src/api/dependencies.ts

@@ -31,3 +31,31 @@ export async function clearDependencyOverrides(id: string): Promise<void> {
     throw new Error(error.message || `HTTP error ${response.status}`);
   }
 }
+
+export interface DependencyEnrichmentInput {
+  displayName?: string | null;
+  description?: string | null;
+  impact?: string | null;
+}
+
+export async function enrichDependency(
+  id: string,
+  input: DependencyEnrichmentInput,
+): Promise<Dependency> {
+  const response = await fetch(`/api/dependencies/${id}/enrich`, {
+    method: 'PATCH',
+    headers: withCsrfToken({ 'Content-Type': 'application/json' }),
+    body: JSON.stringify(input),
+    credentials: 'include',
+  });
+  return handleResponse<Dependency>(response);
+}
+
+export async function listDiscoveredDependencies(
+  serviceId: string,
+): Promise<Dependency[]> {
+  const response = await fetch(`/api/services/${serviceId}/discovered-dependencies`, {
+    credentials: 'include',
+  });
+  return handleResponse<Dependency[]>(response);
+}

client/src/api/externalNodes.ts

@@ -0,0 +1,55 @@
+import { handleResponse } from './common';
+import { withCsrfToken } from './csrf';
+
+export interface ExternalNodeEnrichment {
+  id: string;
+  canonical_name: string;
+  display_name: string | null;
+  description: string | null;
+  impact: string | null;
+  contact: string | null;
+  service_type: string | null;
+  created_at: string;
+  updated_at: string;
+  updated_by: string | null;
+}
+
+export interface UpsertExternalNodeInput {
+  displayName?: string | null;
+  description?: string | null;
+  impact?: string | null;
+  contact?: Record<string, unknown> | null;
+  serviceType?: string | null;
+}
+
+export async function fetchExternalNodes(): Promise<ExternalNodeEnrichment[]> {
+  const response = await fetch('/api/external-nodes', {
+    credentials: 'include',
+  });
+  return handleResponse<ExternalNodeEnrichment[]>(response);
+}
+
+export async function upsertExternalNode(
+  canonicalName: string,
+  input: UpsertExternalNodeInput,
+): Promise<ExternalNodeEnrichment> {
+  const response = await fetch(`/api/external-nodes/${encodeURIComponent(canonicalName)}`, {
+    method: 'PUT',
+    headers: withCsrfToken({ 'Content-Type': 'application/json' }),
+    body: JSON.stringify(input),
+    credentials: 'include',
+  });
+  return handleResponse<ExternalNodeEnrichment>(response);
+}
+
+export async function deleteExternalNode(canonicalName: string): Promise<void> {
+  const response = await fetch(`/api/external-nodes/${encodeURIComponent(canonicalName)}`, {
+    method: 'DELETE',
+    headers: withCsrfToken(),
+    credentials: 'include',
+  });
+  if (!response.ok) {
+    const error = await response.json().catch(() => ({ message: 'Delete failed' }));
+    throw new Error(error.message || `HTTP error ${response.status}`);
+  }
+}