A configurable PII classification engine with regex/ML-based pattern …

backend/services/shared/apiResponse.ts

@@ -20,6 +20,7 @@
  */
 
 import { randomUUID } from 'crypto';
+import { piiClassifier, type ClassificationLevel } from './piiClassifier';
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Core types
@@ -327,3 +328,58 @@ export const API_VERSION_VALUE = '1';
  * so that the requestId in the response meta can be correlated with server logs.
  */
 export const REQUEST_ID_HEADER = 'X-Request-ID';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// #668 – PII Redaction middleware helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Redact PII from an ApiSuccessResponse before sending it over the wire.
+ *
+ * @example
+ * // In an Express handler:
+ * const response = ok(userData, requestId);
+ * res.json(redactResponse(response));               // standard level
+ * res.json(redactResponse(response, 'strict'));      // strict level
+ */
+export function redactResponse<T>(
+  response: ApiSuccessResponse<T>,
+  level: ClassificationLevel = 'standard',
+  allowList?: string[]
+): ApiSuccessResponse<unknown> {
+  return {
+    ...response,
+    data: piiClassifier.redact(response.data, { level, allowList }),
+  };
+}
+
+/**
+ * Express/Fastify-compatible middleware factory that automatically redacts PII
+ * from every outgoing JSON response body.
+ *
+ * Usage:
+ * ```ts
+ * app.use(createPiiRedactionMiddleware());              // standard
+ * app.use(createPiiRedactionMiddleware('strict'));       // strict
+ * ```
+ */
+export function createPiiRedactionMiddleware(
+  level: ClassificationLevel = 'standard',
+  allowList?: string[]
+) {
+  return function piiRedactionMiddleware(
+    _req: unknown,
+    res: {
+      json: (body: unknown) => void;
+      send: (body: unknown) => void;
+    },
+    next: () => void
+  ): void {
+    const originalJson = res.json.bind(res);
+    res.json = (body: unknown) => {
+      const redacted = piiClassifier.redact(body, { level, allowList });
+      return originalJson(redacted);
+    };
+    next();
+  };
+}

backend/services/shared/index.ts

@@ -23,7 +23,10 @@ export type { AuditAction, AuditEvent, AuditReport, ExportFormat, RetentionPolic
 export { exportUserData, deleteUserData, anonymizeUserData, updateConsent } from './gdpr';
 export type { UserConsent, ExportResult, DeletionResult, AnonymizationResult } from './gdpr';
 export { piiAuditService, PiiAuditService } from './piiAudit';
-export type { PiiAccessAction, PiiAccessRecord } from './piiAudit';
+export type { PiiAccessAction, PiiAccessRecord, LineageNode, PiiLineageTrail, PiiAuditReport } from './piiAudit';
+export { PiiClassifier, piiClassifier, redact, isPiiField, DEFAULT_PATTERNS } from './piiClassifier';
+export type { ClassificationLevel, PiiPattern, ClassifyResult, RedactOptions } from './piiClassifier';
+export { redactResponse, createPiiRedactionMiddleware } from './apiResponse';
 export { RateLimitingService, rateLimitingService } from './rateLimitingService';
 export { apiClient } from './apiClient';
 export {

backend/services/shared/logging.ts

@@ -1,3 +1,5 @@
+import { piiClassifier, type ClassificationLevel } from './piiClassifier';
+
 export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
 
 const LOG_LEVEL_PRIORITY: Record<LogLevel, number> = {
@@ -24,6 +26,20 @@ export interface LogContext {
   correlationId?: string;
 }
 
+// ─── PII redaction for structured log context ─────────────────────────────────
+
+let _logRedactionLevel: ClassificationLevel = 'standard';
+
+/** Set the classification level used for log PII redaction (default: standard). */
+export function setLogRedactionLevel(level: ClassificationLevel): void {
+  _logRedactionLevel = level;
+}
+
+function sanitizeContext(ctx: LogContext | undefined): LogContext | undefined {
+  if (!ctx) return ctx;
+  return piiClassifier.redact(ctx, { level: _logRedactionLevel }) as LogContext;
+}
+
 function shouldLog(level: LogLevel) {
   return LOG_LEVEL_PRIORITY[level] >= LOG_LEVEL_PRIORITY[CURRENT_LEVEL];
 }
@@ -50,7 +66,7 @@ async function sendToRemote(_logEntry: any) {
 function log(level: LogLevel, message: string, context?: LogContext) {
   if (!shouldLog(level)) return;
 
-  const logEntry = formatLog(level, message, context);
+  const logEntry = formatLog(level, message, sanitizeContext(context));
 
   sendToConsole(logEntry);
 
@@ -66,4 +82,5 @@ export const logger = {
   error: (msg: string, ctx?: LogContext) => log('error', msg, ctx),
 
   createCorrelationId: generateId,
+  setRedactionLevel: setLogRedactionLevel,
 };

backend/services/shared/piiAudit.ts

@@ -1,6 +1,6 @@
 import { AuditService } from './auditService';
 import type { AuditAction, AuditContext, AuditEvent } from './auditTypes';
-import { isPiiField } from './encryption';
+import { isPiiField } from './piiClassifier';
 
 export type PiiAccessAction =
   | 'pii.viewed'
@@ -18,13 +18,75 @@ export interface PiiAccessRecord {
   fieldsAccessed: string[];
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Data lineage
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** A single hop in the PII data lineage graph. */
+export interface LineageNode {
+  /** Unique step identifier */
+  stepId: string;
+  /** Module / service that processed the PII (e.g. 'billing', 'analytics') */
+  module: string;
+  /** Operation performed */
+  operation: string;
+  /** PII field names present at this step */
+  fields: string[];
+  /** Protection applied at this step */
+  protection: 'none' | 'encrypted' | 'redacted' | 'anonymized';
+  timestamp: number;
+}
+
+/** Full lineage trail for a single data subject (userId) */
+export interface PiiLineageTrail {
+  subjectId: string;
+  resourceType: string;
+  nodes: LineageNode[];
+  createdAt: number;
+  lastUpdatedAt: number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Audit report
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface PiiAuditReport {
+  generatedAt: number;
+  periodStart: number;
+  periodEnd: number;
+  totalAccesses: number;
+  /** Count per action type */
+  byAction: Record<string, number>;
+  /** Count per PII field name */
+  byField: Record<string, number>;
+  /** Count per endpoint or module */
+  byModule: Record<string, number>;
+  uniqueActors: number;
+  /** Actors with the highest PII access counts */
+  topActors: Array<{ actorId: string; count: number }>;
+  /** Fields most frequently accessed */
+  topFields: Array<{ field: string; count: number }>;
+  /** High-severity events (exports, deletes) */
+  highRiskEvents: PiiAccessRecord[];
+  /** Lineage summaries keyed by subjectId */
+  lineageSummary: Record<string, { nodeCount: number; modules: string[] }>;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Service
+// ─────────────────────────────────────────────────────────────────────────────
+
 export class PiiAuditService {
   private auditService: AuditService;
+  /** In-memory lineage store — production should use a persistent store */
+  private lineage: Map<string, PiiLineageTrail> = new Map();
 
   constructor(auditService: AuditService) {
     this.auditService = auditService;
   }
 
+  // ── Access logging ─────────────────────────────────────────────────────────
+
   logPiiAccess(
     action: PiiAccessAction,
     actorId: string,
@@ -45,7 +107,7 @@ export class PiiAuditService {
       resourceType,
       {
         ...metadata,
-        piiFields: piiFields,
+        piiFields,
         accessTimestamp: Date.now(),
         isMasked: (process.env['APP_ENV'] ?? 'development') !== 'production',
       },
@@ -56,6 +118,51 @@ export class PiiAuditService {
     return { event, fieldsAccessed: piiFields };
   }
 
+  // ── Data lineage tracking ──────────────────────────────────────────────────
+
+  /**
+   * Record a lineage hop for a given data subject.
+   *
+   * @param subjectId   - The data subject (e.g. userId)
+   * @param resourceType - e.g. 'User', 'Subscription'
+   * @param node        - The processing step details
+   */
+  trackLineage(subjectId: string, resourceType: string, node: Omit<LineageNode, 'timestamp'>): void {
+    const key = `${resourceType}:${subjectId}`;
+    const now = Date.now();
+    const fullNode: LineageNode = { ...node, timestamp: now };
+
+    if (this.lineage.has(key)) {
+      const trail = this.lineage.get(key)!;
+      trail.nodes.push(fullNode);
+      trail.lastUpdatedAt = now;
+    } else {
+      this.lineage.set(key, {
+        subjectId,
+        resourceType,
+        nodes: [fullNode],
+        createdAt: now,
+        lastUpdatedAt: now,
+      });
+    }
+  }
+
+  /**
+   * Retrieve the full lineage trail for a data subject.
+   */
+  getLineage(subjectId: string, resourceType: string): PiiLineageTrail | undefined {
+    return this.lineage.get(`${resourceType}:${subjectId}`);
+  }
+
+  /**
+   * Clear lineage data for a subject (supports GDPR deletion).
+   */
+  clearLineage(subjectId: string, resourceType: string): void {
+    this.lineage.delete(`${resourceType}:${subjectId}`);
+  }
+
+  // ── Access history ─────────────────────────────────────────────────────────
+
   getPiiAccessHistory(actorId?: string, from?: number, to?: number): PiiAccessRecord[] {
     const piiActions: PiiAccessAction[] = [
       'pii.viewed',
@@ -69,11 +176,7 @@ export class PiiAuditService {
       'pii.searched',
     ];
 
-    const events = this.auditService.query({
-      from,
-      to,
-      actorId,
-    });
+    const events = this.auditService.query({ from, to, actorId });
 
     return events
       .filter((e) => piiActions.includes(e.action as PiiAccessAction))
@@ -111,6 +214,74 @@ export class PiiAuditService {
       uniqueActors: actors.size,
     };
   }
+
+  // ── Full audit report ──────────────────────────────────────────────────────
+
+  /**
+   * Generate a PII audit report for a time window.
+   * Covers: access counts, top actors, top fields, high-risk events, and
+   * lineage summaries per data subject.
+   */
+  generateReport(from: number, to: number): PiiAuditReport {
+    const records = this.getPiiAccessHistory(undefined, from, to);
+    const byAction: Record<string, number> = {};
+    const byField: Record<string, number> = {};
+    const byModule: Record<string, number> = {};
+    const actorCounts: Record<string, number> = {};
+    const highRiskActions = new Set<PiiAccessAction>(['pii.exported', 'pii.deleted']);
+    const highRiskEvents: PiiAccessRecord[] = [];
+
+    for (const record of records) {
+      const { event, fieldsAccessed } = record;
+      byAction[event.action] = (byAction[event.action] ?? 0) + 1;
+
+      for (const field of fieldsAccessed) {
+        byField[field] = (byField[field] ?? 0) + 1;
+      }
+
+      const module = (event.metadata?.module as string) ?? event.resourceType ?? 'unknown';
+      byModule[module] = (byModule[module] ?? 0) + 1;
+
+      actorCounts[event.actorId] = (actorCounts[event.actorId] ?? 0) + 1;
+
+      if (highRiskActions.has(event.action as PiiAccessAction)) {
+        highRiskEvents.push(record);
+      }
+    }
+
+    const topActors = Object.entries(actorCounts)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10)
+      .map(([actorId, count]) => ({ actorId, count }));
+
+    const topFields = Object.entries(byField)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10)
+      .map(([field, count]) => ({ field, count }));
+
+    // Lineage summary
+    const lineageSummary: Record<string, { nodeCount: number; modules: string[] }> = {};
+    for (const [key, trail] of this.lineage.entries()) {
+      const [, subjectId] = key.split(':');
+      const modules = [...new Set(trail.nodes.map((n) => n.module))];
+      lineageSummary[subjectId] = { nodeCount: trail.nodes.length, modules };
+    }
+
+    return {
+      generatedAt: Date.now(),
+      periodStart: from,
+      periodEnd: to,
+      totalAccesses: records.length,
+      byAction,
+      byField,
+      byModule,
+      uniqueActors: Object.keys(actorCounts).length,
+      topActors,
+      topFields,
+      highRiskEvents,
+      lineageSummary,
+    };
+  }
 }
 
 export const piiAuditService = new PiiAuditService(