From dea4668778a162969f9c3889fec622580b7c3e2b Mon Sep 17 00:00:00 2001 From: Adeyemi-cmd Date: Sun, 28 Jun 2026 23:29:47 +0100 Subject: [PATCH] feat(data-retention): add automated pruning for audit logs and analytics events --- .env.example | 6 ++ src/analytics/analytics.module.ts | 2 + .../tasks/analytics-retention.task.ts | 65 ++++++++++++++++++ src/audit-log/audit-log.module.ts | 2 + src/audit-log/tasks/audit-retention.task.ts | 66 ++++++++++++++----- src/config/env.validation.ts | 4 ++ src/config/retention.config.ts | 5 ++ 7 files changed, 135 insertions(+), 15 deletions(-) create mode 100644 src/analytics/tasks/analytics-retention.task.ts diff --git a/.env.example b/.env.example index d972f49d..4a96f1e5 100644 --- a/.env.example +++ b/.env.example @@ -464,6 +464,12 @@ IDEMPOTENCY_TTL_SECONDS=86400 # Segment write key (for analytics, optional) SEGMENT_WRITE_KEY= +# Audit log retention period in days (default: 730 = 2 years) +AUDIT_LOG_RETENTION_DAYS=730 + +# Analytics event retention period in days (default: 365 = 1 year) +ANALYTICS_RETENTION_DAYS=365 + # ───────────────────────────────────────────────────────────────────────────── # 22. CDN CONFIGURATION # ───────────────────────────────────────────────────────────────────────────── diff --git a/src/analytics/analytics.module.ts b/src/analytics/analytics.module.ts index c7c73d6b..d087ad26 100644 --- a/src/analytics/analytics.module.ts +++ b/src/analytics/analytics.module.ts @@ -11,6 +11,7 @@ import { AnalyticsEvent } from './entities/event.entity'; import { EventBatchingService } from './services/event-batching.service'; import { EventValidationService } from './services/event-validation.service'; import { EventTrackingSDK } from './sdk/event-tracking.sdk'; +import { AnalyticsRetentionTask } from './tasks/analytics-retention.task'; @Module({ imports: [ @@ -24,6 +25,7 @@ import { EventTrackingSDK } from './sdk/event-tracking.sdk'; EventBatchingService, EventValidationService, EventTrackingSDK, + AnalyticsRetentionTask, { provide: APP_INTERCEPTOR, useClass: FingerprintInterceptor }, ], controllers: [AnalyticsController], diff --git a/src/analytics/tasks/analytics-retention.task.ts b/src/analytics/tasks/analytics-retention.task.ts new file mode 100644 index 00000000..c3dd3fca --- /dev/null +++ b/src/analytics/tasks/analytics-retention.task.ts @@ -0,0 +1,65 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, DeleteResult } from 'typeorm'; +import { Counter } from 'prom-client'; +import { AnalyticsEvent } from '../entities/event.entity'; +import { MetricsCollectionService } from '../../monitoring/metrics/metrics-collection.service'; +import { ConfigService } from '@nestjs/config'; + +@Injectable() +export class AnalyticsRetentionTask { + private readonly logger = new Logger(AnalyticsRetentionTask.name); + private readonly retentionDays: number; + private readonly batchSize = 1000; + private deletedCounter: Counter<'table'>; + + constructor( + @InjectRepository(AnalyticsEvent) + private readonly eventRepository: Repository, + private readonly configService: ConfigService, + private readonly metrics: MetricsCollectionService, + ) { + this.retentionDays = this.configService.get('ANALYTICS_RETENTION_DAYS', 365); + const registry = this.metrics.getRegistry(); + const prom = require('prom-client'); + this.deletedCounter = + (registry.getSingleMetric('deleted_count') as Counter<'table'>) ?? + new prom.Counter({ + name: 'deleted_count', + help: 'Number of rows deleted by data retention policies', + labelNames: ['table'] as const, + registers: [registry], + }); + } + + @Cron('30 2 * * *') + async handleDailyRetention(): Promise { + this.logger.log('Starting daily analytics event retention policy...'); + let totalDeleted = 0; + try { + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - this.retentionDays); + + let deleted = 0; + do { + const result: DeleteResult = await this.eventRepository + .createQueryBuilder() + .delete() + .from(AnalyticsEvent) + .where('timestamp < :cutoff', { cutoff }) + .limit(this.batchSize) + .execute(); + deleted = result.affected || 0; + totalDeleted += deleted; + } while (deleted >= this.batchSize); + + this.deletedCounter.inc({ table: 'analytics_events' }, totalDeleted); + this.logger.log( + `Daily analytics retention policy completed. Deleted ${totalDeleted} old events.`, + ); + } catch (error) { + this.logger.error('Failed to apply analytics retention policy:', error); + } + } +} diff --git a/src/audit-log/audit-log.module.ts b/src/audit-log/audit-log.module.ts index bc6154c9..b0ece040 100644 --- a/src/audit-log/audit-log.module.ts +++ b/src/audit-log/audit-log.module.ts @@ -7,6 +7,7 @@ import { AuditQueryService } from './services/audit-query.service'; import { AuditReportingService } from './services/audit-reporting.service'; import { AuditExportService } from './services/audit-export.service'; import { AuditRetentionTask } from './tasks/audit-retention.task'; +import { MetricsCollectionService } from '../monitoring/metrics/metrics-collection.service'; /** * Audit Log Module @@ -26,6 +27,7 @@ import { AuditRetentionTask } from './tasks/audit-retention.task'; AuditExportService, AuditRetentionTask, AuditLogService, + MetricsCollectionService, ], exports: [AuditLogService], }) diff --git a/src/audit-log/tasks/audit-retention.task.ts b/src/audit-log/tasks/audit-retention.task.ts index 545bc799..9fc9e507 100644 --- a/src/audit-log/tasks/audit-retention.task.ts +++ b/src/audit-log/tasks/audit-retention.task.ts @@ -1,31 +1,69 @@ import { Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, DeleteResult } from 'typeorm'; +import { Counter } from 'prom-client'; +import { AuditLog } from '../audit-log.entity'; import { AuditLogService } from '../audit-log.service'; +import { MetricsCollectionService } from '../../monitoring/metrics/metrics-collection.service'; +import { ConfigService } from '@nestjs/config'; -/** - * Provides audit Retention Task behavior. - */ @Injectable() export class AuditRetentionTask { private readonly logger = new Logger(AuditRetentionTask.name); - constructor(private readonly auditLogService: AuditLogService) {} - /** - * Run retention policy daily at 2 AM - */ + private readonly retentionDays: number; + private readonly batchSize = 1000; + private deletedCounter: Counter<'table'>; + + constructor( + @InjectRepository(AuditLog) + private readonly auditLogRepo: Repository, + private readonly auditLogService: AuditLogService, + private readonly configService: ConfigService, + private readonly metrics: MetricsCollectionService, + ) { + this.retentionDays = this.configService.get('AUDIT_LOG_RETENTION_DAYS', 730); + const registry = this.metrics.getRegistry(); + const prom = require('prom-client'); + this.deletedCounter = + (registry.getSingleMetric('deleted_count') as Counter<'table'>) ?? + new prom.Counter({ + name: 'deleted_count', + help: 'Number of rows deleted by data retention policies', + labelNames: ['table'] as const, + registers: [registry], + }); + } + @Cron(CronExpression.EVERY_DAY_AT_2AM) async handleDailyRetention(): Promise { this.logger.log('Starting daily audit log retention policy...'); + let totalDeleted = 0; try { - const deletedCount = await this.auditLogService.applyRetentionPolicy(); - this.logger.log(`Daily retention policy completed. Deleted ${deletedCount} old audit logs.`); + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - this.retentionDays); + + let deleted = 0; + do { + const result: DeleteResult = await this.auditLogRepo + .createQueryBuilder() + .delete() + .from(AuditLog) + .where('timestamp < :cutoff', { cutoff }) + .limit(this.batchSize) + .execute(); + deleted = result.affected || 0; + totalDeleted += deleted; + } while (deleted >= this.batchSize); + + this.deletedCounter.inc({ table: 'audit_logs' }, totalDeleted); + this.logger.log(`Daily retention policy completed. Deleted ${totalDeleted} old audit logs.`); } catch (error) { this.logger.error('Failed to apply retention policy:', error); } } - /** - * Generate weekly report every Monday at 3 AM - */ - @Cron('0 3 * * 1') // Every Monday at 3 AM + + @Cron('0 3 * * 1') async handleWeeklyReport(): Promise { this.logger.log('Generating weekly audit report...'); try { @@ -38,8 +76,6 @@ export class AuditRetentionTask { criticalEvents: report.eventsBySeverity['CRITICAL'] || 0, errorEvents: report.eventsBySeverity['ERROR'] || 0, }); - // In a real implementation, you might send this report via email - // or store it for compliance purposes } catch (error) { this.logger.error('Failed to generate weekly report:', error); } diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index 9079fa2f..0d8d48af 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -158,6 +158,10 @@ export const envValidationSchema = Joi.object({ // Segment Analytics SEGMENT_WRITE_KEY: Joi.string().optional(), + // Data Retention + AUDIT_LOG_RETENTION_DAYS: Joi.number().integer().min(1).default(730), + ANALYTICS_RETENTION_DAYS: Joi.number().integer().min(1).default(365), + // Circuit Breaker Configuration CIRCUIT_BREAKER_TIMEOUT_MS: Joi.number().integer().min(100).default(3000), CIRCUIT_BREAKER_ERROR_THRESHOLD: Joi.number().integer().min(1).max(100).default(50), diff --git a/src/config/retention.config.ts b/src/config/retention.config.ts index 1fac9c51..53035b4a 100644 --- a/src/config/retention.config.ts +++ b/src/config/retention.config.ts @@ -17,6 +17,11 @@ export const retentionConfig = registerAs('retention', () => ({ */ notificationRetentionDays: parseInt(process.env.RETENTION_NOTIFICATION_DAYS || '30', 10), + /** + * Retention period for analytics events in days. + */ + analyticsRetentionDays: parseInt(process.env.ANALYTICS_RETENTION_DAYS || '365', 10), + /** * Whether to archive data before purging. */