/** * Data Retention Service * Automatically cleans up old data based on tenant retention policies */ const cron = require('node-cron'); const { Op } = require('sequelize'); const { securityLogger } = require('../middleware/logger'); class DataRetentionService { constructor() { this.isRunning = false; this.lastCleanup = null; this.cleanupStats = { totalRuns: 0, totalDetectionsDeleted: 0, totalHeartbeatsDeleted: 0, totalLogsDeleted: 0, lastRunDuration: 0 }; } /** * Start the data retention cleanup service * Runs daily at 2 AM */ start() { console.log('๐Ÿ—‚๏ธ Starting Data Retention Service...'); // Run daily at 2:00 AM cron.schedule('0 2 * * *', async () => { await this.performCleanup(); }, { scheduled: true, timezone: "UTC" }); // Also run immediately if NODE_ENV is development if (process.env.NODE_ENV === 'development') { console.log('๐Ÿงน Development mode: Running initial data retention cleanup...'); setTimeout(() => this.performCleanup(), 5000); // Wait 5 seconds for app to fully start } console.log('โœ… Data Retention Service started - will run daily at 2:00 AM UTC'); } /** * Perform cleanup for all tenants */ async performCleanup() { if (this.isRunning) { console.log('โณ Data retention cleanup already running, skipping...'); return; } this.isRunning = true; const startTime = Date.now(); try { console.log('๐Ÿงน Starting data retention cleanup...'); const { Tenant, DroneDetection, Heartbeat, SecurityLog } = require('../models'); // Get all tenants with their retention policies const tenants = await Tenant.findAll({ attributes: ['id', 'slug', 'features'], where: { is_active: true } }); let totalDetectionsDeleted = 0; let totalHeartbeatsDeleted = 0; let totalLogsDeleted = 0; for (const tenant of tenants) { const retentionDays = tenant.features?.data_retention_days; // Skip if unlimited retention (-1) if (retentionDays === -1) { console.log(`โญ๏ธ Skipping tenant ${tenant.slug} - unlimited retention`); continue; } // Default to 90 days if not specified const effectiveRetentionDays = retentionDays || 90; const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - effectiveRetentionDays); console.log(`๐Ÿงน Cleaning tenant ${tenant.slug} - removing data older than ${effectiveRetentionDays} days (before ${cutoffDate.toISOString()})`); try { // Clean up drone detections const deletedDetections = await DroneDetection.destroy({ where: { tenant_id: tenant.id, timestamp: { [Op.lt]: cutoffDate } } }); // Clean up heartbeats const deletedHeartbeats = await Heartbeat.destroy({ where: { tenant_id: tenant.id, timestamp: { [Op.lt]: cutoffDate } } }); // Clean up security logs (if they have tenant_id) let deletedLogs = 0; try { deletedLogs = await SecurityLog.destroy({ where: { tenant_id: tenant.id, timestamp: { [Op.lt]: cutoffDate } } }); } catch (error) { // SecurityLog might not have tenant_id field, skip if error console.log(`โš ๏ธ Skipping security logs cleanup for tenant ${tenant.slug}: ${error.message}`); } totalDetectionsDeleted += deletedDetections; totalHeartbeatsDeleted += deletedHeartbeats; totalLogsDeleted += deletedLogs; console.log(`โœ… Tenant ${tenant.slug}: Deleted ${deletedDetections} detections, ${deletedHeartbeats} heartbeats, ${deletedLogs} logs`); // Log significant cleanup events if (deletedDetections > 100 || deletedHeartbeats > 100) { securityLogger.logSecurityEvent('info', 'Data retention cleanup performed', { action: 'data_retention_cleanup', tenantId: tenant.id, tenantSlug: tenant.slug, retentionDays: effectiveRetentionDays, cutoffDate: cutoffDate.toISOString(), deletedDetections, deletedHeartbeats, deletedLogs }); } } catch (error) { console.error(`โŒ Error cleaning tenant ${tenant.slug}:`, error); securityLogger.logSecurityEvent('error', 'Data retention cleanup failed', { action: 'data_retention_cleanup_error', tenantId: tenant.id, tenantSlug: tenant.slug, error: error.message, stack: error.stack }); } } const duration = Date.now() - startTime; this.lastCleanup = new Date(); this.cleanupStats.totalRuns++; this.cleanupStats.totalDetectionsDeleted += totalDetectionsDeleted; this.cleanupStats.totalHeartbeatsDeleted += totalHeartbeatsDeleted; this.cleanupStats.totalLogsDeleted += totalLogsDeleted; this.cleanupStats.lastRunDuration = duration; console.log(`โœ… Data retention cleanup completed in ${duration}ms`); console.log(`๐Ÿ“Š Total deleted: ${totalDetectionsDeleted} detections, ${totalHeartbeatsDeleted} heartbeats, ${totalLogsDeleted} logs`); // Log cleanup summary securityLogger.logSecurityEvent('info', 'Data retention cleanup completed', { action: 'data_retention_cleanup_summary', duration, tenantsProcessed: tenants.length, totalDetectionsDeleted, totalHeartbeatsDeleted, totalLogsDeleted, timestamp: new Date().toISOString() }); } catch (error) { console.error('โŒ Data retention cleanup failed:', error); securityLogger.logSecurityEvent('error', 'Data retention cleanup service failed', { action: 'data_retention_service_error', error: error.message, stack: error.stack, timestamp: new Date().toISOString() }); } finally { this.isRunning = false; } } /** * Get cleanup statistics */ getStats() { return { ...this.cleanupStats, isRunning: this.isRunning, lastCleanup: this.lastCleanup, nextScheduledRun: '2:00 AM UTC daily' }; } /** * Manually trigger cleanup (for testing/admin use) */ async triggerManualCleanup() { console.log('๐Ÿ”ง Manual data retention cleanup triggered'); await this.performCleanup(); } /** * Preview what would be deleted for a specific tenant */ async previewCleanup(tenantId) { try { const { Tenant, DroneDetection, Heartbeat, SecurityLog } = require('../models'); const tenant = await Tenant.findByPk(tenantId); if (!tenant) { throw new Error('Tenant not found'); } const retentionDays = tenant.features?.data_retention_days || 90; if (retentionDays === -1) { return { tenantSlug: tenant.slug, retentionDays: 'unlimited', toDelete: { detections: 0, heartbeats: 0, logs: 0 } }; } const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - retentionDays); const [detectionsCount, heartbeatsCount] = await Promise.all([ DroneDetection.count({ where: { tenant_id: tenant.id, timestamp: { [Op.lt]: cutoffDate } } }), Heartbeat.count({ where: { tenant_id: tenant.id, timestamp: { [Op.lt]: cutoffDate } } }) ]); let logsCount = 0; try { logsCount = await SecurityLog.count({ where: { tenant_id: tenant.id, timestamp: { [Op.lt]: cutoffDate } } }); } catch (error) { // SecurityLog might not have tenant_id } return { tenantSlug: tenant.slug, retentionDays, cutoffDate: cutoffDate.toISOString(), toDelete: { detections: detectionsCount, heartbeats: heartbeatsCount, logs: logsCount } }; } catch (error) { console.error('Error previewing cleanup:', error); throw error; } } } module.exports = DataRetentionService;