drone-detector/server/services/dataRetention.js

/**
 * Data Retention Service
 * Automatically cleans up old data based on tenant retention policies
 */

const cron = require('node-cron');
const { Op } = require('sequelize');
const { securityLogger } = require('../middleware/logger');

class DataRetentionService {
  constructor() {
    this.isRunning = false;
    this.lastCleanup = null;
    this.cleanupStats = {
      totalRuns: 0,
      totalDetectionsDeleted: 0,
      totalHeartbeatsDeleted: 0,
      totalLogsDeleted: 0,
      lastRunDuration: 0
    };
  }

  /**
   * Start the data retention cleanup service
   * Runs daily at 2 AM
   */
  start() {
    console.log('🗂️  Starting Data Retention Service...');
    
    // Run daily at 2:00 AM
    cron.schedule('0 2 * * *', async () => {
      await this.performCleanup();
    }, {
      scheduled: true,
      timezone: "UTC"
    });

    // Also run immediately if NODE_ENV is development
    if (process.env.NODE_ENV === 'development') {
      console.log('🧹 Development mode: Running initial data retention cleanup...');
      setTimeout(() => this.performCleanup(), 5000); // Wait 5 seconds for app to fully start
    }

    console.log('✅ Data Retention Service started - will run daily at 2:00 AM UTC');
  }

  /**
   * Perform cleanup for all tenants
   */
  async performCleanup() {
    if (this.isRunning) {
      console.log('⏳ Data retention cleanup already running, skipping...');
      return;
    }

    this.isRunning = true;
    const startTime = Date.now();
    
    try {
      console.log('🧹 Starting data retention cleanup...');
      
      const { Tenant, DroneDetection, Heartbeat, SecurityLog } = require('../models');
      
      // Get all tenants with their retention policies
      const tenants = await Tenant.findAll({
        attributes: ['id', 'slug', 'features'],
        where: {
          is_active: true
        }
      });

      let totalDetectionsDeleted = 0;
      let totalHeartbeatsDeleted = 0;
      let totalLogsDeleted = 0;

      for (const tenant of tenants) {
        const retentionDays = tenant.features?.data_retention_days;
        
        // Skip if unlimited retention (-1)
        if (retentionDays === -1) {
          console.log(`⏭️  Skipping tenant ${tenant.slug} - unlimited retention`);
          continue;
        }

        // Default to 90 days if not specified
        const effectiveRetentionDays = retentionDays || 90;
        const cutoffDate = new Date();
        cutoffDate.setDate(cutoffDate.getDate() - effectiveRetentionDays);

        console.log(`🧹 Cleaning tenant ${tenant.slug} - removing data older than ${effectiveRetentionDays} days (before ${cutoffDate.toISOString()})`);

        try {
          // Clean up drone detections
          const deletedDetections = await DroneDetection.destroy({
            where: {
              tenant_id: tenant.id,
              timestamp: {
                [Op.lt]: cutoffDate
              }
            }
          });

          // Clean up heartbeats
          const deletedHeartbeats = await Heartbeat.destroy({
            where: {
              tenant_id: tenant.id,
              timestamp: {
                [Op.lt]: cutoffDate
              }
            }
          });

          // Clean up security logs (if they have tenant_id)
          let deletedLogs = 0;
          try {
            deletedLogs = await SecurityLog.destroy({
              where: {
                tenant_id: tenant.id,
                timestamp: {
                  [Op.lt]: cutoffDate
                }
              }
            });
          } catch (error) {
            // SecurityLog might not have tenant_id field, skip if error
            console.log(`⚠️  Skipping security logs cleanup for tenant ${tenant.slug}: ${error.message}`);
          }

          totalDetectionsDeleted += deletedDetections;
          totalHeartbeatsDeleted += deletedHeartbeats;
          totalLogsDeleted += deletedLogs;

          console.log(`✅ Tenant ${tenant.slug}: Deleted ${deletedDetections} detections, ${deletedHeartbeats} heartbeats, ${deletedLogs} logs`);

          // Log significant cleanup events
          if (deletedDetections > 100 || deletedHeartbeats > 100) {
            securityLogger.logSecurityEvent('info', 'Data retention cleanup performed', {
              action: 'data_retention_cleanup',
              tenantId: tenant.id,
              tenantSlug: tenant.slug,
              retentionDays: effectiveRetentionDays,
              cutoffDate: cutoffDate.toISOString(),
              deletedDetections,
              deletedHeartbeats,
              deletedLogs
            });
          }

        } catch (error) {
          console.error(`❌ Error cleaning tenant ${tenant.slug}:`, error);
          
          securityLogger.logSecurityEvent('error', 'Data retention cleanup failed', {
            action: 'data_retention_cleanup_error',
            tenantId: tenant.id,
            tenantSlug: tenant.slug,
            error: error.message,
            stack: error.stack
          });
        }
      }

      const duration = Date.now() - startTime;
      this.lastCleanup = new Date();
      this.cleanupStats.totalRuns++;
      this.cleanupStats.totalDetectionsDeleted += totalDetectionsDeleted;
      this.cleanupStats.totalHeartbeatsDeleted += totalHeartbeatsDeleted;
      this.cleanupStats.totalLogsDeleted += totalLogsDeleted;
      this.cleanupStats.lastRunDuration = duration;

      console.log(`✅ Data retention cleanup completed in ${duration}ms`);
      console.log(`📊 Total deleted: ${totalDetectionsDeleted} detections, ${totalHeartbeatsDeleted} heartbeats, ${totalLogsDeleted} logs`);

      // Log cleanup summary
      securityLogger.logSecurityEvent('info', 'Data retention cleanup completed', {
        action: 'data_retention_cleanup_summary',
        duration,
        tenantsProcessed: tenants.length,
        totalDetectionsDeleted,
        totalHeartbeatsDeleted,
        totalLogsDeleted,
        timestamp: new Date().toISOString()
      });

    } catch (error) {
      console.error('❌ Data retention cleanup failed:', error);
      
      securityLogger.logSecurityEvent('error', 'Data retention cleanup service failed', {
        action: 'data_retention_service_error',
        error: error.message,
        stack: error.stack,
        timestamp: new Date().toISOString()
      });
    } finally {
      this.isRunning = false;
    }
  }

  /**
   * Get cleanup statistics
   */
  getStats() {
    return {
      ...this.cleanupStats,
      isRunning: this.isRunning,
      lastCleanup: this.lastCleanup,
      nextScheduledRun: '2:00 AM UTC daily'
    };
  }

  /**
   * Manually trigger cleanup (for testing/admin use)
   */
  async triggerManualCleanup() {
    console.log('🔧 Manual data retention cleanup triggered');
    await this.performCleanup();
  }

  /**
   * Preview what would be deleted for a specific tenant
   */
  async previewCleanup(tenantId) {
    try {
      const { Tenant, DroneDetection, Heartbeat, SecurityLog } = require('../models');
      
      const tenant = await Tenant.findByPk(tenantId);
      if (!tenant) {
        throw new Error('Tenant not found');
      }

      const retentionDays = tenant.features?.data_retention_days || 90;
      
      if (retentionDays === -1) {
        return {
          tenantSlug: tenant.slug,
          retentionDays: 'unlimited',
          toDelete: {
            detections: 0,
            heartbeats: 0,
            logs: 0
          }
        };
      }

      const cutoffDate = new Date();
      cutoffDate.setDate(cutoffDate.getDate() - retentionDays);

      const [detectionsCount, heartbeatsCount] = await Promise.all([
        DroneDetection.count({
          where: {
            tenant_id: tenant.id,
            timestamp: { [Op.lt]: cutoffDate }
          }
        }),
        Heartbeat.count({
          where: {
            tenant_id: tenant.id,
            timestamp: { [Op.lt]: cutoffDate }
          }
        })
      ]);

      let logsCount = 0;
      try {
        logsCount = await SecurityLog.count({
          where: {
            tenant_id: tenant.id,
            timestamp: { [Op.lt]: cutoffDate }
          }
        });
      } catch (error) {
        // SecurityLog might not have tenant_id
      }

      return {
        tenantSlug: tenant.slug,
        retentionDays,
        cutoffDate: cutoffDate.toISOString(),
        toDelete: {
          detections: detectionsCount,
          heartbeats: heartbeatsCount,
          logs: logsCount
        }
      };

    } catch (error) {
      console.error('Error previewing cleanup:', error);
      throw error;
    }
  }
}

module.exports = DataRetentionService;