drone-detector/data-retention-service/index.js

/**
 * Data Retention Service
 * Standalone microservice for automated data cleanup
 */

const cron = require('node-cron');
const { Op } = require('sequelize');
require('dotenv').config();

// Initialize database connection
const { initializeDatabase, getModels } = require('./database');

class DataRetentionService {
  constructor() {
    this.isRunning = false;
    this.lastCleanup = null;
    this.cleanupStats = {
      totalRuns: 0,
      totalDetectionsDeleted: 0,
      totalHeartbeatsDeleted: 0,
      totalLogsDeleted: 0,
      lastRunDuration: 0,
      errors: []
    };
  }

  /**
   * Start the data retention cleanup service
   */
  async start() {
    console.log('🗂️  Starting Data Retention Service...');
    console.log(`📅 Environment: ${process.env.NODE_ENV || 'development'}`);
    console.log(`💾 Database: ${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`);
    
    try {
      // Initialize database connection
      await initializeDatabase();
      console.log('✅ Database connection established');
      
      // Schedule daily cleanup at 2:00 AM UTC
      cron.schedule('0 2 * * *', async () => {
        await this.performCleanup();
      }, {
        scheduled: true,
        timezone: "UTC"
      });

      console.log('⏰ Scheduled cleanup: Daily at 2:00 AM UTC');

      // Run immediate cleanup in development or if IMMEDIATE_CLEANUP is set
      if (process.env.NODE_ENV === 'development' || process.env.IMMEDIATE_CLEANUP === 'true') {
        console.log('🧹 Running immediate cleanup...');
        setTimeout(() => this.performCleanup(), 5000);
      }

      // Health check endpoint simulation
      setInterval(() => {
        this.logHealthStatus();
      }, 60000); // Every minute

      console.log('✅ Data Retention Service started successfully');
      
    } catch (error) {
      console.error('❌ Failed to start Data Retention Service:', error);
      process.exit(1);
    }
  }

  /**
   * Perform cleanup for all tenants
   */
  async performCleanup() {
    if (this.isRunning) {
      console.log('⏳ Data retention cleanup already running, skipping...');
      return;
    }

    this.isRunning = true;
    const startTime = Date.now();
    
    try {
      console.log('🧹 Starting data retention cleanup...');
      console.log(`⏰ Cleanup started at: ${new Date().toISOString()}`);
      
      const { Tenant, DroneDetection, Heartbeat, SecurityLog } = await getModels();
      
      // Get all active tenants with their retention policies
      const tenants = await Tenant.findAll({
        attributes: ['id', 'slug', 'features'],
        where: {
          is_active: true
        }
      });

      console.log(`🏢 Found ${tenants.length} active tenants to process`);

      let totalDetectionsDeleted = 0;
      let totalHeartbeatsDeleted = 0;
      let totalLogsDeleted = 0;
      let errors = [];

      for (const tenant of tenants) {
        try {
          const result = await this.cleanupTenant(tenant);
          totalDetectionsDeleted += result.detections;
          totalHeartbeatsDeleted += result.heartbeats;
          totalLogsDeleted += result.logs;
        } catch (error) {
          console.error(`❌ Error cleaning tenant ${tenant.slug}:`, error);
          errors.push({
            tenantSlug: tenant.slug,
            error: error.message,
            timestamp: new Date().toISOString()
          });
        }
      }

      const duration = Date.now() - startTime;
      this.lastCleanup = new Date();
      this.cleanupStats.totalRuns++;
      this.cleanupStats.totalDetectionsDeleted += totalDetectionsDeleted;
      this.cleanupStats.totalHeartbeatsDeleted += totalHeartbeatsDeleted;
      this.cleanupStats.totalLogsDeleted += totalLogsDeleted;
      this.cleanupStats.lastRunDuration = duration;
      this.cleanupStats.errors = errors;

      console.log('✅ Data retention cleanup completed');
      console.log(`⏱️  Duration: ${duration}ms`);
      console.log(`📊 Deleted: ${totalDetectionsDeleted} detections, ${totalHeartbeatsDeleted} heartbeats, ${totalLogsDeleted} logs`);
      
      if (errors.length > 0) {
        console.log(`⚠️  Errors encountered: ${errors.length}`);
        errors.forEach(err => console.log(`   - ${err.tenantSlug}: ${err.error}`));
      }

    } catch (error) {
      console.error('❌ Data retention cleanup failed:', error);
      this.cleanupStats.errors.push({
        error: error.message,
        timestamp: new Date().toISOString()
      });
    } finally {
      this.isRunning = false;
    }
  }

  /**
   * Clean up data for a specific tenant
   */
  async cleanupTenant(tenant) {
    const retentionDays = tenant.features?.data_retention_days;
    
    // Skip if unlimited retention (-1)
    if (retentionDays === -1) {
      console.log(`⏭️  Skipping tenant ${tenant.slug} - unlimited retention`);
      return { detections: 0, heartbeats: 0, logs: 0 };
    }

    // Default to 90 days if not specified
    const effectiveRetentionDays = retentionDays || 90;
    const cutoffDate = new Date();
    cutoffDate.setDate(cutoffDate.getDate() - effectiveRetentionDays);

    console.log(`🧹 Cleaning tenant ${tenant.slug} - removing data older than ${effectiveRetentionDays} days (before ${cutoffDate.toISOString()})`);

    const { DroneDetection, Heartbeat, SecurityLog } = await getModels();

    // Clean up drone detections
    const deletedDetections = await DroneDetection.destroy({
      where: {
        tenant_id: tenant.id,
        timestamp: {
          [Op.lt]: cutoffDate
        }
      }
    });

    // Clean up heartbeats
    const deletedHeartbeats = await Heartbeat.destroy({
      where: {
        tenant_id: tenant.id,
        timestamp: {
          [Op.lt]: cutoffDate
        }
      }
    });

    // Clean up security logs (if they have tenant_id)
    let deletedLogs = 0;
    try {
      deletedLogs = await SecurityLog.destroy({
        where: {
          tenant_id: tenant.id,
          timestamp: {
            [Op.lt]: cutoffDate
          }
        }
      });
    } catch (error) {
      // SecurityLog might not have tenant_id field
      console.log(`⚠️  Skipping security logs for tenant ${tenant.slug}: ${error.message}`);
    }

    console.log(`✅ Tenant ${tenant.slug}: Deleted ${deletedDetections} detections, ${deletedHeartbeats} heartbeats, ${deletedLogs} logs`);

    return {
      detections: deletedDetections,
      heartbeats: deletedHeartbeats,
      logs: deletedLogs
    };
  }

  /**
   * Log health status
   */
  logHealthStatus() {
    const memUsage = process.memoryUsage();
    const uptime = process.uptime();
    
    console.log(`💚 Health Check - Uptime: ${Math.floor(uptime)}s, Memory: ${Math.round(memUsage.heapUsed / 1024 / 1024)}MB, Last Cleanup: ${this.lastCleanup ? this.lastCleanup.toISOString() : 'Never'}`);
  }

  /**
   * Get service statistics
   */
  getStats() {
    return {
      ...this.cleanupStats,
      isRunning: this.isRunning,
      lastCleanup: this.lastCleanup,
      uptime: process.uptime(),
      memoryUsage: process.memoryUsage(),
      nextScheduledRun: '2:00 AM UTC daily'
    };
  }

  /**
   * Graceful shutdown
   */
  async shutdown() {
    console.log('🔄 Graceful shutdown initiated...');
    
    // Wait for current cleanup to finish
    while (this.isRunning) {
      console.log('⏳ Waiting for cleanup to finish...');
      await new Promise(resolve => setTimeout(resolve, 1000));
    }
    
    console.log('✅ Data Retention Service stopped');
    process.exit(0);
  }
}

// Initialize and start the service
const service = new DataRetentionService();

// Handle graceful shutdown
process.on('SIGTERM', () => service.shutdown());
process.on('SIGINT', () => service.shutdown());

// Start the service
service.start().catch(error => {
  console.error('Failed to start service:', error);
  process.exit(1);
});

module.exports = DataRetentionService;