Fix jwt-token
This commit is contained in:
292
server/services/dataRetention.js
Normal file
292
server/services/dataRetention.js
Normal file
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Data Retention Service
|
||||
* Automatically cleans up old data based on tenant retention policies
|
||||
*/
|
||||
|
||||
const cron = require('node-cron');
|
||||
const { Op } = require('sequelize');
|
||||
const { securityLogger } = require('../middleware/logger');
|
||||
|
||||
class DataRetentionService {
|
||||
constructor() {
|
||||
this.isRunning = false;
|
||||
this.lastCleanup = null;
|
||||
this.cleanupStats = {
|
||||
totalRuns: 0,
|
||||
totalDetectionsDeleted: 0,
|
||||
totalHeartbeatsDeleted: 0,
|
||||
totalLogsDeleted: 0,
|
||||
lastRunDuration: 0
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the data retention cleanup service
|
||||
* Runs daily at 2 AM
|
||||
*/
|
||||
start() {
|
||||
console.log('🗂️ Starting Data Retention Service...');
|
||||
|
||||
// Run daily at 2:00 AM
|
||||
cron.schedule('0 2 * * *', async () => {
|
||||
await this.performCleanup();
|
||||
}, {
|
||||
scheduled: true,
|
||||
timezone: "UTC"
|
||||
});
|
||||
|
||||
// Also run immediately if NODE_ENV is development
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
console.log('🧹 Development mode: Running initial data retention cleanup...');
|
||||
setTimeout(() => this.performCleanup(), 5000); // Wait 5 seconds for app to fully start
|
||||
}
|
||||
|
||||
console.log('✅ Data Retention Service started - will run daily at 2:00 AM UTC');
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform cleanup for all tenants
|
||||
*/
|
||||
async performCleanup() {
|
||||
if (this.isRunning) {
|
||||
console.log('⏳ Data retention cleanup already running, skipping...');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
console.log('🧹 Starting data retention cleanup...');
|
||||
|
||||
const { Tenant, DroneDetection, Heartbeat, SecurityLog } = require('../models');
|
||||
|
||||
// Get all tenants with their retention policies
|
||||
const tenants = await Tenant.findAll({
|
||||
attributes: ['id', 'slug', 'features'],
|
||||
where: {
|
||||
is_active: true
|
||||
}
|
||||
});
|
||||
|
||||
let totalDetectionsDeleted = 0;
|
||||
let totalHeartbeatsDeleted = 0;
|
||||
let totalLogsDeleted = 0;
|
||||
|
||||
for (const tenant of tenants) {
|
||||
const retentionDays = tenant.features?.data_retention_days;
|
||||
|
||||
// Skip if unlimited retention (-1)
|
||||
if (retentionDays === -1) {
|
||||
console.log(`⏭️ Skipping tenant ${tenant.slug} - unlimited retention`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Default to 90 days if not specified
|
||||
const effectiveRetentionDays = retentionDays || 90;
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - effectiveRetentionDays);
|
||||
|
||||
console.log(`🧹 Cleaning tenant ${tenant.slug} - removing data older than ${effectiveRetentionDays} days (before ${cutoffDate.toISOString()})`);
|
||||
|
||||
try {
|
||||
// Clean up drone detections
|
||||
const deletedDetections = await DroneDetection.destroy({
|
||||
where: {
|
||||
tenant_id: tenant.id,
|
||||
timestamp: {
|
||||
[Op.lt]: cutoffDate
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Clean up heartbeats
|
||||
const deletedHeartbeats = await Heartbeat.destroy({
|
||||
where: {
|
||||
tenant_id: tenant.id,
|
||||
timestamp: {
|
||||
[Op.lt]: cutoffDate
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Clean up security logs (if they have tenant_id)
|
||||
let deletedLogs = 0;
|
||||
try {
|
||||
deletedLogs = await SecurityLog.destroy({
|
||||
where: {
|
||||
tenant_id: tenant.id,
|
||||
timestamp: {
|
||||
[Op.lt]: cutoffDate
|
||||
}
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
// SecurityLog might not have tenant_id field, skip if error
|
||||
console.log(`⚠️ Skipping security logs cleanup for tenant ${tenant.slug}: ${error.message}`);
|
||||
}
|
||||
|
||||
totalDetectionsDeleted += deletedDetections;
|
||||
totalHeartbeatsDeleted += deletedHeartbeats;
|
||||
totalLogsDeleted += deletedLogs;
|
||||
|
||||
console.log(`✅ Tenant ${tenant.slug}: Deleted ${deletedDetections} detections, ${deletedHeartbeats} heartbeats, ${deletedLogs} logs`);
|
||||
|
||||
// Log significant cleanup events
|
||||
if (deletedDetections > 100 || deletedHeartbeats > 100) {
|
||||
securityLogger.logSecurityEvent('info', 'Data retention cleanup performed', {
|
||||
action: 'data_retention_cleanup',
|
||||
tenantId: tenant.id,
|
||||
tenantSlug: tenant.slug,
|
||||
retentionDays: effectiveRetentionDays,
|
||||
cutoffDate: cutoffDate.toISOString(),
|
||||
deletedDetections,
|
||||
deletedHeartbeats,
|
||||
deletedLogs
|
||||
});
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error cleaning tenant ${tenant.slug}:`, error);
|
||||
|
||||
securityLogger.logSecurityEvent('error', 'Data retention cleanup failed', {
|
||||
action: 'data_retention_cleanup_error',
|
||||
tenantId: tenant.id,
|
||||
tenantSlug: tenant.slug,
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
this.lastCleanup = new Date();
|
||||
this.cleanupStats.totalRuns++;
|
||||
this.cleanupStats.totalDetectionsDeleted += totalDetectionsDeleted;
|
||||
this.cleanupStats.totalHeartbeatsDeleted += totalHeartbeatsDeleted;
|
||||
this.cleanupStats.totalLogsDeleted += totalLogsDeleted;
|
||||
this.cleanupStats.lastRunDuration = duration;
|
||||
|
||||
console.log(`✅ Data retention cleanup completed in ${duration}ms`);
|
||||
console.log(`📊 Total deleted: ${totalDetectionsDeleted} detections, ${totalHeartbeatsDeleted} heartbeats, ${totalLogsDeleted} logs`);
|
||||
|
||||
// Log cleanup summary
|
||||
securityLogger.logSecurityEvent('info', 'Data retention cleanup completed', {
|
||||
action: 'data_retention_cleanup_summary',
|
||||
duration,
|
||||
tenantsProcessed: tenants.length,
|
||||
totalDetectionsDeleted,
|
||||
totalHeartbeatsDeleted,
|
||||
totalLogsDeleted,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Data retention cleanup failed:', error);
|
||||
|
||||
securityLogger.logSecurityEvent('error', 'Data retention cleanup service failed', {
|
||||
action: 'data_retention_service_error',
|
||||
error: error.message,
|
||||
stack: error.stack,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
} finally {
|
||||
this.isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cleanup statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
...this.cleanupStats,
|
||||
isRunning: this.isRunning,
|
||||
lastCleanup: this.lastCleanup,
|
||||
nextScheduledRun: '2:00 AM UTC daily'
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually trigger cleanup (for testing/admin use)
|
||||
*/
|
||||
async triggerManualCleanup() {
|
||||
console.log('🔧 Manual data retention cleanup triggered');
|
||||
await this.performCleanup();
|
||||
}
|
||||
|
||||
/**
|
||||
* Preview what would be deleted for a specific tenant
|
||||
*/
|
||||
async previewCleanup(tenantId) {
|
||||
try {
|
||||
const { Tenant, DroneDetection, Heartbeat, SecurityLog } = require('../models');
|
||||
|
||||
const tenant = await Tenant.findByPk(tenantId);
|
||||
if (!tenant) {
|
||||
throw new Error('Tenant not found');
|
||||
}
|
||||
|
||||
const retentionDays = tenant.features?.data_retention_days || 90;
|
||||
|
||||
if (retentionDays === -1) {
|
||||
return {
|
||||
tenantSlug: tenant.slug,
|
||||
retentionDays: 'unlimited',
|
||||
toDelete: {
|
||||
detections: 0,
|
||||
heartbeats: 0,
|
||||
logs: 0
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
|
||||
const [detectionsCount, heartbeatsCount] = await Promise.all([
|
||||
DroneDetection.count({
|
||||
where: {
|
||||
tenant_id: tenant.id,
|
||||
timestamp: { [Op.lt]: cutoffDate }
|
||||
}
|
||||
}),
|
||||
Heartbeat.count({
|
||||
where: {
|
||||
tenant_id: tenant.id,
|
||||
timestamp: { [Op.lt]: cutoffDate }
|
||||
}
|
||||
})
|
||||
]);
|
||||
|
||||
let logsCount = 0;
|
||||
try {
|
||||
logsCount = await SecurityLog.count({
|
||||
where: {
|
||||
tenant_id: tenant.id,
|
||||
timestamp: { [Op.lt]: cutoffDate }
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
// SecurityLog might not have tenant_id
|
||||
}
|
||||
|
||||
return {
|
||||
tenantSlug: tenant.slug,
|
||||
retentionDays,
|
||||
cutoffDate: cutoffDate.toISOString(),
|
||||
toDelete: {
|
||||
detections: detectionsCount,
|
||||
heartbeats: heartbeatsCount,
|
||||
logs: logsCount
|
||||
}
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error previewing cleanup:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = DataRetentionService;
|
||||
Reference in New Issue
Block a user