Limit the number of logs to 10 millions
MODIFY
src/scheduler/tasks/LoggingDatabaseTableCleanupTask.ts
src/scheduler/tasks/LoggingDatabaseTableCleanupTask.ts
@@ -9,34 +9,42 @@ import PerformanceStorage from '../../storage/mongodb/PerformanceStorage';
import SchedulerTask from '../SchedulerTask';
import { ServerAction } from '../../types/Server';
import Tenant from '../../types/Tenant';
+import global from './../../types/GlobalType';
import moment from 'moment';
const MODULE_NAME = 'LoggingDatabaseTableCleanupTask';
export default class LoggingDatabaseTableCleanupTask extends SchedulerTask {
public async run(name: string, config: TaskConfig): Promise<void> {
- // Delete
- await this.deleteLogs(Constants.DEFAULT_TENANT_OBJECT, config);
- // Delete Perfs Records
- await this.deletePerformanceRecords(Constants.DEFAULT_TENANT, config);
+ // Delete Default Tenant Logs
+ await this.deleteLogs(Constants.DEFAULT_TENANT_OBJECT);
+ // Delete Default Tenant Perfs Records
+ await this.deletePerformanceRecords(Constants.DEFAULT_TENANT_OBJECT);
// Call for all Tenants
await super.run(name, config);
}
async processTenant(tenant: Tenant, config: LoggingDatabaseTableCleanupTaskConfig): Promise<void> {
- // Delete Logs
- await this.deleteLogs(tenant, config);
+ // Delete Tenant Logs
+ await this.deleteLogs(tenant);
}
- private async deleteLogs(tenant: Tenant, config: LoggingDatabaseTableCleanupTaskConfig) {
+ private async deleteLogs(tenant: Tenant) {
// Get the lock
const logsCleanUpLock = LockingManager.createExclusiveLock(tenant.id, LockEntity.LOGGING, 'cleanup');
if (await LockingManager.acquire(logsCleanUpLock)) {
try {
+ const lastLogMDB = global.database.getCollection(tenant.id, 'logs').find({})
+ .sort({ timestamp: -1 })
+ .skip(10 * 1000 * 1000)
+ .limit(1)
+ .project({ timestamp: 1 });
+ const lastLog = await lastLogMDB.toArray();
+ if (lastLog.length > 0) {
// Delete Standard Logs
- const deleteUpToDate = moment().subtract(config.retentionPeriodWeeks, 'w').toDate();
+ const deleteUpToDate = lastLog[0].timestamp as Date;
// Delete
- let result = await LoggingStorage.deleteLogs(tenant, deleteUpToDate);
+ const result = await LoggingStorage.deleteLogs(tenant, deleteUpToDate);
// Ok?
if (result.acknowledged) {
await Logging.logSecurityInfo({
@@ -54,26 +62,6 @@ export default class LoggingDatabaseTableCleanupTask extends SchedulerTask {
detailedMessages: { result }
});
}
- // Delete Security Logs
- const securityDeleteUpToDate: Date = moment().subtract(config.securityRetentionPeriodWeeks, 'w').startOf('week').toDate();
- // Delete
- result = await LoggingStorage.deleteSecurityLogs(tenant, securityDeleteUpToDate);
- // Ok?
- if (result.acknowledged) {
- await Logging.logSecurityInfo({
- tenantID: tenant.id,
- action: ServerAction.LOGS_CLEANUP,
- module: MODULE_NAME, method: 'deleteLogs',
- message: `${result.deletedCount} Security Log(s) have been deleted before '${moment(securityDeleteUpToDate).format('DD/MM/YYYY h:mm A')}'`
- });
- } else {
- await Logging.logSecurityError({
- tenantID: tenant.id,
- action: ServerAction.LOGS_CLEANUP,
- module: MODULE_NAME, method: 'deleteLogs',
- message: `An error occurred when deleting Security Logs before '${moment(securityDeleteUpToDate).format('DD/MM/YYYY h:mm A')}'`,
- detailedMessages: { result }
- });
}
} catch (error) {
await Logging.logActionExceptionMessage(tenant.id, ServerAction.LOGS_CLEANUP, error);
@@ -83,35 +71,43 @@ export default class LoggingDatabaseTableCleanupTask extends SchedulerTask {
}
}
- private async deletePerformanceRecords(tenantID: string, config: LoggingDatabaseTableCleanupTaskConfig) {
+ private async deletePerformanceRecords(tenant: Tenant) {
// Get the lock
- const performanceCleanUpLock = LockingManager.createExclusiveLock(tenantID, LockEntity.PERFORMANCE, 'cleanup');
+ const performanceCleanUpLock = LockingManager.createExclusiveLock(tenant.id, LockEntity.PERFORMANCE, 'cleanup');
if (await LockingManager.acquire(performanceCleanUpLock)) {
try {
- // Delete Performance Records (keep only 2 weeks)
- const deleteUpToDate = moment().subtract(2, 'w').toDate();
+ const lastLogMDB = global.database.getCollection(tenant.id, 'performances').find({})
+ .sort({ timestamp: -1 })
+ .skip(20 * 1000 * 1000)
+ .limit(1)
+ .project({ timestamp: 1 });
+ const lastLog = await lastLogMDB.toArray();
+ if (lastLog.length > 0) {
+ // Delete Standard Logs
+ const deleteUpToDate = lastLog[0].timestamp as Date;
// Delete
const result = await PerformanceStorage.deletePerformanceRecords({ deleteUpToDate });
// Ok?
if (result.acknowledged) {
await Logging.logSecurityInfo({
- tenantID: tenantID,
+ tenantID: tenant.id,
action: ServerAction.PERFORMANCES_CLEANUP,
module: MODULE_NAME, method: 'deletePerformanceRecords',
message: `${result.deletedCount} Performance Record(s) have been deleted successfully before '${moment(deleteUpToDate).format('DD/MM/YYYY h:mm A')}'`
});
} else {
await Logging.logError({
- tenantID: tenantID,
+ tenantID: tenant.id,
action: ServerAction.PERFORMANCES_CLEANUP,
module: MODULE_NAME, method: 'deletePerformanceRecords',
message: `An error occurred when deleting Performance Record(s) before '${moment(deleteUpToDate).format('DD/MM/YYYY h:mm A')}'`,
detailedMessages: { result }
});
}
+ }
} catch (error) {
// Log error
- await Logging.logActionExceptionMessage(tenantID, ServerAction.PERFORMANCES_CLEANUP, error);
+ await Logging.logActionExceptionMessage(tenant.id, ServerAction.PERFORMANCES_CLEANUP, error);
} finally {
// Release the lock
await LockingManager.release(performanceCleanUpLock);

MODIFY
src/storage/mongodb/LoggingStorage.ts
src/storage/mongodb/LoggingStorage.ts
@@ -15,31 +15,6 @@ export default class LoggingStorage {
DatabaseUtils.checkTenantObject(tenant);
// Build filter
const filters: FilterParams = {};
- // Do Not Delete Security Logs
- filters.type = {};
- filters.type.$ne = 'S';
- // Date provided?
- if (deleteUpToDate) {
- filters.timestamp = {};
- filters.timestamp.$lte = Utils.convertToDate(deleteUpToDate);
- } else {
- return;
- }
- // Delete Logs
- const result = await global.database.getCollection<Log>(tenant.id, 'logs')
- .deleteMany(filters);
- // Return the result
- return { acknowledged: result.acknowledged, deletedCount: result.deletedCount };
- }
-
- public static async deleteSecurityLogs(tenant: Tenant, deleteUpToDate: Date): Promise<DeletedResult> {
- // Check Tenant
- DatabaseUtils.checkTenantObject(tenant);
- // Build filter
- const filters: FilterParams = {};
- // Delete Only Security Logs
- filters.type = {};
- filters.type.$eq = 'S';
// Date provided?
if (deleteUpToDate) {
filters.timestamp = {};

Limit the number of logs to 10 millions