Added TTL indexes for logs
MODIFY
src/storage/mongodb/MongoDBStorage.ts
src/storage/mongodb/MongoDBStorage.ts
-import { ChangeStream, ChangeStreamOptions, ClientSession, Collection, Db, GridFSBucket, MongoClient, ReadPreferenceMode } from 'mongodb';
+import { ChangeStream, ChangeStreamOptions, ClientSession, Collection, CreateIndexesOptions, Db, GridFSBucket, IndexSpecification, MongoClient, ReadPreferenceMode } from 'mongodb';
import mongoUriBuilder, { MongoUriConfig } from 'mongo-uri-builder';
import BackendError from '../../exception/BackendError';
@@ -13,7 +13,6 @@ import { ServerAction } from '../../types/Server';
import StorageConfiguration from '../../types/configuration/StorageConfiguration';
import Utils from '../../utils/Utils';
import chalk from 'chalk';
-import cluster from 'cluster';
import urlencode from 'urlencode';
const MODULE_NAME = 'MongoDBStorage';
@@ -88,7 +87,7 @@ export default class MongoDBStorage {
]);
// Logs
await this.handleIndexesInCollection(tenantID, 'logs', [
- { fields: { timestamp: 1 } },
+ { fields: { timestamp: 1 }, options: { expireAfterSeconds: 14 * 24 * 3600 } },
{ fields: { type: 1, timestamp: 1 } },
{ fields: { action: 1, timestamp: 1 } },
{ fields: { level: 1, timestamp: 1 } },
@@ -202,7 +201,7 @@ export default class MongoDBStorage {
}
public async start(): Promise<void> {
- console.log(`Connecting to '${this.dbConfig.implementation}' ${cluster.isWorker ? 'in worker ' + cluster.worker.id.toString() : 'in master'}...`);
+ console.log(`Connecting to '${this.dbConfig.implementation}'...`);
// Build EVSE URL
let mongoUrl: string;
// URI provided?
@@ -245,7 +244,7 @@ export default class MongoDBStorage {
if (this.migrationConfig.active) {
await this.checkDatabase();
}
- console.log(`Connected to '${this.dbConfig.implementation}' successfully ${cluster.isWorker ? 'in worker ' + cluster.worker.id.toString() : 'in master'}`);
+ console.log(`Connected to '${this.dbConfig.implementation}' successfully`);
}
private async checkDatabase(): Promise<void> {
@@ -266,6 +265,7 @@ export default class MongoDBStorage {
}
private async handleCheckDefaultTenant() {
+ try {
// Database creation Lock
const databaseLock = LockingManager.createExclusiveLock(Constants.DEFAULT_TENANT, LockEntity.DATABASE, 'check-database');
if (await LockingManager.acquire(databaseLock)) {
@@ -291,7 +291,7 @@ export default class MongoDBStorage {
]);
// Logs
await this.handleIndexesInCollection(Constants.DEFAULT_TENANT, 'logs', [
- { fields: { timestamp: 1 } },
+ { fields: { timestamp: 1 }, options: { expireAfterSeconds: 14 * 24 * 3600 } },
{ fields: { type: 1, timestamp: 1 } },
{ fields: { action: 1, timestamp: 1 } },
{ fields: { level: 1, timestamp: 1 } },
@@ -304,6 +304,17 @@ export default class MongoDBStorage {
await LockingManager.release(databaseLock);
}
}
+ } catch (error) {
+ const message = 'Error while checking Database in tenant \'default\'';
+ Utils.isDevelopmentEnv() && console.log(chalk.red(message));
+ await Logging.logError({
+ tenantID: Constants.DEFAULT_TENANT,
+ action: ServerAction.MONGO_DB,
+ module: MODULE_NAME, method: 'handleIndexesInCollection',
+ message,
+ detailedMessages: { error: error.stack }
+ });
+ }
}
private async handleCheckTenants() {
@@ -312,6 +323,7 @@ export default class MongoDBStorage {
.find({}).toArray();
const tenantIds = tenantsMDB.map((t): string => t._id.toString());
for (const tenantId of tenantIds) {
+ try {
// Database creation Lock
const databaseLock = LockingManager.createExclusiveLock(tenantId, LockEntity.DATABASE, 'check-database');
if (await LockingManager.acquire(databaseLock)) {
@@ -323,11 +335,22 @@ export default class MongoDBStorage {
await LockingManager.release(databaseLock);
}
}
+ } catch (error) {
+ const message = `Error while checking Database in tenant '${tenantId}'`;
+ Utils.isDevelopmentEnv() && console.log(chalk.red(message));
+ await Logging.logError({
+ tenantID: Constants.DEFAULT_TENANT,
+ action: ServerAction.MONGO_DB,
+ module: MODULE_NAME, method: 'handleIndexesInCollection',
+ message,
+ detailedMessages: { error: error.stack }
+ });
+ }
}
}
private async handleIndexesInCollection(tenantID: string,
- name: string, indexes?: { fields: any; options?: any }[]): Promise<void> {
+ name: string, indexes?: { fields: IndexSpecification; options?: CreateIndexesOptions }[]): Promise<void> {
// Safety check
if (!this.db) {
throw new BackendError({
@@ -349,7 +372,7 @@ export default class MongoDBStorage {
await this.db.createCollection(tenantCollectionName);
} catch (error) {
const message = `Error in creating collection '${tenantID}.${tenantCollectionName}': ${error.message as string}`;
- console.error(chalk.red(message));
+ Utils.isDevelopmentEnv() && console.error(chalk.red(message));
await Logging.logError({
tenantID: Constants.DEFAULT_TENANT,
action: ServerAction.MONGO_DB,
@@ -369,18 +392,21 @@ export default class MongoDBStorage {
continue;
}
let foundIndex = indexes.find((index) => this.buildIndexName(index.fields) === databaseIndex.name);
- // Check DB unique index
- const databaseIndexISUnique = !!databaseIndex?.unique;
- const indexIsUnique = !!foundIndex?.options?.unique;
- if (indexIsUnique !== databaseIndexISUnique) {
- // Delete the index
+ // Check DB 'unique' index option
+ if (!Utils.areObjectPropertiesEqual(databaseIndex, foundIndex?.options, 'unique')) {
+ // Force not found: Create
+ foundIndex = null;
+ }
+ // Check DB 'expireAfterSeconds' index option
+ if (!Utils.areObjectPropertiesEqual(databaseIndex, foundIndex?.options, 'expireAfterSeconds')) {
+ // Force nound: Create
foundIndex = null;
}
// Delete the index
if (!foundIndex) {
if (Utils.isDevelopmentEnv()) {
const message = `Drop index '${databaseIndex.name}' in collection ${tenantCollectionName}`;
- console.log(message);
+ Utils.isDevelopmentEnv() && console.log(message);
await Logging.logInfo({
tenantID: Constants.DEFAULT_TENANT,
action: ServerAction.MONGO_DB,
@@ -393,7 +419,7 @@ export default class MongoDBStorage {
await this.db.collection(tenantCollectionName).dropIndex(databaseIndex.key);
} catch (error) {
const message = `Error in dropping index '${databaseIndex.name}' in '${tenantCollectionName}': ${error.message}`;
- console.error(chalk.red(message));
+ Utils.isDevelopmentEnv() && console.error(chalk.red(message));
await Logging.logError({
tenantID: Constants.DEFAULT_TENANT,
action: ServerAction.MONGO_DB,
@@ -413,7 +439,7 @@ export default class MongoDBStorage {
if (!foundDatabaseIndex) {
if (Utils.isDevelopmentEnv()) {
const message = `Create index ${JSON.stringify(index)} in collection ${tenantCollectionName}`;
- console.log(message);
+ Utils.isDevelopmentEnv() && console.log(message);
await Logging.logInfo({
tenantID: Constants.DEFAULT_TENANT,
action: ServerAction.MONGO_DB,
@@ -426,7 +452,7 @@ export default class MongoDBStorage {
await this.db.collection(tenantCollectionName).createIndex(index.fields, index.options);
} catch (error) {
const message = `Error in creating index '${JSON.stringify(index.fields)}' with options '${JSON.stringify(index.options)}' in '${tenantCollectionName}': ${error.message as string}`;
- console.error(chalk.red(message));
+ Utils.isDevelopmentEnv() && console.error(chalk.red(message));
await Logging.logError({
tenantID: Constants.DEFAULT_TENANT,
action: ServerAction.MONGO_DB,
@@ -440,7 +466,7 @@ export default class MongoDBStorage {
}
} catch (error) {
const message = `Unexpected error in handling Collection '${tenantID}.${name}': ${error.message as string}`;
- console.error(chalk.red(message));
+ Utils.isDevelopmentEnv() && console.error(chalk.red(message));
await Logging.logError({
tenantID: Constants.DEFAULT_TENANT,
action: ServerAction.MONGO_DB,
@@ -451,9 +477,9 @@ export default class MongoDBStorage {
}
}
- private buildIndexName(indexes: { [key: string]: string }): string {
+ private buildIndexName(indexes: IndexSpecification): string {
const indexNameValues: string[] = [];
- for (const indexKey in indexes) {
+ for (const indexKey in indexes as any) {
indexNameValues.push(indexKey);
indexNameValues.push(indexes[indexKey]);
}

MODIFY
src/utils/Utils.ts
src/utils/Utils.ts
@@ -241,6 +241,15 @@ export default class Utils {
return InactivityStatus.ERROR;
}
+ public static areObjectPropertiesEqual(objCmp1: any = {}, objCmp2: any = {}, key: string): boolean {
+ // Check DB expireAfterSeconds index
+ if ((Utils.objectHasProperty(objCmp1, key) !== Utils.objectHasProperty(objCmp2, key)) ||
+ (objCmp1[key] !== objCmp2[key])) {
+ return false;
+ }
+ return true;
+ }
+
public static objectHasProperty(obj: any, key: string): boolean {
return _.has(obj, key);
}

Added TTL indexes for logs